eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject [02/51] [partial] incubator-eagle git commit: EAGLE-184 Migrate eagle website from https://github.com/eaglemonitoring/eaglemonitoring.github.io to document branch
Date Thu, 03 Mar 2016 18:09:35 GMT
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/eagle-singlenode.json
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/eagle-singlenode.json b/eagle-external/eagle-docker/resource/eagle-singlenode.json
deleted file mode 100644
index a693c7d..0000000
--- a/eagle-external/eagle-docker/resource/eagle-singlenode.json
+++ /dev/null
@@ -1,122 +0,0 @@
-{
-"configurations": [
-    {
-      "hdfs-site": {
-        "dfs.permissions.enabled": "false"
-      },
-      "hive-site": {
-        "javax.jdo.option.ConnectionUserName": "hive",
-        "javax.jdo.option.ConnectionPassword": "hive"
-      }
-    },
-    {
-       "hadoop-env": {
-          "properties" : {
-            "content" : "\r\n# Set Hadoop-specific environment variables here.\r\n\r\n# The only required environment variable is JAVA_HOME.  All others are\r\n# optional.  When running a distributed configuration it is best to\r\n# set JAVA_HOME in this file, so that it is correctly defined on\r\n# remote nodes.\r\n\r\n# The java implementation to use.  Required.\r\nexport JAVA_HOME={{java_home}}\r\nexport HADOOP_HOME_WARN_SUPPRESS=1\r\n\r\n# Hadoop home directory\r\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\r\n\r\n# Hadoop Configuration Directory\r\n\r\n{# this is different for HDP1 #}\r\n# Path to jsvc required by secure HDP 2.0 datanode\r\nexport JSVC_HOME={{jsvc_path}}\r\n\r\n\r\n# The maximum amount of heap to use, in MB. Default is 1000.\r\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\r\n\r\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\r\n\r\n# Extra Java runtime options.  Empty by default.\r\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=tr
 ue ${HADOOP_OPTS}\"\r\n\r\n# Command specific options appended to HADOOP_OPTS when specified\r\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT,KAFKA_HDFS_AUDIT ${HADOOP_NAMENODE_OPTS}\"\r\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d
 %H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\r\n\r\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\r\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\r\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCE
 R_OPTS}\"\r\n\r\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\r\n\r\n# The following applies to multiple commands (fs, dfs, fsck, distcp etc)\r\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\r\n\r\n# On secure datanodes, user to run the datanode as after dropping privileges\r\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\r\n\r\n# Extra ssh options.  Empty by default.\r\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\r\n\r\n# Where log files are stored.  $HADOOP_HOME/logs by default.\r\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\r\n\r\n# History server logs\r\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\r\n\r\n# Where log files are stored in the secure data environment.\r\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\r\n\r\n# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.\r\n# e
 xport HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\r\n\r\n# host:path where hadoop code should be rsync'd from.  Unset by default.\r\n# export HADOOP_MASTER=master:/home/$USER/src/hadoop\r\n\r\n# Seconds to sleep between slave commands.  Unset by default.  This\r\n# can be useful in large clusters, where, e.g., slave rsyncs can\r\n# otherwise arrive faster than the master can service them.\r\n# export HADOOP_SLAVE_SLEEP=0.1\r\n\r\n# The directory where pid files are stored. /tmp by default.\r\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\r\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\r\n\r\n# History server pid\r\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\r\n\r\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\r\n\r\n# A string representing this instance of hadoop. $USER by default.\r\nexport HADOOP_IDENT_STRING=$USER\r\n\r\n# The scheduling priority for daemon processes.  See 'm
 an nice'.\r\n\r\n# export HADOOP_NICENESS=10\r\n\r\n# Use libraries from standard classpath\r\nJAVA_JDBC_LIBS=\"\"\r\n#Add libraries required by mysql connector\r\nfor jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`\r\ndo\r\n  JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\ndone\r\n# Add libraries required by oracle connector\r\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\r\ndo\r\n  JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\r\ndone\r\n# Add libraries required by nodemanager\r\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\r\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}:/usr/hdp/current/eagle/lib/log4jkafka/lib/*\r\n\r\n# added to the HADOOP_CLASSPATH\r\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\r\n  if [ -d \"/etc/tez/conf/\" ]; then\r\n    # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\r\n    export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez
 -client/lib/*:/etc/tez/conf/\r\n  fi\r\nfi\r\n\r\n# Setting path to hdfs command line\r\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\r\n\r\n# Mostly required for hadoop 2.0\r\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\r\n\r\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\r\n"
-	  }
-        }
-    },
-    {
-      "hdfs-log4j": {
-        "properties": {
-          "content" : "\r\n#\r\n# Licensed to the Apache Software Foundation (ASF) under one\r\n# or more contributor license agreements.  See the NOTICE file\r\n# distributed with this work for additional information\r\n# regarding copyright ownership.  The ASF licenses this file\r\n# to you under the Apache License, Version 2.0 (the\r\n# \"License\"); you may not use this file except in compliance\r\n# with the License.  You may obtain a copy of the License at\r\n#\r\n#  http://www.apache.org/licenses/LICENSE-2.0\r\n#\r\n# Unless required by applicable law or agreed to in writing,\r\n# software distributed under the License is distributed on an\r\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\r\n# KIND, either express or implied.  See the License for the\r\n# specific language governing permissions and limitations\r\n# under the License.\r\n#\r\n\r\n\r\n# Define some default values that can be overridden by system properties\r\n# To change daemon root logger use hadoo
 p_root_logger in hadoop-env\r\nhadoop.root.logger=INFO,console\r\nhadoop.log.dir=.\r\nhadoop.log.file=hadoop.log\r\n\r\n\r\n# Define the root logger to the system property \"hadoop.root.logger\".\r\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\r\n\r\n# Logging Threshold\r\nlog4j.threshhold=ALL\r\n\r\n#\r\n# Daily Rolling File Appender\r\n#\r\n\r\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Rollver at midnight\r\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\r\n\r\n# 30-day backup\r\n#log4j.appender.DRFA.MaxBackupIndex=30\r\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\r\n\r\n# Pattern format: Date LogLevel LoggerName LogMessage\r\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n# Debugging Pattern format\r\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n#\r\n# console\r\n# Add \"console\" to rootlog
 ger above if you want to use this\r\n#\r\n\r\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\r\nlog4j.appender.console.target=System.err\r\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\r\n\r\n#\r\n# TaskLog Appender\r\n#\r\n\r\n#Default values\r\nhadoop.tasklog.taskid=null\r\nhadoop.tasklog.iscleanup=false\r\nhadoop.tasklog.noKeepSplits=4\r\nhadoop.tasklog.totalLogFileSize=100\r\nhadoop.tasklog.purgeLogSplits=true\r\nhadoop.tasklog.logsRetainHours=12\r\n\r\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\r\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\r\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\r\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\r\n\r\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\n\r\n#\r\n#Security audit appender\r\
 n#\r\nhadoop.security.logger=INFO,console\r\nhadoop.security.log.maxfilesize=256MB\r\nhadoop.security.log.maxbackupindex=20\r\nlog4j.category.SecurityLogger=${hadoop.security.logger}\r\nhadoop.security.log.file=SecurityAuth.audit\r\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\r\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\r\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\r\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\r\n\r\n#\r\n# hdfs audit l
 ogging\r\n#\r\nhdfs.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\r\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\r\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\nlog4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.eagle.incubator.apache.org:6667\r\nlog4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer\r\nlog4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.KAFKA_HDFS_AUDIT
 .Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async\r\nlog4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1\r\nlog4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1\r\n\r\n#\r\n# mapred audit logging\r\n#\r\nmapred.audit.logger=INFO,console\r\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\r\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\r\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\r\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\r\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\r\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\r\n\r\n#\r\n# Rolling File Appender\r\n#\r\n\r\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\r\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\r\n\r\n# Logfile size and and 30-day backups\r\nlog4j.appender.RFA.MaxFileSize=256MB\r\
 nlog4j.appender.RFA.MaxBackupIndex=10\r\n\r\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\r\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\r\n\r\n\r\n# Custom Logging levels\r\n\r\nhadoop.metrics.log.level=INFO\r\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\r\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\r\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\r\n\r\n# Jets3t library\r\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\r\n\r\n#\r\n# Null Appender\r\n# Trap security logger on the hadoop client side\r\n#\r\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\r\n\r\n#\r\n# Event Counter Appender\r\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\r\n#\r\nlog4j.appender.EventCounter
 =org.apache.hadoop.log.metrics.EventCounter\r\n\r\n# Removes \"deprecated\" messages\r\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\r\n\r\n#\r\n# HDFS block state change log from block manager\r\n#\r\n# Uncomment the following to suppress normal block state change\r\n# messages from BlockManager in NameNode.\r\n#log4j.logger.BlockStateChange=WARN\r\n"
-         }
-      }
-    }
-  ],
-  "host_groups": [
-    {
-      "name": "master",
-      "components": [
-        {
-          "name": "APP_TIMELINE_SERVER"
-        },
-        {
-          "name": "HISTORYSERVER"
-        },
-        {
-          "name": "HBASE_REGIONSERVER"
-        },
-        {
-          "name": "WEBHCAT_SERVER"
-        },
-        {
-          "name": "HCAT"
-        },
-        {
-          "name": "HBASE_CLIENT"
-        },
-        {
-          "name": "NAMENODE"
-        },
-        {
-          "name": "AMBARI_SERVER"
-        },
-        {
-          "name": "HDFS_CLIENT"
-        },
-        {
-          "name": "HIVE_CLIENT"
-        },
-        {
-          "name": "NODEMANAGER"
-        },
-        {
-          "name": "DATANODE"
-        },
-        {
-          "name": "RESOURCEMANAGER"
-        },
-        {
-          "name": "ZOOKEEPER_SERVER"
-        },
-        {
-          "name": "ZOOKEEPER_CLIENT"
-        },
-        {
-          "name": "HBASE_MASTER"
-        },
-        {
-          "name": "HIVE_SERVER"
-        },
-        {
-          "name": "SECONDARY_NAMENODE"
-        },
-        {
-          "name": "HIVE_METASTORE"
-        },
-        {
-          "name": "YARN_CLIENT"
-        },
-        {
-          "name": "MAPREDUCE2_CLIENT"
-        },
-        {
-          "name": "MYSQL_SERVER"
-        },
-        { "name": "GANGLIA_SERVER"},
-
-	{ "name": "GANGLIA_MONITOR"},
-	
-	{ "name": "KAFKA_BROKER"},
-        {
-          "name" : "DRPC_SERVER"
-        },
-        {
-          "name" : "NIMBUS"
-        },
-        {
-          "name" : "STORM_UI_SERVER"
-        },
-        { "name" : "SUPERVISOR"}
-      ],
-
-      "cardinality": "1"
-    }
-  ],
-  "Blueprints": {
-    "blueprint_name": "hdp-singlenode-eagle",
-    "stack_name": "HDP",
-    "stack_version": "2.2"
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/install-cluster.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/install-cluster.sh b/eagle-external/eagle-docker/resource/install-cluster.sh
deleted file mode 100755
index 4acd384..0000000
--- a/eagle-external/eagle-docker/resource/install-cluster.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-export PATH=/usr/jdk64/jdk1.7.0_67/bin:$PATH
-
-
-./ambari-shell.sh << EOF
-blueprint add --file /tmp/eagle-singlenode.json
-blueprint add --file /tmp/eagle-multinode.json
-cluster build --blueprint $BLUEPRINT
-cluster autoAssign
-cluster create --exitOnFinish true
-EOF
-
-clear
-
-SERF_RPC_ADDR=${AMBARISERVER_PORT_7373_TCP##*/}
-serf event --rpc-addr=$SERF_RPC_ADDR eagle
-
-./wait-for-eagle.sh
-

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/serf/etc/ambari.json
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/serf/etc/ambari.json b/eagle-external/eagle-docker/resource/serf/etc/ambari.json
deleted file mode 100644
index 4409e84..0000000
--- a/eagle-external/eagle-docker/resource/serf/etc/ambari.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
-  "event_handlers": [
-    "member-join=/usr/local/serf/handlers/ambari-bootstrap",
-    "user:eagle=/usr/local/serf/handlers/eagle"
-  ],
-  "tags" : {
-    "ambari-agent": "true"
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/serf/handlers/eagle
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/serf/handlers/eagle b/eagle-external/eagle-docker/resource/serf/handlers/eagle
deleted file mode 100755
index 7f7df4b..0000000
--- a/eagle-external/eagle-docker/resource/serf/handlers/eagle
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# install Eagle on the Ambari server instance
-if [[ "$SERF_TAG_AMBARI_SERVER" == "true" ]] ;then
-  echo run eagle install script
-  nohup /usr/hdp/current/eagle/deploy.sh > /var/log/eagle-deploy.log
-fi

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-docker/resource/wait-for-eagle.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-docker/resource/wait-for-eagle.sh b/eagle-external/eagle-docker/resource/wait-for-eagle.sh
deleted file mode 100755
index 3b6ab11..0000000
--- a/eagle-external/eagle-docker/resource/wait-for-eagle.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-: ${EAGLE_HOST:=$AMBARISERVER_PORT_9099_TCP_ADDR}
-: ${SLEEP:=2}
-: ${DEBUG:=1}
-
-: ${EAGLE_HOST:? eagle server address is mandatory, fallback is a linked containers exposed 9099}
-
-debug() {
-  [ $DEBUG -gt 0 ] && echo [DEBUG] "$@" 1>&2
-}
-
-get-server-state() {
-  curl -s -o /dev/null -w "%{http_code}" $AMBARISERVER_PORT_9099_TCP_ADDR:9099/eagle-service/index.html
-}
-
-debug waits for eagle to start on: $EAGLE_HOST
-while ! get-server-state | grep 200 &>/dev/null ; do
-  [ $DEBUG -gt 0 ] && echo -n .
-  sleep $SLEEP
-done
-[ $DEBUG -gt 0 ] && echo
-debug eagle web started: $EAGLE_HOST:9099/eagle-service

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/pom.xml b/eagle-external/eagle-log4jkafka/pom.xml
deleted file mode 100644
index 2c4871b..0000000
--- a/eagle-external/eagle-log4jkafka/pom.xml
+++ /dev/null
@@ -1,116 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one or more
-  ~ contributor license agreements.  See the NOTICE file distributed with
-  ~ this work for additional information regarding copyright ownership.
-  ~ The ASF licenses this file to You under the Apache License, Version 2.0
-  ~ (the "License"); you may not use this file except in compliance with
-  ~ the License.  You may obtain a copy of the License at
-  ~
-  ~    http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>eagle-external-parent</artifactId>
-        <groupId>eagle</groupId>
-        <version>0.3.0</version>
-        <relativePath>../pom.xml</relativePath>
-    </parent>
-    <packaging>jar</packaging>
-    <modelVersion>4.0.0</modelVersion>
-    <artifactId>eagle-log4jkafka</artifactId>
-
-    <dependencies>
-        <dependency>
-            <groupId>log4j</groupId>
-            <artifactId>log4j</artifactId>
-        </dependency>
-        <!--<dependency>-->
-            <!--<groupId>org.slf4j</groupId>-->
-            <!--<artifactId>slf4j-log4j12</artifactId>-->
-        <!--</dependency>-->
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_${scala.version}</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.version}</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <sourceDirectory>src/main/scala</sourceDirectory>
-        <plugins>
-            <plugin>
-                <groupId>org.scala-tools</groupId>
-                <artifactId>maven-scala-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>compile</goal>
-                            <goal>testCompile</goal>
-                        </goals>
-                        <configuration>
-                            <args>
-                                <arg>-make:transitive</arg>
-                                <arg>-dependencyfile</arg>
-                                <arg>${project.build.directory}/.scala_dependencies</arg>
-                            </args>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <configuration>
-                    <useFile>false</useFile>
-                    <disableXmlReport>true</disableXmlReport>
-                    <includes>
-                        <include>**/*Test.*</include>
-                        <include>**/Test*.*</include>
-                        <include>**/*Suite.*</include>
-                    </includes>
-                </configuration>
-            </plugin>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <descriptor>src/assembly/build.xml</descriptor>
-                    <finalName>eagle-log4jkafka</finalName>
-                </configuration>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                        <configuration>
-                            <tarLongFileMode>posix</tarLongFileMode>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/assembly/build.xml
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/assembly/build.xml b/eagle-external/eagle-log4jkafka/src/assembly/build.xml
deleted file mode 100644
index 54030eb..0000000
--- a/eagle-external/eagle-log4jkafka/src/assembly/build.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>build</id>
-    <formats>
-        <format>dir</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>lib/</outputDirectory>
-            <useProjectArtifact>false</useProjectArtifact>
-            <unpack>false</unpack>
-            <scope>runtime</scope>
-            <includes>
-                <include>org.apache.kafka:kafka_2.10</include>
-                <include>org.apache.kafka:kafka-clients</include>
-                <include>com.yammer.metrics:metrics-core</include>
-                <include>org.scala-lang:scala-library</include>
-            </includes>
-        </dependencySet>
-    </dependencySets>
-    <fileSets>
-        <fileSet>
-            <directory>${project.basedir}/target</directory>
-            <outputDirectory>lib/</outputDirectory>
-            <includes>
-                <include>*.jar</include>
-            </includes>
-            <excludes>
-                <exclude>*-tests.jar</exclude>
-                <exclude>*-sources.jar</exclude>
-            </excludes>
-        </fileSet>
-        <fileSet>
-            <directory>${project.basedir}/target/conf</directory>
-            <outputDirectory>conf/</outputDirectory>
-            <includes>
-                <include>*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-</assembly>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties b/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties
deleted file mode 100644
index d67ea03..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/resources/conf/log4j-template.properties
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Name configure audit appender through "hdfs.audit.logger"
-#hdfs.audit.logger=INFO,console, KAFKA_HDFS_AUDIT,DRFAAUDIT
-#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-#log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-#
-
-log4j.appender.KAFKA_HDFS_AUDIT=org.apache.eagle.log4j.kafka.KafkaLog4jAppender
-log4j.appender.KAFKA_HDFS_AUDIT.Topic=sandbox_hdfs_audit_log
-log4j.appender.KAFKA_HDFS_AUDIT.BrokerList=sandbox.hortonworks.com:6667
-log4j.appender.KAFKA_HDFS_AUDIT.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer
-log4j.appender.KAFKA_HDFS_AUDIT.Layout=org.apache.log4j.PatternLayout
-log4j.appender.KAFKA_HDFS_AUDIT.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.KAFKA_HDFS_AUDIT.ProducerType=async
-log4j.appender.KAFKA_HDFS_AUDIT.BatchSize=1
-log4j.appender.KAFKA_HDFS_AUDIT.QueueSize=1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala
deleted file mode 100644
index b3519cc..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/JKafkaLog4jAppender.scala
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.log4j.kafka
-
-import java.util.{Date, Properties}
-
-import kafka.producer.async.MissingConfigException
-import kafka.utils.{Logging, Utils}
-import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
-import org.apache.log4j.AppenderSkeleton
-import org.apache.log4j.helpers.LogLog
-import org.apache.log4j.spi.LoggingEvent
-
-class JKafkaLog4jAppender extends AppenderSkeleton with Logging {
-  var topic: String = null
-  var brokerList: String = null
-  var compressionType: String = null
-  var retries: Int = 0
-  var requiredNumAcks: Int = Int.MaxValue
-  var syncSend: Boolean = false
-
-  var keyClass: String = null
-  var keyer: Keyer = null
-
-  var topicClass: String = null
-  var topicPicker: TopicPicker = null
-
-  private var producer: KafkaProducer[Array[Byte],Array[Byte]] = null
-
-  def getTopic: String = topic
-  def setTopic(topic: String) { this.topic = topic }
-
-  def getBrokerList: String = brokerList
-  def setBrokerList(brokerList: String) { this.brokerList = brokerList }
-
-  def getCompressionType: String = compressionType
-  def setCompressionType(compressionType: String) { this.compressionType = compressionType }
-
-  def getRequiredNumAcks: Int = requiredNumAcks
-  def setRequiredNumAcks(requiredNumAcks: Int) { this.requiredNumAcks = requiredNumAcks }
-
-  def getSyncSend: Boolean = syncSend
-  def setSyncSend(syncSend: Boolean) { this.syncSend = syncSend }
-
-  def getRetries: Int = retries
-  def setRetries(retries: Int) { this.retries = retries }
-
-  def getKeyClass: String = keyClass
-  def setKeyClass(keyClass: String) { this.keyClass = keyClass }
-
-  def getTopicClass: String  = topicClass
-  def setTopicClass(topicClass: String) { this.topicClass = topicClass }
-
-  var keyPattern: String = null
-
-  def getKeyPattern:String = keyPattern
-
-  def setKeyPattern(keyPattern: String) { this.keyPattern = keyPattern }
-
-  override def activateOptions() {
-    // check for config parameter validity
-    val props = new Properties()
-    if(brokerList != null)
-      props.put(org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)
-    if(props.isEmpty)
-      throw new MissingConfigException("The bootstrap servers property should be specified")
-    
-    // if(topic == null)
-    if(topic == null && topicClass == null)
-      throw new MissingConfigException("topic must be specified by the Kafka log4j appender")
-
-    props.put("topic",topic)
-
-    if(keyPattern != null) props.put("keyPattern", keyPattern)
-
-    if(compressionType != null) props.put(org.apache.kafka.clients.producer.ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType)
-    if(requiredNumAcks != Int.MaxValue) props.put(org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG, requiredNumAcks.toString)
-    if(retries > 0) props.put(org.apache.kafka.clients.producer.ProducerConfig.RETRIES_CONFIG, retries.toString)
-    props.put(org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer")
-    props.put(org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer")
-    producer = new KafkaProducer[Array[Byte],Array[Byte]](props)
-
-    if(keyClass != null){
-      keyer = Utils.createObject[Keyer](keyClass,props)
-      LogLog.debug("Instantiated Key class " +  keyClass)
-    }
-
-    if(topicClass != null){
-      topicPicker = Utils.createObject[TopicPicker](topicClass,props)
-      LogLog.debug("Instantiated Topic class: " + topicClass)
-    }
-    LogLog.debug("Kafka producer connected to " +  brokerList)
-    LogLog.debug("Logging for topic: " + topic)
-  }
-
-  override def append(event: LoggingEvent)  {
-    val message = subAppend(event)
-    LogLog.debug("[" + new Date(event.getTimeStamp).toString + "]" + message)
-    // val response = producer.send(new ProducerRecord[Array[Byte],Array[Byte]](topic, message.getBytes()))
-    val keyBytes = if(keyer != null) keyer.getKey(message).getBytes() else null
-    val messageTopic = if(topicPicker != null) topicPicker.getTopic(message) else topic
-    val response = producer.send(new ProducerRecord[Array[Byte],Array[Byte]](messageTopic, keyBytes, message.getBytes()))
-    if (syncSend) response.get
-  }
-
-  def subAppend(event: LoggingEvent): String = {
-    if(this.layout == null)
-      event.getRenderedMessage
-    else
-      this.layout.format(event)
-  }
-
-  override def close() {
-    if(!this.closed) {
-      this.closed = true
-      producer.close()
-    }
-  }
-
-  override def requiresLayout: Boolean = true
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala
deleted file mode 100644
index ee09294..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/KafkaLog4jAppender.scala
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka
-
-import java.util.{Date, Properties}
-
-import kafka.producer.async.MissingConfigException
-import kafka.producer.{KeyedMessage, Producer, ProducerConfig}
-import kafka.utils.{Logging, Utils}
-import org.apache.log4j.AppenderSkeleton
-import org.apache.log4j.helpers.LogLog
-import org.apache.log4j.spi.LoggingEvent
-
-class KafkaLog4jAppender extends AppenderSkeleton with Logging {
-  var topic:String = null
-  var serializerClass:String = null
-  var brokerList:String = null
-  var producerType:String = null
-  var compressionCodec:String = null
-  var enqueueTimeout:String = null
-  var queueSize:String = null
-  var requiredNumAcks: Int = Int.MaxValue
-
-  var keyClass: String = null
-  var keyer: Keyer = null
-
-  var topicClass: String = null
-  var topicPicker: TopicPicker = null
-
-  var batchSize:Int = 20
-
-  var keyPattern: String = null
-
-  def getKeyPattern:String = keyPattern
-
-  def setKeyPattern(keyPattern: String) { this.keyPattern = keyPattern }
-
-  private var producer: Producer[String, String] = null
-
-  def getTopic:String = topic
-  def setTopic(topic: String) { this.topic = topic }
-
-  def getBrokerList:String = brokerList
-  def setBrokerList(brokerList: String) { this.brokerList = brokerList }
-
-  def getSerializerClass:String = serializerClass
-  def setSerializerClass(serializerClass:String) { this.serializerClass = serializerClass }
-
-  def getProducerType:String = producerType
-  def setProducerType(producerType:String) { this.producerType = producerType }
-
-  def getCompressionCodec:String = compressionCodec
-  def setCompressionCodec(compressionCodec:String) { this.compressionCodec = compressionCodec }
-
-  def getEnqueueTimeout:String = enqueueTimeout
-  def setEnqueueTimeout(enqueueTimeout:String) { this.enqueueTimeout = enqueueTimeout }
-
-  def getQueueSize:String = queueSize
-  def setQueueSize(queueSize:String) { this.queueSize = queueSize }
-
-  def getRequiredNumAcks:Int = requiredNumAcks
-  def setRequiredNumAcks(requiredNumAcks:Int) { this.requiredNumAcks = requiredNumAcks }
-
-  def getKeyClass: String = keyClass
-  def setKeyClass(keyClass: String) { this.keyClass = keyClass }
-
-  def getTopicClass: String  = topicClass
-  def setTopicClass(topicClass: String) { this.topicClass = topicClass }
-
-  def getBatchSize:Int = batchSize
-  def setBatchSize(batchSize:Int) { this.batchSize = batchSize };
-
-  override def activateOptions() {
-    // check for config parameter validity
-    val props = new Properties()
-    if(brokerList != null)
-      props.put("metadata.broker.list", brokerList)
-    if(props.isEmpty)
-      throw new MissingConfigException("The metadata.broker.list property should be specified")
-    if(topic == null)
-      throw new MissingConfigException("topic must be specified by the Kafka log4j appender")
-    if(serializerClass == null) {
-      serializerClass = "kafka.serializer.StringEncoder"
-      LogLog.debug("Using default encoder - kafka.serializer.StringEncoder")
-    }
-
-    props.put("serializer.class", serializerClass)
-    props.put("batch.num.messages",batchSize.toString)
-
-    if(keyPattern != null) props.put("keyPattern", keyPattern)
-
-    //These have default values in ProducerConfig and AsyncProducerConfig. We don't care if they're not specified
-    if(producerType != null) props.put("producer.type", producerType)
-    if(compressionCodec != null) props.put("compression.codec", compressionCodec)
-    if(enqueueTimeout != null) props.put("queue.enqueue.timeout.ms", enqueueTimeout)
-    if(queueSize != null) props.put("queue.buffering.max.messages", queueSize)
-    if(requiredNumAcks != Int.MaxValue) props.put("request.required.acks", requiredNumAcks.toString)
-
-    val config : ProducerConfig = new ProducerConfig(props)
-
-    producer = new Producer[String, String](config)
-
-    if(keyClass != null){
-      keyer = Utils.createObject[Keyer](keyClass,props)
-      LogLog.debug("Instantiated Key class " +  keyClass)
-    }
-
-    if(topicClass != null) {
-      try {
-        topicPicker = Utils.createObject[TopicPicker](topicClass, props)
-        LogLog.debug("Instantiated Topic class: " + topicClass)
-      }catch {
-        case e: Exception =>{
-          LogLog.error("Failed to instantiate Topic class: "+topicClass,e)
-        }
-      }
-    }
-
-    LogLog.debug("Kafka producer connected to " +  config.brokerList)
-    LogLog.debug("Logging for topic: " + topic)
-  }
-
-  override def append(event: LoggingEvent)  {
-    val message = subAppend(event)
-    LogLog.debug("[" + new Date(event.getTimeStamp).toString + "]" + message)
-
-    val messageKey = if(keyer != null) keyer.getKey(message) else null
-    val messageTopic = if(topicPicker != null) topicPicker.getTopic(message) else topic
-
-    val messageData = new KeyedMessage[String, String](messageTopic, messageKey, message)
-    producer.send(messageData)
-  }
-
-  def subAppend(event: LoggingEvent): String = {
-    if(this.layout == null)
-      event.getRenderedMessage
-    else
-      this.layout.format(event)
-  }
-
-  override def close() {
-    if(!this.closed) {
-      this.closed = true
-      producer.close()
-    }
-  }
-
-  override def requiresLayout: Boolean = false
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala
deleted file mode 100644
index b25ffbe..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/Keyer.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka
-
-/**
- * Keyer is used to derive a key from the message. A Key is used to determine kafka partition for the message.
- *
- * Implementations will be constructed via reflection and are required to have a constructor that takes a single
- * VerifiableProperties instance--this allows passing configuration properties into the Keyer implementation.
- */
-trait Keyer {
-  /**
-    * Uses the key to calculate a partition bucket id for routing
-    * the data to the appropriate broker partition
-   * @return returns a key based on the message
-   */
-  def getKey(value: String): String
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala
deleted file mode 100644
index 8ef1ff9..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/TopicPicker.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka
-
-/**
- * TopicPicker is used to derive a topic from the message.
- *
- * Implementations will be constructed via reflection and are required to have a constructor that takes a single
- * VerifiableProperties instance--this allows passing configuration properties into the TopicPicker implementation.
- *
- * @since  6/26/15
- */
-trait TopicPicker {
-  /**
-   * returns a topic based on the message.
-   */
-  def getTopic(value: String): String
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala
deleted file mode 100644
index f435d9f..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogKeyer.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka.hadoop
-
-import java.util.Properties
-import org.apache.eagle.log4j.kafka.Keyer
-
-class AuditLogKeyer(props:Properties) extends Keyer{
-  /**
-   * Uses the key to calculate a partition bucket id for routing
-   * the data to the appropriate broker partition
-   * @return returns a key based on the message
-   */
-  override def getKey(msg: String): String = if(msg != null) AuditLogKeyer.getKey(msg.split("\\s+")) else null
-}
-
-object AuditLogKeyer{
-  def getKey(fields: Array[String]):String = if(fields.length > 5) AuditLogUtils.parseUser(fields(5)) else null
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala
deleted file mode 100644
index 8d206f6..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditLogUtils.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka.hadoop
-
-object AuditLogUtils {
-  def parseUser(userText: String): String = {
-    var user:String = userText.substring(userText.indexOf("=")+1)
-    if(user.indexOf("@") > -1){
-      user = user.substring(0,user.indexOf("@"))
-    }
-    user
-  }
-
-  def parseIP(ipPort:String):String = ipPort.substring(0,ipPort.indexOf(':') + 1)
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala
deleted file mode 100644
index 38ff72a..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/AuditSecurityLogKeyer.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka.hadoop
-
-import java.util.Properties
-import org.apache.eagle.log4j.kafka.Keyer
-
-/**
- * @since  6/26/15
- */
-class AuditSecurityLogKeyer(props:Properties) extends Keyer{
-  /**
-   * Uses the key to calculate a partition bucket id for routing
-   * the data to the appropriate broker partition
-   * @return returns a key based on the message
-   *
-   */
-  val AUDITLOG_SOURCE:String = "FSNamesystem.audit:"
-
-  override def getKey(msg: String): String = {
-    val fields: Array[String] = msg.split("\\s+")
-    if(fields.length > 3){
-      if(fields(3).equals(AUDITLOG_SOURCE))
-        AuditLogKeyer.getKey(fields)
-      else
-        SecurityLogKeyer.getKey(fields)
-    } else null
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala
deleted file mode 100644
index 25d781b..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/FailedAuthenticationFilter.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka.hadoop
-
-import org.apache.log4j.spi.{Filter, LoggingEvent}
-
-/**
- * @since  6/26/15
- */
-class FailedAuthenticationFilter extends Filter{
-
-  override def decide(event: LoggingEvent): Int = {
-    event.getMessage match {
-      case message: String =>
-        val fields: Array[String] = message.split("\\s+")
-        if (fields.length > 5 && fields(3).equals(FailedAuthenticationFilter.AUTH_LOG_SOURCE) && fields(5).equals(FailedAuthenticationFilter.FAILED)) {
-          Filter.DENY
-        }
-      case _ =>
-    }
-    Filter.NEUTRAL
-  }
-}
-
-object FailedAuthenticationFilter{
-  val AUTH_LOG_SOURCE: String = "SecurityLogger.org.apache.hadoop.ipc.Server:"
-  val FAILED: String = "failed"
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala
deleted file mode 100644
index 0deb152..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/GenericLogKeyer.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.log4j.kafka.hadoop
-
-import java.util.Properties
-import java.util.regex.{Matcher, Pattern}
-import org.apache.eagle.log4j.kafka.Keyer
-
-import scala.util.matching.Regex
-
-class GenericLogKeyer(props:Properties) extends Keyer {
-  /**
-   * Uses the key to calculate a partition bucket id for routing
-   * the data to the appropriate broker partition
-   * @return returns a key based on the message
-   */
-  override def getKey(msg: String): String = {
-    val keyPattern = props.getProperty("keyPattern");
-    if(keyPattern == null) return null
-    val pattern = new Regex(keyPattern, "key");
-    val result = pattern.findFirstMatchIn(msg).getOrElse(null);
-    if (result == null) {
-      return null;
-    }
-    return result.group("key");
-  }
-}
-
-

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala
deleted file mode 100644
index 4bf87da..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogKeyer.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka.hadoop
-
-import java.util.Properties
-
-import org.apache.eagle.log4j.kafka.Keyer
-
-/**
- * @since  6/26/15
- */
-class SecurityLogKeyer(props:Properties) extends Keyer{
-  /**
-   * Uses the key to calculate a partition bucket id for routing
-   * the data to the appropriate broker partition
-   * @return returns a key based on the message
-   */
-  override def getKey(msg: String): String = SecurityLogKeyer.getKey(msg.split("\\s+"))
-}
-
-object SecurityLogKeyer{
-  val FAILED:String = "failed"
-  val SUCCESSFUL: String = "successful"
-
-  def getKey(fields:Array[String]):String = {
-    if (fields.length > 5) {
-      if (fields(5) == SUCCESSFUL) {
-        return if (fields.length > 7) AuditLogUtils.parseUser(fields(7)) else null
-      }
-      else if (fields(5) == FAILED) {
-        return if (fields.length > 7) AuditLogUtils.parseIP(fields(7)) else null
-      }
-    }
-    null
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala b/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala
deleted file mode 100644
index 8a2e0ab..0000000
--- a/eagle-external/eagle-log4jkafka/src/main/scala/org/apache/eagle/log4j/kafka/hadoop/SecurityLogTopicPicker.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log4j.kafka.hadoop
-
-import java.util.Properties
-import org.apache.eagle.log4j.kafka.TopicPicker
-
-/**
- * @since  6/26/15
- */
-class SecurityLogTopicPicker extends TopicPicker{
-  private var failedAuthnTopic: String = null
-  private var securityTopic: String = null
-
-  val FAILED: String = "failed"
-  val TOPIC: String = "Topic"
-
-  def this(props:Properties){
-    this()
-    val topic = props.getProperty(TOPIC)
-    val topics = if(topic!=null) topic.split(",") else Array(topic)
-
-    if(topics != null){
-      if(topics.length > 0) failedAuthnTopic = topics(0)
-      if(topics.length > 1) securityTopic = topics(1)
-    }
-  }
-
-  /**
-   * returns a topic based on the message.
-   */
-  override def getTopic(value: String): String = {
-    val fields = value.split("\\s+")
-    if(fields.length > 5 && fields(5).equals(FAILED))
-      failedAuthnTopic
-    else
-      securityTopic
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties b/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties
deleted file mode 100644
index 85895a8..0000000
--- a/eagle-external/eagle-log4jkafka/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-log4j.rootLogger=DEBUG, console
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-log4j.appender.KAFKA=org.apache.eagle.log4j.kafka.KafkaLog4jAppender
-log4j.appender.KAFKA.Topic=test
-log4j.appender.KAFKA.BrokerList=sandbox.hortonworks.com:6667
-log4j.appender.KAFKA.KeyClass=org.apache.eagle.log4j.kafka.hadoop.AuditLogKeyer
-log4j.appender.KAFKA.Layout=org.apache.log4j.PatternLayout
-log4j.appender.KAFKA.Layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.KAFKA.ProducerType=async
-log4j.appender.KAFKA.BatchSize=1
-log4j.appender.KAFKA.QueueSize=1
-
-log4j.logger.eagle.kafka.producer.TestKafkaAppender$=console,KAFKA
-
-#log4j.logger.kafka.utils.VerifiableProperties=INFO,console

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala
deleted file mode 100644
index 840dfe7..0000000
--- a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestGenericLogKeyer.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.log4j.kafka
-
-import java.util.Properties
-import org.apache.eagle.log4j.kafka.hadoop.GenericLogKeyer
-import org.scalatest.{FlatSpec, Matchers}
-
-abstract class TestGenericLogKeyer extends FlatSpec with Matchers  {
-  
-    val hdfsMsg = "2015-07-31 01:54:35,161 INFO FSNamesystem.audit: allowed=true ugi=root (auth:TOKEN) ip=/10.0.0.1 cmd=open src=/tmp/private dst=null perm=null"
-    val props = new Properties()
-    props.put("keyPattern", "ugi=(\\w+)[@\\s+]")
-    props.put("keyPattern2", "user=(\\w+),\\s+")
-    val test = new GenericLogKeyer(props)
-    var keyVal = test.getKey(hdfsMsg)
-    println(keyVal)
-  
-    val hbaseMsg = "2015-11-06 13:14:00,741 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user root; reason: All users allowed; remote address: /192.168.56.101; request: scan; context: (user=root, scope=hbase:meta, family=info, action=READ)"
-    props.put("keyPattern", "user=(\\w+),\\s+")
-    keyVal = test.getKey(hbaseMsg)
-    println(keyVal)
-  
-    //props.put("keyPattern", "user=(\\w+),\\s+")
-    val props1 = new Properties()
-    val test1 = new GenericLogKeyer(props1)
-    keyVal = test1.getKey(hbaseMsg)
-    println(keyVal)
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala b/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala
deleted file mode 100644
index 3bd9e0e..0000000
--- a/eagle-external/eagle-log4jkafka/src/test/scala/org/apache/eagle/log4j/kafka/TestKafkaLog4jAppender.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.log4j.kafka
-
-import org.scalatest.{Matchers, FlatSpec}
-
-abstract class TestKafkaLog4jAppender extends FlatSpec with Matchers {
-  
-    val test = new KafkaLog4jAppender();
-    test.topic = "sandbox_hdfs_audit_log"
-    test.brokerList = "sandbox.hortonworks.com:6667"
-    test.keyClass = "eagle.log4j.kafka.hadoop.AuditLogKeyer"
-    test.keyPattern = "user=(\\w+),\\s+"
-    test.producerType = "async"
-    test.activateOptions()
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/.gitignore b/eagle-external/hadoop_jmx_collector/.gitignore
deleted file mode 100644
index adad264..0000000
--- a/eagle-external/hadoop_jmx_collector/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-config.json
-*.pyc
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/README.md
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/README.md b/eagle-external/hadoop_jmx_collector/README.md
deleted file mode 100644
index cd8d887..0000000
--- a/eagle-external/hadoop_jmx_collector/README.md
+++ /dev/null
@@ -1,75 +0,0 @@
-<!--
-{% comment %}
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to you under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-{% endcomment %}
--->
-
-
-# Hadoop Jmx Collector
-
-These scripts help to collect Hadoop jmx and evently sent the metrics to stdout or Kafka. Tested with Python 2.7.
-
-### How to use it
-
-  1. Edit the configuration file (json file). For example:
-  
-            {
-             "env": {
-              "site": "sandbox"
-             },
-             "input": {
-              "component": "namenode",
-              "port": "50070",
-              "https": false
-             },
-             "filter": {
-              "monitoring.group.selected": ["hadoop", "java.lang"]
-             },
-             "output": {
-             }
-            }
-
-  2. Run the scripts
-  
-        # for general use
-        python hadoop_jmx_kafka.py > 1.txt
-
-
-### Edit `eagle-collector.conf`
-
-* input
-
-  "port" defines the hadoop service port, such as 50070 => "namenode", 60010 => "hbase master".
-
-* filter
-
-  "monitoring.group.selected" can filter out beans which we care about.
-
-* output 
-  
-  if we left it empty, then the output is stdout by default. 
-
-        "output": {}
-        
-  It also supports Kafka as its output. 
-
-        "output": {
-          "kafka": {
-            "topic": "test_topic",
-            "brokerList": [ "sandbox.hortonworks.com:6667"]
-          }
-        }
-      

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/config-sample.json
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/config-sample.json b/eagle-external/hadoop_jmx_collector/config-sample.json
deleted file mode 100644
index 57c9aae..0000000
--- a/eagle-external/hadoop_jmx_collector/config-sample.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-   "env": {
-    "site": "sandbox",
-     "name_node": {
-       "hosts":["sandbox.hortonworks.com"],
-       "port":50070,
-       "https":false
-     },
-     "resource_manager":{
-       "hosts":["sandbox.hortonworks.com"],
-       "port":50030,
-       "https":false
-     }
-   },
-   "input": {
-    "component": "namenode",
-    "port": "50070",
-    "https": false
-   },
-   "filter": {
-    "monitoring.group.selected": ["hadoop", "java.lang"]
-   },
-   "output": {
-     "kafka": {
-       "topic": "nn_jmx_metric_sandbox",
-       "brokerList": ["sandbox.hortonworks.com:6667"]
-     }
-   }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py b/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
deleted file mode 100644
index 2a5096b..0000000
--- a/eagle-external/hadoop_jmx_collector/hadoop_ha_checker.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# !/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from metric_collector import MetricCollector, JmxReader, YarnWSReader, Runner
-import logging
-
-class HadoopNNHAChecker(MetricCollector):
-    def run(self):
-        if not self.config["env"].has_key("name_node"):
-            logging.warn("Do nothing for HadoopNNHAChecker as config of env.name_node not found")
-            return
-        name_node_config = self.config["env"]["name_node"]
-        hosts = name_node_config["hosts"]
-        port = name_node_config["port"]
-        https = name_node_config["https"]
-
-        total_count = len(hosts)
-
-        self.collect({
-            "component": "namenode",
-            "metric": "hadoop.namenode.hastate.total.count",
-            "value": total_count
-        })
-
-        active_count = 0
-        standby_count = 0
-        failed_count = 0
-
-        for host in hosts:
-            try:
-                bean = JmxReader(host, port, https).open().get_jmx_bean_by_name(
-                        "Hadoop:service=NameNode,name=FSNamesystem")
-                logging.debug(host + " is " + bean["tag.HAState"])
-                if bean["tag.HAState"] == "active":
-                    active_count += 1
-                else:
-                    standby_count += 1
-            except Exception as e:
-                logging.exception("failed to read jmx from " + host)
-                failed_count += 1
-
-        self.collect({
-            "component": "namenode",
-            "metric": "hadoop.namenode.hastate.active.count",
-            "value": active_count
-        })
-
-        self.collect({
-            "component": "namenode",
-            "metric": "hadoop.namenode.hastate.standby.count",
-            "value": standby_count
-        })
-
-        self.collect({
-            "component": "namenode",
-            "metric": "hadoop.namenode.hastate.failed.count",
-            "value": failed_count
-        })
-
-
-class HadoopRMHAChecker(MetricCollector):
-    def run(self):
-        if not self.config["env"].has_key("resource_manager"):
-            logging.warn("Do nothing for HadoopRMHAChecker as config of env.resource_manager not found")
-            return
-        name_node_config = self.config["env"]["resource_manager"]
-        hosts = name_node_config["hosts"]
-        port = name_node_config["port"]
-        https = name_node_config["https"]
-
-        total_count = len(hosts)
-
-        self.collect({
-            "component": "namenode",
-            "metric": "hadoop.resourcemanager.hastate.total.count",
-            "value": total_count
-        })
-
-        active_count = 0
-        standby_count = 0
-        failed_count = 0
-
-        for host in hosts:
-            try:
-                cluster_info = YarnWSReader(host, port, https).read_cluster_info()
-                if cluster_info["clusterInfo"]["haState"] == "ACTIVE":
-                    active_count += 1
-                else:
-                    standby_count += 1
-            except Exception as e:
-                logging.exception("Failed to read yarn ws from " + host)
-                failed_count += 1
-
-        self.collect({
-            "component": "resourcemanager",
-            "metric": "hadoop.resourcemanager.hastate.active.count",
-            "value": active_count
-        })
-
-        self.collect({
-            "component": "resourcemanager",
-            "metric": "hadoop.resourcemanager.hastate.standby.count",
-            "value": standby_count
-        })
-
-        self.collect({
-            "component": "resourcemanager",
-            "metric": "hadoop.resourcemanager.hastate.failed.count",
-            "value": failed_count
-        })
-
-if __name__ == '__main__':
-    Runner.run(HadoopNNHAChecker(), HadoopRMHAChecker())
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py b/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py
deleted file mode 100644
index 8837c30..0000000
--- a/eagle-external/hadoop_jmx_collector/hadoop_jmx_kafka.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# !/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from metric_collector import JmxMetricCollector,JmxMetricListener,Runner
-import json
-
-class NNSafeModeMetric(JmxMetricListener):
-    def on_metric(self, metric):
-        if metric["metric"] == "hadoop.namenode.fsnamesystemstate.fsstate":
-            if metric["value"] == "safeMode":
-                metric["value"] = 1
-            else:
-                metric["value"] = 0
-            self.collector.collect(metric)
-
-
-class NNHAMetric(JmxMetricListener):
-    PREFIX = "hadoop.namenode.fsnamesystem"
-
-    def on_bean(self, bean):
-        if bean["name"] == "Hadoop:service=NameNode,name=FSNamesystem":
-            if bean[u"tag.HAState"] == "active":
-                self.collector.on_bean_kv(self.PREFIX, "hastate", 0)
-            else:
-                self.collector.on_bean_kv(self.PREFIX, "hastate", 1)
-
-
-class MemortUsageMetric(JmxMetricListener):
-    PREFIX = "hadoop.namenode.jvm"
-
-    def on_bean(self, bean):
-        if bean["name"] == "Hadoop:service=NameNode,name=JvmMetrics":
-            memnonheapusedusage = round(float(bean['MemNonHeapUsedM']) / float(bean['MemNonHeapMaxM']) * 100.0, 2)
-            self.collector.on_bean_kv(self.PREFIX, "memnonheapusedusage", memnonheapusedusage)
-            memnonheapcommittedusage = round(float(bean['MemNonHeapCommittedM']) / float(bean['MemNonHeapMaxM']) * 100,
-                                             2)
-            self.collector.on_bean_kv(self.PREFIX, "memnonheapcommittedusage", memnonheapcommittedusage)
-            memheapusedusage = round(float(bean['MemHeapUsedM']) / float(bean['MemHeapMaxM']) * 100, 2)
-            self.collector.on_bean_kv(self.PREFIX, "memheapusedusage", memheapusedusage)
-            memheapcommittedusage = round(float(bean['MemHeapCommittedM']) / float(bean['MemHeapMaxM']) * 100, 2)
-            self.collector.on_bean_kv(self.PREFIX, "memheapcommittedusage", memheapcommittedusage)
-
-
-class JournalTransactionInfoMetric(JmxMetricListener):
-    PREFIX = "hadoop.namenode.journaltransaction"
-
-    def on_bean(self, bean):
-        if bean.has_key("JournalTransactionInfo"):
-            JournalTransactionInfo = json.loads(bean.get("JournalTransactionInfo"))
-            LastAppliedOrWrittenTxId = float(JournalTransactionInfo.get("LastAppliedOrWrittenTxId"))
-            MostRecentCheckpointTxId = float(JournalTransactionInfo.get("MostRecentCheckpointTxId"))
-            self.collector.on_bean_kv(self.PREFIX, "LastAppliedOrWrittenTxId", LastAppliedOrWrittenTxId)
-            self.collector.on_bean_kv(self.PREFIX, "MostRecentCheckpointTxId", MostRecentCheckpointTxId)
-
-
-if __name__ == '__main__':
-    collector = JmxMetricCollector()
-    collector.register(
-            NNSafeModeMetric(),
-            NNHAMetric(),
-            MemortUsageMetric(),
-            JournalTransactionInfoMetric()
-    )
-    Runner.run(collector)

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore b/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore
deleted file mode 100644
index 260dbea..0000000
--- a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitignore
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Eagle - Git Ignore Configuration
-#
-# See: https://github.com/github/gitignore/
-
-*.egg-info
-*.pyc
-.tox
-build
-dist
-MANIFEST
-env
-servers/*/kafka-bin
-.coverage
-.noseids
-docs/_build

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0ea130ef/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules b/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules
deleted file mode 100644
index fe0f4fb..0000000
--- a/eagle-external/hadoop_jmx_collector/lib/kafka-python/.gitmodules
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Eagle - Git Ignore Configuration
-#
-# See: https://github.com/github/gitignore/
\ No newline at end of file



Mime
View raw message