chukwa-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject [12/22] chukwa git commit: CHUKWA-759. Configuration for Chukwa to monitor HBase. (Eric Yang)
Date Thu, 25 Jun 2015 20:48:43 GMT
CHUKWA-759. Configuration for Chukwa to monitor HBase.  (Eric Yang)


Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo
Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/e1f0df55
Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/e1f0df55
Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/e1f0df55

Branch: refs/heads/master
Commit: e1f0df55dcaadb48398bb680fc85c2983f897279
Parents: d8abe97
Author: Eric Yang <eyang@apache.org>
Authored: Fri Jun 19 16:31:49 2015 -0700
Committer: Eric Yang <eyang@apache.org>
Committed: Fri Jun 19 16:31:49 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                           |  6 ++-
 conf/chukwa-demux-conf.xml            |  9 +++-
 conf/hadoop-log4j.properties          |  2 -
 conf/hadoop-metrics2-hbase.properties | 31 ++++++++++++
 conf/hbase-log4j.properties           | 78 ++++++++++++++++++++++++++++++
 conf/initial_adaptors                 |  7 +++
 6 files changed, 128 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index adcd624..449dce7 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -14,10 +14,12 @@ Trunk (unreleased changes)
     
     CHUKWA-737. Heartbeat adaptor to push status to remote http server (Shreyas Subramanya)
 
-    CHUKWA-747.  Update Widget API to store data in HBase.  (Eric Yang)
+    CHUKWA-747. Update Widget API to store data in HBase.  (Eric Yang)
 
   IMPROVEMENTS
 
+    CHUKWA-759. Configuration for Chukwa to monitor HBase.  (Eric Yang)
+
     CHUKWA-757. Updated Hadoop configuration to stream data to Chukwa.  (Eric Yang)
 
     CHUKWA-749. Added Chukwa tags to Solr schema for indexing logs.  (Eric Yang)
@@ -42,7 +44,7 @@ Trunk (unreleased changes)
 
   BUGS
 
-    CHUKWA-758.  Updated default dashboard to include system metrics.  (Eric Yang)
+    CHUKWA-758. Updated default dashboard to include system metrics.  (Eric Yang)
 
     CHUKWA-743. Revised pid locking using standard Hadoop approach.  (Eric Yang)
 

http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/chukwa-demux-conf.xml
----------------------------------------------------------------------
diff --git a/conf/chukwa-demux-conf.xml b/conf/chukwa-demux-conf.xml
index fca9f1e..fe380c6 100644
--- a/conf/chukwa-demux-conf.xml
+++ b/conf/chukwa-demux-conf.xml
@@ -126,6 +126,12 @@
   </property>
 
   <property>
+    <name>HBaseMetrics</name>
+    <value>org.apache.hadoop.chukwa.extraction.hbase.HadoopMetricsProcessor</value>
+    <description>Parser class for HBase Metrics </description>
+  </property>
+
+  <property>
     <name>SysLog</name>
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.SysLog</value>
     <description>Parser class for SysLog</description>
@@ -246,6 +252,7 @@
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.ZookeeperProcessor</value>
    </property>
 
+<!--
    <property>
     <name>HBaseMasterProcessor</name>
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HBaseMasterProcessor</value>
@@ -254,7 +261,7 @@
    <property>
     <name>HBaseRegionServerProcessor</name>
     <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.HBaseRegionServerProcessor</value>
-   </property>
+   </property> -->
 
     <!-- Demux configs for reducer -->
     <property>

http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/hadoop-log4j.properties
----------------------------------------------------------------------
diff --git a/conf/hadoop-log4j.properties b/conf/hadoop-log4j.properties
index aad7cc7..6ef5b59 100644
--- a/conf/hadoop-log4j.properties
+++ b/conf/hadoop-log4j.properties
@@ -35,7 +35,6 @@ log4j.appender.RFA=org.apache.log4j.net.SocketAppender
 log4j.appender.RFA.Port=${hadoop.log.port}
 log4j.appender.RFA.RemoteHost=localhost
 log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
 
 
@@ -47,7 +46,6 @@ log4j.appender.DRFA=org.apache.log4j.net.SocketAppender
 log4j.appender.DRFA.Port=${hadoop.log.port}
 log4j.appender.DRFA.RemoteHost=localhost
 log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
 log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
 
 

http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/hadoop-metrics2-hbase.properties
----------------------------------------------------------------------
diff --git a/conf/hadoop-metrics2-hbase.properties b/conf/hadoop-metrics2-hbase.properties
new file mode 100644
index 0000000..038d31b
--- /dev/null
+++ b/conf/hadoop-metrics2-hbase.properties
@@ -0,0 +1,31 @@
+# syntax: [prefix].[source|sink].[instance].[options]
+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+*.sink.socket.class=org.apache.hadoop.chukwa.inputtools.log4j.Log4jMetricsSink
+hbase.sink.socket.host=localhost
+hbase.sink.socket.port=9104
+# *.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink
+# default sampling period
+*.period=10
+
+# Below are some examples of sinks that could be used
+# to monitor different hbase daemons.
+
+# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file-all.filename=all.metrics
+
+# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file0.context=hmaster
+# hbase.sink.file0.filename=master.metrics
+
+# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file1.context=thrift-one
+# hbase.sink.file1.filename=thrift-one.metrics
+
+# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file2.context=thrift-two
+# hbase.sink.file2.filename=thrift-one.metrics
+
+# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink
+# hbase.sink.file3.context=rest
+# hbase.sink.file3.filename=rest.metrics

http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/hbase-log4j.properties
----------------------------------------------------------------------
diff --git a/conf/hbase-log4j.properties b/conf/hbase-log4j.properties
new file mode 100644
index 0000000..d1ba3fa
--- /dev/null
+++ b/conf/hbase-log4j.properties
@@ -0,0 +1,78 @@
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.security.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.net.SocketAppender
+log4j.appender.DRFA.Port=${hbase.log.port}
+log4j.appender.DRFA.RemoteHost=localhost
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Rolling File Appender
+log4j.appender.RFA=org.apache.log4j.net.SocketAppender
+log4j.appender.RFA.Port=${hbase.log.port}
+log4j.appender.RFA.RemoteHost=localhost
+log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+#
+# Security audit appender
+#
+log4j.appender.RFAS=org.apache.log4j.net.SocketAppender
+log4j.appender.RFAS.Port=${hbase.security.log.port}
+log4j.appender.RFAS.RemoteHost=localhost
+log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+log4j.category.SecurityLogger=${hbase.security.logger}
+log4j.additivity.SecurityLogger=false
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.visibility.VisibilityController=TRACE
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this 
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+# Custom Logging levels
+
+log4j.logger.org.apache.zookeeper=INFO
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.hbase=INFO
+# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+#log4j.logger.org.apache.hadoop.dfs=DEBUG
+# Set this class to log INFO only otherwise its OTT
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
+
+
+# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+# Uncomment the below if you want to remove logging of client region caching'
+# and scan of hbase:meta messages
+# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO

http://git-wip-us.apache.org/repos/asf/chukwa/blob/e1f0df55/conf/initial_adaptors
----------------------------------------------------------------------
diff --git a/conf/initial_adaptors b/conf/initial_adaptors
index 5bc26be..08b1e34 100644
--- a/conf/initial_adaptors
+++ b/conf/initial_adaptors
@@ -8,3 +8,10 @@ add SocketAdaptor HadoopNMLog 9100 0
 add SocketAdaptor HadoopHSLog 9101 0
 add SocketAdaptor HadoopRMSUMMARY 9102 0
 add SocketAdaptor HadoopTLA 9103 0
+add SocketAdaptor HBaseMetrics 9104 0
+add SocketAdaptor HBaseSecurityLog 9105 0
+add SocketAdaptor HBaseMSLog 9106 0
+add SocketAdaptor HBaseRSLog 9107 0
+add SocketAdaptor HBaseTFLog 9108 0
+add SocketAdaptor HBaseZKLog 9109 0
+add SocketAdaptor HBaseRESTLog 9110 0


Mime
View raw message