chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1164955 - in /incubator/chukwa/trunk: ./ conf/ src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/ src/java/org/apache/hadoop/chukwa/inputtools/log4j/
Date Sun, 04 Sep 2011 02:36:48 GMT
Author: eyang
Date: Sun Sep  4 02:36:48 2011
New Revision: 1164955

URL: http://svn.apache.org/viewvc?rev=1164955&view=rev
Log:
CHUKWA-599. Added Metrics 2 support for Hadoop 0.20.203+. (Eric Yang)

Added:
    incubator/chukwa/trunk/conf/hadoop-metrics2.properties.template
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4jMetricsSink.java
Modified:
    incubator/chukwa/trunk/build.xml
    incubator/chukwa/trunk/conf/hbase.schema
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java

Modified: incubator/chukwa/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/build.xml?rev=1164955&r1=1164954&r2=1164955&view=diff
==============================================================================
--- incubator/chukwa/trunk/build.xml (original)
+++ incubator/chukwa/trunk/build.xml Sun Sep  4 02:36:48 2011
@@ -662,6 +662,7 @@
 			</fileset>
 			<fileset dir="${build.classes}">
 				<include name="org/apache/hadoop/mapred/**/*.class" />
+				<include name="org/apache/hadoop/chukwa/inputtools/log4j/**/*.class" />
 				<include name="org/apache/hadoop/chukwa/datacollection/client/**/*.class" />
 				<include name="org/apache/hadoop/chukwa/util/**/*.class" />
 				<include name="org/apache/hadoop/chukwa/datacollection/controller/*.class" />
@@ -1004,6 +1005,7 @@
 				</replacetokens>
 			</filterchain>
 		</copy>
+		<copy file="${basedir}/conf/hadoop-metrics2.properties.template" tofile="${build.dir}/conf/hadoop-metrics2.properties"/>
 		<copy file="${basedir}/conf/chukwa-agent-conf.xml.template" tofile="${build.dir}/conf/chukwa-agent-conf.xml">
 			<filterchain>
 				<replacetokens>

Added: incubator/chukwa/trunk/conf/hadoop-metrics2.properties.template
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/hadoop-metrics2.properties.template?rev=1164955&view=auto
==============================================================================
--- incubator/chukwa/trunk/conf/hadoop-metrics2.properties.template (added)
+++ incubator/chukwa/trunk/conf/hadoop-metrics2.properties.template Sun Sep  4 02:36:48 2011
@@ -0,0 +1,20 @@
+# Stream metrics to Chukwa SocketAdaptor
+*.sink.socket.class=org.apache.hadoop.metrics2.sink.SocketSink
+namenode.sink.socket.host=localhost
+namenode.sink.socket.port=9095
+datanode.sink.socket.host=localhost
+datanode.sink.socket.port=9095
+jobtracker.sink.socket.host=localhost
+jobtracker.sink.socket.port=9095
+tasktracker.sink.socket.host=localhost
+tasktracker.sink.socket.port=9095
+# Stream metrics to Chukwa SocketAdaptor
+*.sink.socket.class=org.apache.hadoop.metrics2.sink.SocketSink
+namenode.sink.socket.host=localhost
+namenode.sink.socket.port=9095
+datanode.sink.socket.host=localhost
+datanode.sink.socket.port=9095
+jobtracker.sink.socket.host=localhost
+jobtracker.sink.socket.port=9095
+tasktracker.sink.socket.host=localhost
+tasktracker.sink.socket.port=9095

Modified: incubator/chukwa/trunk/conf/hbase.schema
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/conf/hbase.schema?rev=1164955&r1=1164954&r2=1164955&view=diff
==============================================================================
--- incubator/chukwa/trunk/conf/hbase.schema (original)
+++ incubator/chukwa/trunk/conf/hbase.schema Sun Sep  4 02:36:48 2011
@@ -8,11 +8,25 @@ create "Hadoop",{NAME => "ClientTrace", 
 {NAME => "mapred_tasktracker", VERSIONS => 65535},
 {NAME => "mapred_job", VERSIONS => 65535},
 {NAME => "jvm_metrics", VERSIONS => 65535},
-{NAME => "rpc_metrics", VERSIONS => 65535}
+{NAME => "dfs", VERSIONS => 65535},
+{NAME => "jvm", VERSIONS => 65535},
+{NAME => "mapred_Queue", VERSIONS => 65535},
+{NAME => "metricssystem_MetricsSystem", VERSIONS => 65535},
+{NAME => "rpc_rpc", VERSIONS => 65535},
+{NAME => "rpcdetailed_rpcdetailed", VERSIONS => 65535},
+{NAME => "ugi_ugi", VERSIONS => 65535}
 create "HadoopLog", {NAME => "NameNode", VERSIONS => 65535},
 {NAME => "DataNode", VERSIONS => 65535},
 {NAME => "Audit", VERSIONS => 65535},
 {NAME => "HadoopLog", VERSIONS => 65535}
+create "HDFS", {NAME => "FileSystem"},
+{NAME => "namenode"},
+{NAME => "datanode"},
+{NAME => "clienttrace"}
+create "Mapreduce", {NAME => "jobs"},
+{NAME => "jobtracker"},
+{NAME => "tasktracker"},
+{NAME => "clienttrace"}
 create "SystemMetrics", 
 {NAME => "cpu", VERSIONS => 65535},
 {NAME => "system", VERSION => 65535},

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java?rev=1164955&r1=1164954&r2=1164955&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/writer/hbase/HBaseWriter.java
Sun Sep  4 02:36:48 2011
@@ -204,6 +204,7 @@ public class HBaseWriter extends Pipelin
               pool.putTable(hbase);
             }
           } catch (Exception e) {
+            log.warn(output.getKeyValues());
             log.warn(ExceptionUtil.getStackTrace(e));
           }
           dataSize += chunk.getData().length;

Added: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4jMetricsSink.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4jMetricsSink.java?rev=1164955&view=auto
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4jMetricsSink.java
(added)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/inputtools/log4j/Log4jMetricsSink.java
Sun Sep  4 02:36:48 2011
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.inputtools.log4j;
+
+import org.json.simple.JSONObject;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.hadoop.metrics2.Metric;
+import org.apache.hadoop.metrics2.MetricsException;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
+import org.apache.hadoop.metrics2.MetricsTag;
+
+public class Log4jMetricsSink implements MetricsSink {
+  /* Configuration attribute names */
+  private static final String HOST_PROPERTY = "host";
+  private static final String PORT_PROPERTY = "port";
+  private static final String TIMESTAMP = "timestamp";
+  private static String CONTEXT = "context";
+  private static final String CONTEXT_NAME = "contextName";
+  private static final String RECORD_NAME = "recordName";
+  protected String context = "HadoopMetrics";
+  protected String host = "localhost";
+  protected int port = 9095;
+  protected Logger out = null;
+
+  @Override
+  public void init(SubsetConfiguration conf) {
+    String host = conf.getString(HOST_PROPERTY);
+    if (host != null) {
+      this.host = host;
+    }
+    String port = conf.getString(PORT_PROPERTY);
+    if (port != null) {
+      this.port = Integer.parseInt(port);
+    }
+    String context = conf.getString(CONTEXT);
+    if (context != null) {
+      this.context = context;
+    }
+
+    PatternLayout layout = new PatternLayout("%d{ISO8601} %p %c: %m%n");
+
+    org.apache.log4j.net.SocketAppender appender = new org.apache.log4j.net.SocketAppender(this.host,
this.port);
+
+    appender.setName("chukwa.metrics." + this.context);
+    appender.setLayout(layout);
+
+    Logger logger = Logger.getLogger("chukwa.metrics." + this.context);
+    logger.setAdditivity(false);
+    logger.addAppender(appender);
+    appender.activateOptions();
+    out = logger;
+  }
+  
+  @Override
+  @SuppressWarnings("unchecked")
+  public void putMetrics(MetricsRecord record) {
+      JSONObject json = new JSONObject();
+      json.put(TIMESTAMP, Long.valueOf(record.timestamp()));
+      json.put(CONTEXT_NAME, record.context());
+      json.put(RECORD_NAME, record.name());
+      for (MetricsTag tag : record.tags()) {
+        json.put(tag.name(), tag.value());
+      }
+      for (Metric metric : record.metrics()) {
+        json.put(metric.name(), metric.value());
+      }
+      out.info(json);
+  }
+
+  @Override
+  public void flush() {
+  }
+}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.inputtools.log4j;
+
+import org.json.simple.JSONObject;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.hadoop.metrics2.Metric;
+import org.apache.hadoop.metrics2.MetricsException;
+import org.apache.hadoop.metrics2.MetricsRecord;
+import org.apache.hadoop.metrics2.MetricsSink;
+import org.apache.hadoop.metrics2.MetricsTag;
+
+public class Log4jMetricsSink implements MetricsSink {
+  /* Configuration attribute names */
+  private static final String HOST_PROPERTY = "host";
+  private static final String PORT_PROPERTY = "port";
+  private static final String TIMESTAMP = "timestamp";
+  private static String CONTEXT = "context";
+  private static final String CONTEXT_NAME = "contextName";
+  private static final String RECORD_NAME = "recordName";
+  protected String context = "HadoopMetrics";
+  protected String host = "localhost";
+  protected int port = 9095;
+  protected Logger out = null;
+
+  @Override
+  public void init(SubsetConfiguration conf) {
+    String host = conf.getString(HOST_PROPERTY);
+    if (host != null) {
+      this.host = host;
+    }
+    String port = conf.getString(PORT_PROPERTY);
+    if (port != null) {
+      this.port = Integer.parseInt(port);
+    }
+    String context = conf.getString(CONTEXT);
+    if (context != null) {
+      this.context = context;
+    }
+
+    PatternLayout layout = new PatternLayout("%d{ISO8601} %p %c: %m%n");
+
+    org.apache.log4j.net.SocketAppender appender = new org.apache.log4j.net.SocketAppender(this.host,
this.port);
+
+    appender.setName("chukwa.metrics." + this.context);
+    appender.setLayout(layout);
+
+    Logger logger = Logger.getLogger("chukwa.metrics." + this.context);
+    logger.setAdditivity(false);
+    logger.addAppender(appender);
+    appender.activateOptions();
+    out = logger;
+  }
+  
+  @Override
+  @SuppressWarnings("unchecked")
+  public void putMetrics(MetricsRecord record) {
+      JSONObject json = new JSONObject();
+      json.put(TIMESTAMP, Long.valueOf(record.timestamp()));
+      json.put(CONTEXT_NAME, record.context());
+      json.put(RECORD_NAME, record.name());
+      for (MetricsTag tag : record.tags()) {
+        json.put(tag.name(), tag.value());
+      }
+      for (Metric metric : record.metrics()) {
+        json.put(metric.name(), metric.value());
+      }
+      out.info(json);
+  }
+
+  @Override
+  public void flush() {
+  }
+}



Mime
View raw message