chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r1037310 - in /incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa: datacollection/agent/ datacollection/agent/metrics/ datacollection/sender/ datacollection/sender/metrics/ extraction/demux/processor/mapper/
Date Sat, 20 Nov 2010 19:45:10 GMT
Author: eyang
Date: Sat Nov 20 19:45:10 2010
New Revision: 1037310

URL: http://svn.apache.org/viewvc?rev=1037310&view=rev
Log:
CHUKWA-557.  Improve metrics differentation between recordName and contextName and clean up
chukwa metrics.  (Eric Yang)

Added:
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChukwaMetricsProcessor.java
Modified:
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/AgentMetrics.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/ChunkQueueMetrics.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/metrics/HttpSenderMetrics.java
    incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/ChukwaAgent.java
Sat Nov 20 19:45:10 2010
@@ -76,7 +76,7 @@ import edu.berkeley.confspell.*;
  */
 public class ChukwaAgent implements AdaptorManager {
   // boolean WRITE_CHECKPOINTS = true;
-  static AgentMetrics agentMetrics = new AgentMetrics("ChukwaAgent", "chukwaAgent");
+  static AgentMetrics agentMetrics = new AgentMetrics("ChukwaAgent", "metrics");
 
   private static final int HTTP_SERVER_THREADS = 120;
   private static Server jettyServer = null;

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/MemLimitQueue.java
Sat Nov 20 19:45:10 2010
@@ -36,7 +36,7 @@ import org.apache.log4j.Logger;
  */
 public class MemLimitQueue implements ChunkQueue {
   static Logger log = Logger.getLogger(WaitingQueue.class);
-  static final ChunkQueueMetrics metrics = new ChunkQueueMetrics("ChukwaAgent", "chunkQueue");;
+  static final ChunkQueueMetrics metrics = new ChunkQueueMetrics("chukwaAgent", "chunkQueue");;
   private Queue<Chunk> queue = new LinkedList<Chunk>();
   private long dataSize = 0;
   private final long MAX_MEM_USAGE;

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/AgentMetrics.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/AgentMetrics.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/AgentMetrics.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/AgentMetrics.java
Sat Nov 20 19:45:10 2010
@@ -27,7 +27,7 @@ import org.apache.hadoop.metrics.util.Me
 import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt;
 
 public class AgentMetrics implements Updater {
-  public static final AgentMetrics agentMetrics = new AgentMetrics("ChukwaAgent", "chukwaAgent");;
+  public static final AgentMetrics agentMetrics = new AgentMetrics("chukwaAgent", "metrics");
   
   public MetricsRegistry registry = new MetricsRegistry();
   private MetricsRecord metricsRecord;
@@ -44,7 +44,7 @@ public class AgentMetrics implements Upd
   
   /** Creates a new instance of AgentMetrics */
   public AgentMetrics(String processName, String recordName) {
-      MetricsContext context = MetricsUtil.getContext(recordName);
+      MetricsContext context = MetricsUtil.getContext(processName);
       metricsRecord = MetricsUtil.createRecord(context, recordName);
       metricsRecord.setTag("process", processName);
       agentActivityMBean = new AgentActivityMBean(registry, recordName);

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/ChunkQueueMetrics.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/ChunkQueueMetrics.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/ChunkQueueMetrics.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/agent/metrics/ChunkQueueMetrics.java
Sat Nov 20 19:45:10 2010
@@ -52,7 +52,7 @@ public class ChunkQueueMetrics implement
   
   /** Creates a new instance of QueueMetrics */
   public ChunkQueueMetrics(String processName, String recordName) {
-      MetricsContext context = MetricsUtil.getContext(recordName);
+      MetricsContext context = MetricsUtil.getContext(processName);
       metricsRecord = MetricsUtil.createRecord(context, recordName);
       mbean = new ChunkQueueActivityMBean(registry, recordName);
       context.registerUpdater(this);

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/ChukwaHttpSender.java
Sat Nov 20 19:45:10 2010
@@ -73,7 +73,7 @@ public class ChukwaHttpSender implements
   public static final String COLLECTOR_TIMEOUT_OPT = "chukwaAgent.sender.collectorTimeout";
   // FIXME: this should really correspond to the timer in RetryListOfCollectors
 
-  static final HttpSenderMetrics metrics = new HttpSenderMetrics("ChukwaAgent", "chukwaHttpSender");
+  static final HttpSenderMetrics metrics = new HttpSenderMetrics("chukwaAgent", "httpSender");
   
   static Logger log = Logger.getLogger(ChukwaHttpSender.class);
   static HttpClient client = null;

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/metrics/HttpSenderMetrics.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/metrics/HttpSenderMetrics.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/metrics/HttpSenderMetrics.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/datacollection/sender/metrics/HttpSenderMetrics.java
Sat Nov 20 19:45:10 2010
@@ -49,7 +49,7 @@ public class HttpSenderMetrics implement
   
   /** Creates a new instance of HttpSenderMetrics */
   public HttpSenderMetrics(String processName, String recordName) {
-      MetricsContext context = MetricsUtil.getContext(recordName);
+      MetricsContext context = MetricsUtil.getContext(processName);
       metricsRecord = MetricsUtil.createRecord(context, recordName);
       metricsRecord.setTag("process", processName);
       mbean = new HttpSenderActivityMBean(registry, recordName);

Added: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChukwaMetricsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChukwaMetricsProcessor.java?rev=1037310&view=auto
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChukwaMetricsProcessor.java
(added)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ChukwaMetricsProcessor.java
Sat Nov 20 19:45:10 2010
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.extraction.demux.processor.mapper;
+
+import org.apache.hadoop.chukwa.datacollection.writer.hbase.Annotation.Tables;
+import org.apache.hadoop.chukwa.datacollection.writer.hbase.Annotation.Table;
+import org.apache.log4j.Logger;
+
+@Tables(annotations={
+@Table(name="chukwa",columnFamily="chukwaAgent_chunkQueue"),
+@Table(name="chukwa",columnFamily="chukwaAgent_metrics"),
+@Table(name="chukwa",columnFamily="chukwaAgent_httpSender")
+})
+public class ChukwaMetricsProcessor extends HadoopMetricsProcessor {  
+  static Logger log = Logger.getLogger(ChukwaMetricsProcessor.class);
+}

Modified: incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java?rev=1037310&r1=1037309&r2=1037310&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java
(original)
+++ incubator/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/HadoopMetricsProcessor.java
Sat Nov 20 19:45:10 2010
@@ -37,33 +37,33 @@ import org.json.JSONException;
 import org.json.JSONObject;
 
 @Tables(annotations={
-@Table(name="Hadoop",columnFamily="Hadoop_jvm_metrics"),
-@Table(name="Hadoop",columnFamily="Hadoop_mapred_metrics"),
-@Table(name="Hadoop",columnFamily="Hadoop_dfs_metrics"),
-@Table(name="Hadoop",columnFamily="Hadoop_dfs_namenode"),
-@Table(name="Hadoop",columnFamily="Hadoop_dfs_FSDirectory"),
-@Table(name="Hadoop",columnFamily="Hadoop_dfs_FSNamesystem"),
-@Table(name="Hadoop",columnFamily="Hadoop_dfs_datanode"),
-@Table(name="Hadoop",columnFamily="Hadoop_mapred_jobtracker"),
-@Table(name="Hadoop",columnFamily="Hadoop_mapred_shuffleInput"),
-@Table(name="Hadoop",columnFamily="Hadoop_mapred_shuffleOutput"),
-@Table(name="Hadoop",columnFamily="Hadoop_mapred_tasktracker"),
-@Table(name="Hadoop",columnFamily="Hadoop_mapred_job"),
-@Table(name="Hadoop",columnFamily="Hadoop_rpc_metrics")
+@Table(name="Hadoop",columnFamily="jvm_metrics"),
+@Table(name="Hadoop",columnFamily="mapred_metrics"),
+@Table(name="Hadoop",columnFamily="dfs_metrics"),
+@Table(name="Hadoop",columnFamily="dfs_namenode"),
+@Table(name="Hadoop",columnFamily="dfs_FSDirectory"),
+@Table(name="Hadoop",columnFamily="dfs_FSNamesystem"),
+@Table(name="Hadoop",columnFamily="dfs_datanode"),
+@Table(name="Hadoop",columnFamily="mapred_jobtracker"),
+@Table(name="Hadoop",columnFamily="mapred_shuffleInput"),
+@Table(name="Hadoop",columnFamily="mapred_shuffleOutput"),
+@Table(name="Hadoop",columnFamily="mapred_tasktracker"),
+@Table(name="Hadoop",columnFamily="mapred_job"),
+@Table(name="Hadoop",columnFamily="rpc_metrics")
 })
 public class HadoopMetricsProcessor extends AbstractProcessor {
-  public static final String jvm = "Hadoop_jvm_metrics";
-  public static final String mapred = "Hadoop_mapred_metrics";
-  public static final String dfs = "Hadoop_dfs_metrics";
-  public static final String namenode = "Hadoop_dfs_namenode";
-  public static final String fsdir = "Hadoop_dfs_FSDirectory";
-  public static final String fsname = "Hadoop_dfs_FSNamesystem";
-  public static final String datanode = "Hadoop_dfs_datanode";
-  public static final String jobtracker = "Hadoop_mapred_jobtracker";
-  public static final String shuffleIn = "Hadoop_mapred_shuffleInput";
-  public static final String shuffleOut = "Hadoop_mapred_shuffleOutput";
-  public static final String tasktracker = "Hadoop_mapred_tasktracker";
-  public static final String mr = "Hadoop_mapred_job";
+//  public static final String jvm = "jvm_metrics";
+//  public static final String mapred = "mapred_metrics";
+//  public static final String dfs = "dfs_metrics";
+//  public static final String namenode = "dfs_namenode";
+//  public static final String fsdir = "dfs_FSDirectory";
+//  public static final String fsname = "dfs_FSNamesystem";
+//  public static final String datanode = "dfs_datanode";
+//  public static final String jobtracker = "mapred_jobtracker";
+//  public static final String shuffleIn = "mapred_shuffleInput";
+//  public static final String shuffleOut = "mapred_shuffleOutput";
+//  public static final String tasktracker = "mapred_tasktracker";
+//  public static final String mr = "mapred_job";
   
   static Logger log = Logger.getLogger(HadoopMetricsProcessor.class);
   static final String chukwaTimestampField = "chukwa_timestamp";
@@ -100,7 +100,8 @@ public class HadoopMetricsProcessor exte
       JSONObject json = new JSONObject(body);
 
       ChukwaRecord record = new ChukwaRecord();
-      String datasource = null;
+      StringBuilder datasource = new StringBuilder();
+      String contextName = null;
       String recordName = null;
 
       Iterator<String> ki = json.keys();
@@ -114,7 +115,7 @@ public class HadoopMetricsProcessor exte
           cal.set(Calendar.MILLISECOND, 0);
           d.setTime(cal.getTimeInMillis());
         } else if (contextNameField.intern() == keyName.intern()) {
-          datasource = "Hadoop_" + json.getString(keyName);
+          contextName = json.getString(keyName);
         } else if (recordNameField.intern() == keyName.intern()) {
           recordName = json.getString(keyName);
           record.add(keyName, json.getString(keyName));
@@ -122,9 +123,12 @@ public class HadoopMetricsProcessor exte
           record.add(keyName, json.getString(keyName));
         }
       }
-
-      datasource = datasource + "_" + recordName;
-      buildGenericRecord(record, null, d.getTime(), datasource);
+      if(contextName!=null) {
+        datasource.append(contextName);
+        datasource.append("_");
+      }
+      datasource.append(recordName);
+      buildGenericRecord(record, null, d.getTime(), datasource.toString());
       output.collect(key, record);
     } catch (ParseException e) {
       e.printStackTrace();



Mime
View raw message