chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r773105 - in /hadoop/chukwa/trunk: conf/ src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/
Date Fri, 08 May 2009 21:19:00 GMT
Author: eyang
Date: Fri May  8 21:18:59 2009
New Revision: 773105

URL: http://svn.apache.org/viewvc?rev=773105&view=rev
Log:
Chukwa-205. Load ChukwaAgent's metrics to database.

Added:
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Log4JMetricsContextProcessor.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestLog4JMetricsContextChukwaRecord.java
Modified:
    hadoop/chukwa/trunk/conf/aggregator.sql
    hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml
    hadoop/chukwa/trunk/conf/database_create_tables.sql
    hadoop/chukwa/trunk/conf/mdl.xml

Modified: hadoop/chukwa/trunk/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/aggregator.sql?rev=773105&r1=773104&r2=773105&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/aggregator.sql (original)
+++ hadoop/chukwa/trunk/conf/aggregator.sql Fri May  8 21:18:59 2009
@@ -103,3 +103,18 @@
 replace into [util_quarter] (select timestamp,user,queue,sum(bytes)/3,sum(slot_hours)/3 from
[util_month] where timestamp between '[past_90_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/1800),user);
 replace into [util_year] (select timestamp,user,queue,sum(bytes)/8,sum(slot_hours)/8 from
[util_quarter] where timestamp between '[past_540_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/10800),user);
 replace into [util_decade] (select timestamp,user,queue,sum(bytes)/4,sum(slot_hours)/4 from
[util_year] where timestamp between '[past_2160_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/43200),user);
+#
+replace into [chunkqueue_month] (select chukwa_timestamp,recordname,hostname,contextname,AVG(removedchunk),avg(queuesize),AVG(removedchunk_raw),AVG(datasize),AVG(fullqueue),avg(addedchunk_rate),AVG(addedchunk_raw),AVG(period),AVG(addedchunk),avg(removedchunk_rate)
 from [chunkqueue_week] where chukwa_timestamp between '[past_15_minutes]' and '[now]' group
by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/300),recordname,hostname,contextname);
+replace into [chunkqueue_quarter] (select chukwa_timestamp,recordname,hostname,contextname,AVG(removedchunk),avg(queuesize),AVG(removedchunk_raw),AVG(datasize),AVG(fullqueue),avg(addedchunk_rate),AVG(addedchunk_raw),AVG(period),AVG(addedchunk),avg(removedchunk_rate)
 from [chunkqueue_month] where chukwa_timestamp between '[past_90_minutes]' and '[now]' group
by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/1800),recordname,hostname,contextname);
+replace into [chunkqueue_year] (select chukwa_timestamp,recordname,hostname,contextname,AVG(removedchunk),avg(queuesize),AVG(removedchunk_raw),AVG(datasize),AVG(fullqueue),avg(addedchunk_rate),AVG(addedchunk_raw),AVG(period),AVG(addedchunk),avg(removedchunk_rate)
 from [chunkqueue_quarter] where chukwa_timestamp between '[past_540_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/10800),recordname,hostname,contextname);
+replace into [chunkqueue_decade] (select chukwa_timestamp,recordname,hostname,contextname,AVG(removedchunk),avg(queuesize),AVG(removedchunk_raw),AVG(datasize),AVG(fullqueue),avg(addedchunk_rate),AVG(addedchunk_raw),AVG(period),AVG(addedchunk),avg(removedchunk_rate)
 from [chunkqueue_year] where chukwa_timestamp between '[past_2160_minutes]' and '[now]' group
by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/43200),recordname,hostname,contextname);
+#
+replace into [chukwaagent_month] (select chukwa_timestamp,recordname,hostname,contextname,avg(addedadaptor_rate),avg(addedadaptor_raw),avg(removedadaptor_rate),avg(removedadaptor),avg(period),avg(adaptorcount),avg(removedadaptor_raw),avg(process),avg(addedadaptor)
 from [chukwaagent_week] where chukwa_timestamp between '[past_15_minutes]' and '[now]' group
by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/300),recordname,hostname,contextname);
+replace into [chukwaagent_quarter] (select chukwa_timestamp,recordname,hostname,contextname,avg(addedadaptor_rate),avg(addedadaptor_raw),avg(removedadaptor_rate),avg(removedadaptor),avg(period),avg(adaptorcount),avg(removedadaptor_raw),avg(process),avg(addedadaptor)
 from [chukwaagent_month] where chukwa_timestamp between '[past_90_minutes]' and '[now]' group
by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/1800),recordname,hostname,contextname);
+replace into [chukwaagent_year] (select chukwa_timestamp,recordname,hostname,contextname,avg(addedadaptor_rate),avg(addedadaptor_raw),avg(removedadaptor_rate),avg(removedadaptor),avg(period),avg(adaptorcount),avg(removedadaptor_raw),avg(process),avg(addedadaptor)
 from [chukwaagent_quarter] where chukwa_timestamp between '[past_540_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/10800),recordname,hostname,contextname);
+replace into [chukwaagent_decade] (select chukwa_timestamp,recordname,hostname,contextname,avg(addedadaptor_rate),avg(addedadaptor_raw),avg(removedadaptor_rate),avg(removedadaptor),avg(period),avg(adaptorcount),avg(removedadaptor_raw),avg(process),avg(addedadaptor)
 from [chukwaagent_year] where chukwa_timestamp between '[past_2160_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/43200),recordname,hostname,contextname);
+#
+replace into [chukwahttpsender_month] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_week] where chukwa_timestamp between '[past_15_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/300),recordname,hostname,contextname);
+replace into [chukwahttpsender_quarter] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_month] where chukwa_timestamp between '[past_90_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/1800),recordname,hostname,contextname);
+replace into [chukwahttpsender_year] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_quarter] where chukwa_timestamp between '[past_540_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/10800),recordname,hostname,contextname);
+replace into [chukwahttpsender_decade] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_year] where chukwa_timestamp between '[past_2160_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/43200),recordname,hostname,contextname);

Modified: hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml?rev=773105&r1=773104&r2=773105&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml (original)
+++ hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml Fri May  8 21:18:59 2009
@@ -32,7 +32,7 @@
 <!-- PostProcessorManager config -->
   <property>
     <name>postProcessorManager.dbloader.ds</name>
-    <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage</value>
+    <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage,chunkQueue,chukwaAgent,chukwaHttpSender</value>
     <description>Reduce count </description>
   </property>
 
@@ -165,5 +165,23 @@
     <description></description>
    </property>
 
+   <property>
+    <name>chukwaAgent</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor</value>
+    <description></description>
+   </property>
+
+   <property>
+    <name>chunkQueue</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor</value>
+    <description></description>
+   </property>
+
+   <property>
+    <name>chukwaHttpSender</name>
+    <value>org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor</value>
+    <description></description>
+   </property>
+   
 </configuration>
  
\ No newline at end of file

Modified: hadoop/chukwa/trunk/conf/database_create_tables.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/database_create_tables.sql?rev=773105&r1=773104&r2=773105&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/database_create_tables.sql (original)
+++ hadoop/chukwa/trunk/conf/database_create_tables.sql Fri May  8 21:18:59 2009
@@ -631,3 +631,65 @@
      index(timestamp)
 ) ENGINE=InnoDB;
 
+create table if not exists chunkqueue_template (
+    chukwa_timestamp timestamp default CURRENT_TIMESTAMP,
+    hostname varchar(50),
+    recordname varchar(32),
+    contextname varchar(32),
+	removedchunk bigint,
+	queuesize int,
+	removedchunk_raw bigint,
+	datasize bigint,
+	fullqueue tinyint,
+	addedchunk_rate int,
+	addedchunk_raw bigint,
+	period bigint,
+	addedchunk bigint,
+	removedchunk_rate int,
+    primary key(chukwa_timestamp,hostname,recordname,contextname),
+    index (chukwa_timestamp)
+) ENGINE=InnoDB;
+
+create table if not exists chukwaagent_template (
+    chukwa_timestamp timestamp default CURRENT_TIMESTAMP,
+    hostname varchar(50),
+    recordname varchar(32),
+    contextname varchar(32),
+	addedadaptor_rate int,
+	addedadaptor_raw int,
+	removedadaptor_rate int,
+	removedadaptor int,
+	period int,
+	adaptorcount int,
+	removedadaptor_raw int,
+	process int,
+	addedadaptor int,
+    primary key(chukwa_timestamp,hostname,recordname,contextname),
+    index (chukwa_timestamp)
+) ENGINE=InnoDB;
+
+create table if not exists chukwahttpsender_template (
+    chukwa_timestamp timestamp default CURRENT_TIMESTAMP,
+    hostname varchar(50),
+    recordname varchar(32),
+    contextname varchar(32),
+	httppost_rate int,
+	httpthrowable_raw int,
+	httpexception_rate int,
+	httpthrowable int,
+	httpthrowable_rate int,
+	collectorrollover_rate int,
+	httppost_raw int,
+	period int,
+	httpexception_raw int,
+	httppost int,
+	httptimeoutexception int,
+	httptimeoutexception_raw int,
+	collectorrollover_raw int,
+	collectorrollover int,
+	httptimeoutexception_rate int,
+	httpexception int,
+    primary key(chukwa_timestamp,hostname,recordname,contextname),
+    index (chukwa_timestamp)
+) ENGINE=InnoDB;
+

Modified: hadoop/chukwa/trunk/conf/mdl.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/mdl.xml?rev=773105&r1=773104&r2=773105&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/mdl.xml (original)
+++ hadoop/chukwa/trunk/conf/mdl.xml Fri May  8 21:18:59 2009
@@ -1894,5 +1894,64 @@
   <value>5,30,180,720</value>
 </property>
 
+<!-- for agent metrics chunkqueue -->
+<property><name>report.db.name.chunkqueue</name><value>chunkqueue</value></property>
+<property><name>consolidator.table.chunkqueue</name><value>5,30,180,720</value></property>
+<property><name>metric.chunkqueue.chukwa_timestamp</name><value>chukwa_timestamp</value></property>
+<property><name>metric.chunkqueue.recordname</name><value>recordname</value></property>
+<property><name>metric.chunkqueue.hostname</name><value>hostname</value></property>
+<property><name>metric.chunkqueue.contextname</name><value>contextname</value></property>
+<property><name>metric.chunkqueue.removedchunk</name><value>removedchunk</value></property>
+<property><name>metric.chunkqueue.queuesize</name><value>queuesize</value></property>
+<property><name>metric.chunkqueue.removedchunk_raw</name><value>removedchunk_raw</value></property>
+<property><name>metric.chunkqueue.datasize</name><value>datasize</value></property>
+<property><name>metric.chunkqueue.fullqueue</name><value>fullqueue</value></property>
+<property><name>metric.chunkqueue.addedchunk_rate</name><value>addedchunk_rate</value></property>
+<property><name>metric.chunkqueue.addedchunk_raw</name><value>addedchunk_raw</value></property>
+<property><name>metric.chunkqueue.period</name><value>period</value></property>
+<property><name>metric.chunkqueue.addedchunk</name><value>addedchunk</value></property>
+<property><name>metric.chunkqueue.removedchunk_rate</name><value>removedchunk_rate</value></property>
+
+<!-- for agent metrics chukwaagent -->
+<property><name>report.db.name.chukwaagent</name><value>chukwaagent</value></property>
+<property><name>consolidator.table.chukwaagent</name><value>5,30,180,720</value></property>
+<property><name>metric.chukwaagent.chukwa_timestamp</name><value>chukwa_timestamp</value></property>
+<property><name>metric.chukwaagent.recordname</name><value>recordname</value></property>
+<property><name>metric.chukwaagent.hostname</name><value>hostname</value></property>
+<property><name>metric.chukwaagent.contextname</name><value>contextname</value></property>
+<property><name>metric.chukwaagent.addedadaptor_rate</name><value>addedadaptor_rate</value></property>
+<property><name>metric.chukwaagent.addedadaptor_raw</name><value>addedadaptor_raw</value></property>
+<property><name>metric.chukwaagent.removedadaptor_rate</name><value>removedadaptor_rate</value></property>
+<property><name>metric.chukwaagent.removedadaptor</name><value>removedadaptor</value></property>
+<property><name>metric.chukwaagent.period</name><value>period</value></property>
+<property><name>metric.chukwaagent.adaptorcount</name><value>adaptorcount</value></property>
+<property><name>metric.chukwaagent.removedadaptor_raw</name><value>removedadaptor_raw</value></property>
+<property><name>metric.chukwaagent.process</name><value>process</value></property>
+<property><name>metric.chukwaagent.addedadaptor</name><value>addedadaptor</value></property>
+
+<!-- for agent metrics chukwahttpsender -->
+<property><name>report.db.name.chukwahttpsender</name><value>chukwahttpsender</value></property>
+<property><name>consolidator.table.chukwahttpsender</name><value>5,30,180,720</value></property>
+<property><name>metric.chukwahttpsender.chukwa_timestamp</name><value>chukwa_timestamp</value></property>
+<property><name>metric.chukwahttpsender.recordname</name><value>recordname</value></property>
+<property><name>metric.chukwahttpsender.hostname</name><value>hostname</value></property>
+<property><name>metric.chukwahttpsender.contextname</name><value>contextname</value></property>
+<property><name>metric.chukwahttpsender.httppost_rate</name><value>httppost_rate</value></property>
+<property><name>metric.chukwahttpsender.httpthrowable_raw</name><value>httpthrowable_raw</value></property>
+<property><name>metric.chukwahttpsender.httpexception_rate</name><value>httpexception_rate</value></property>
+<property><name>metric.chukwahttpsender.httpthrowable</name><value>httpthrowable</value></property>
+<property><name>metric.chukwahttpsender.httpthrowable_rate</name><value>httpthrowable_rate</value></property>
+<property><name>metric.chukwahttpsender.collectorrollover_rate</name><value>collectorrollover_rate</value></property>
+<property><name>metric.chukwahttpsender.httppost_raw</name><value>httppost_raw</value></property>
+<property><name>metric.chukwahttpsender.period</name><value>period</value></property>
+<property><name>metric.chukwahttpsender.httpexception_raw</name><value>httpexception_raw</value></property>
+<property><name>metric.chukwahttpsender.httppost</name><value>httppost</value></property>
+<property><name>metric.chukwahttpsender.httptimeoutexception</name><value>httptimeoutexception</value></property>
+<property><name>metric.chukwahttpsender.httptimeoutexception_raw</name><value>httptimeoutexception_raw</value></property>
+<property><name>metric.chukwahttpsender.collectorrollover_raw</name><value>collectorrollover_raw</value></property>
+<property><name>metric.chukwahttpsender.collectorrollover</name><value>collectorrollover</value></property>
+<property><name>metric.chukwahttpsender.httptimeoutexception_rate</name><value>httptimeoutexception_rate</value></property>
+<property><name>metric.chukwahttpsender.httpexception</name><value>httpexception</value></property>
+
 </configuration>
 

Added: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Log4JMetricsContextProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Log4JMetricsContextProcessor.java?rev=773105&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Log4JMetricsContextProcessor.java
(added)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/Log4JMetricsContextProcessor.java
Fri May  8 21:18:59 2009
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.extraction.demux.processor.mapper;
+
+import java.util.Iterator;
+import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
+import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.log4j.Logger;
+import org.json.JSONObject;
+
+public class Log4JMetricsContextProcessor extends AbstractProcessor {
+
+  static Logger log = Logger.getLogger(Log4JMetricsContextProcessor.class);
+
+  @Override
+  protected void parse(String recordEntry,
+      OutputCollector<ChukwaRecordKey, ChukwaRecord> output, Reporter reporter)
+      throws Throwable 
+  {
+    Log4JMetricsContextChukwaRecord record = new Log4JMetricsContextChukwaRecord(recordEntry);
+    ChukwaRecord chukwaRecord = record.getChukwaRecord();
+    this.buildGenericRecord(chukwaRecord, null, record.getTimestamp(), record.getRecordType());
+    output.collect(key, chukwaRecord);
+  }
+
+  // create a static class to cove most of the code for unit test 
+  static class Log4JMetricsContextChukwaRecord {
+    private String recordType = null;
+    private long timestamp = 0;
+    private ChukwaRecord chukwaRecord = new ChukwaRecord();
+    
+    @SuppressWarnings("unchecked")
+    public Log4JMetricsContextChukwaRecord(String recordEntry) throws Throwable {
+      LogEntry log = new LogEntry(recordEntry);
+      JSONObject json = new JSONObject(log.getBody());
+
+      // round timestamp
+      timestamp = json.getLong("chukwa_timestamp");
+      timestamp = (timestamp / 60000) * 60000;
+
+      // get record type
+      String contextName = json.getString("contextName");
+      String recordName = json.getString("recordName");
+      recordType = contextName;
+      if (!contextName.equals(recordName)) {
+        recordType += "_" + recordName;
+      }
+
+      Iterator<String> ki = json.keys();
+      while (ki.hasNext()) {
+        String key = ki.next();
+        String value = json.getString(key);
+        if(value != null) {
+          chukwaRecord.add(key, value);
+        }
+      }
+    }
+
+    public String getRecordType() {
+      return recordType;
+    }
+
+    public long getTimestamp() {
+      return timestamp;
+    }
+    
+    public ChukwaRecord getChukwaRecord() {
+      return chukwaRecord;
+    }
+  }
+}
+

Added: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestLog4JMetricsContextChukwaRecord.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestLog4JMetricsContextChukwaRecord.java?rev=773105&view=auto
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestLog4JMetricsContextChukwaRecord.java
(added)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestLog4JMetricsContextChukwaRecord.java
Fri May  8 21:18:59 2009
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.chukwa.extraction.demux.processor.mapper;
+
+import java.util.ArrayList;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.chukwa.extraction.demux.processor.mapper.Log4JMetricsContextProcessor.Log4JMetricsContextChukwaRecord;
+import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
+
+public class TestLog4JMetricsContextChukwaRecord extends TestCase {
+  private static String[] chukwaQueueLog = { 
+      "2009-05-06 00:00:21,982 INFO chukwa.metrics.chunkQueue: {\"removedChunk\":1,\"recordName\":\"chunkQueue\",\"queueSize\":94,\"chukwa_timestamp\":1241568021982,\"removedChunk_raw\":0,\"dataSize\":10373608,\"fullQueue\":1,\"addedChunk_rate\":0,\"addedChunk_raw\":0,\"period\":60,\"addedChunk\":95,\"hostName\":\"test.com\",\"removedChunk_rate\":0,\"contextName\":\"chunkQueue\"}",
+      "2009-05-06 00:01:21,981 INFO chukwa.metrics.chunkQueue: {\"removedChunk\":1,\"recordName\":\"chunkQueue\",\"queueSize\":94,\"chukwa_timestamp\":1241568081981,\"removedChunk_raw\":0,\"dataSize\":10373608,\"fullQueue\":1,\"addedChunk_rate\":0,\"addedChunk_raw\":0,\"period\":60,\"addedChunk\":95,\"hostName\":\"test.com\",\"removedChunk_rate\":0,\"contextName\":\"chunkQueue\"}",
+      "2009-05-06 00:02:21,982 INFO chukwa.metrics.chunkQueue: {\"removedChunk\":1,\"recordName\":\"chunkQueue\",\"queueSize\":94,\"chukwa_timestamp\":1241568141982,\"removedChunk_raw\":0,\"dataSize\":10373608,\"fullQueue\":1,\"addedChunk_rate\":0,\"addedChunk_raw\":0,\"period\":60,\"addedChunk\":95,\"hostName\":\"test.com\",\"removedChunk_rate\":0,\"contextName\":\"chunkQueue\"}",
+  };
+  
+  private static String[] chukwaAgentLog = {
+    "2009-05-06 23:33:35,213 INFO chukwa.metrics.chukwaAgent: {\"addedAdaptor_rate\":0,\"addedAdaptor_raw\":0,\"recordName\":\"chukwaAgent\",\"chukwa_timestamp\":1241652815212,\"removedAdaptor_rate\":0,\"removedAdaptor\":0,\"period\":60,\"adaptorCount\":4,\"removedAdaptor_raw\":0,\"process\":\"ChukwaAgent\",\"addedAdaptor\":4,\"hostName\":\"test.com\",\"contextName\":\"chukwaAgent\"}",
+    "2009-05-06 23:34:35,211 INFO chukwa.metrics.chukwaAgent: {\"addedAdaptor_rate\":0,\"addedAdaptor_raw\":0,\"recordName\":\"chukwaAgent\",\"chukwa_timestamp\":1241652875211,\"removedAdaptor_rate\":0,\"removedAdaptor\":0,\"period\":60,\"adaptorCount\":4,\"removedAdaptor_raw\":0,\"process\":\"ChukwaAgent\",\"addedAdaptor\":4,\"hostName\":\"test.com\",\"contextName\":\"chukwaAgent\"}",
+    "2009-05-06 23:35:35,212 INFO chukwa.metrics.chukwaAgent: {\"addedAdaptor_rate\":0,\"addedAdaptor_raw\":0,\"recordName\":\"chukwaAgent\",\"chukwa_timestamp\":1241652935212,\"removedAdaptor_rate\":0,\"removedAdaptor\":0,\"period\":60,\"adaptorCount\":4,\"removedAdaptor_raw\":0,\"process\":\"ChukwaAgent\",\"addedAdaptor\":4,\"hostName\":\"test.com\",\"contextName\":\"chukwaAgent\"}",
+    "2009-05-06 23:39:35,215 INFO chukwa.metrics.chukwaAgent: {\"addedAdaptor_rate\":0,\"addedAdaptor_raw\":0,\"recordName\":\"chukwaAgent\",\"chukwa_timestamp\":1241653175214,\"removedAdaptor_rate\":0,\"removedAdaptor\":0,\"period\":60,\"adaptorCount\":4,\"removedAdaptor_raw\":0,\"process\":\"ChukwaAgent\",\"addedAdaptor\":4,\"hostName\":\"test.com\",\"contextName\":\"CA\"}",
+  };
+
+  public void testLog4JMetricsContextChukwaRecord() throws Throwable {
+    {
+      Log4JMetricsContextChukwaRecord rec = new Log4JMetricsContextChukwaRecord(chukwaQueueLog[0]);
+      ChukwaRecord chukwaRecord = rec.getChukwaRecord();
+      assertEquals("chunkQueue", rec.getRecordType());
+      assertEquals("1241568021982", chukwaRecord.getValue("chukwa_timestamp"));
+      assertEquals((1241568021982l/60000)*60000, rec.getTimestamp());
+      assertEquals("94", chukwaRecord.getValue("queueSize"));
+    }
+    
+    {
+      Log4JMetricsContextChukwaRecord rec = new Log4JMetricsContextChukwaRecord(chukwaAgentLog[3]);
+      assertEquals("CA_chukwaAgent", rec.getRecordType());
+      assertEquals(1241653175214l/60000*60000, rec.getTimestamp());
+    }
+  }
+
+}



Mime
View raw message