chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r774556 - in /hadoop/chukwa/trunk: conf/aggregator.sql conf/chukwa-demux-conf.xml conf/database_create_tables.sql conf/mdl.xml src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java
Date Wed, 13 May 2009 22:04:05 GMT
Author: eyang
Date: Wed May 13 22:04:05 2009
New Revision: 774556

URL: http://svn.apache.org/viewvc?rev=774556&view=rev
Log:
CHUKWA-210. Add new job conf metrics as database columns. (Cheng Zhang via Eric Yang)

Modified:
    hadoop/chukwa/trunk/conf/aggregator.sql
    hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml
    hadoop/chukwa/trunk/conf/database_create_tables.sql
    hadoop/chukwa/trunk/conf/mdl.xml
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java

Modified: hadoop/chukwa/trunk/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/aggregator.sql?rev=774556&r1=774555&r2=774556&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/aggregator.sql (original)
+++ hadoop/chukwa/trunk/conf/aggregator.sql Wed May 13 22:04:05 2009
@@ -118,3 +118,8 @@
 replace into [chukwahttpsender_quarter] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_month] where chukwa_timestamp between '[past_90_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/1800),recordname,hostname,contextname);
 replace into [chukwahttpsender_year] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_quarter] where chukwa_timestamp between '[past_540_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/10800),recordname,hostname,contextname);
 replace into [chukwahttpsender_decade] (select chukwa_timestamp,recordname,hostname,contextname,avg(httppost_rate),avg(httpthrowable_raw),avg(httpexception_rate),avg(httpthrowable),avg(httpthrowable_rate),avg(collectorrollover_rate),avg(httppost_raw),avg(period),avg(httpexception_raw),avg(httppost),avg(httptimeoutexception),avg(httptimeoutexception_raw),avg(collectorrollover_raw),avg(collectorrollover),avg(httptimeoutexception_rate),avg(httpexception)
 from [chukwahttpsender_year] where chukwa_timestamp between '[past_2160_minutes]' and '[now]'
group by FLOOR(UNIX_TIMESTAMP(chukwa_timestamp)/43200),recordname,hostname,contextname);
+
+replace into [mr_job_conf_month] (select * from [mr_job_conf_week] where ts between '[past_15_minutes]'
and '[now]');
+replace into [mr_job_conf_quarter] (select * from [mr_job_conf_week] where ts between '[past_15_minutes]'
and '[now]');
+replace into [mr_job_conf_year] (select * from [mr_job_conf_week] where ts between '[past_15_minutes]'
and '[now]');
+replace into [mr_job_conf_decade] (select * from [mr_job_conf_week] where ts between '[past_15_minutes]'
and '[now]');

Modified: hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml?rev=774556&r1=774555&r2=774556&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml (original)
+++ hadoop/chukwa/trunk/conf/chukwa-demux-conf.xml Wed May 13 22:04:05 2009
@@ -32,7 +32,7 @@
 <!-- PostProcessorManager config -->
   <property>
     <name>postProcessorManager.dbloader.ds</name>
-    <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage,chunkQueue,chukwaAgent,chukwaHttpSender</value>
+    <value>SystemMetrics,Df,Hadoop_dfs,Hadoop_jvm,Hadoop_mapred,Hadoop_rpc,MSSRGraph,MRJobCounters,NodeActivity,HodJob,HodMachine,Hadoop_dfs_FSDirectory,Hadoop_dfs_FSNamesystem,Hadoop_dfs_datanode,Hadoop_dfs_namenode,Hadoop_jvm_metrics,Hadoop_mapred_job,Hadoop_mapred_jobtracker,Hadoop_mapred_shuffleOutput,Hadoop_mapred_tasktracker,Hadoop_rpc_metrics,JobData,TaskData,HDFSUsage,chunkQueue,chukwaAgent,chukwaHttpSender,JobConfData</value>
     <description>Reduce count </description>
   </property>
 

Modified: hadoop/chukwa/trunk/conf/database_create_tables.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/database_create_tables.sql?rev=774556&r1=774555&r2=774556&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/database_create_tables.sql (original)
+++ hadoop/chukwa/trunk/conf/database_create_tables.sql Wed May 13 22:04:05 2009
@@ -693,3 +693,17 @@
     index (chukwa_timestamp)
 ) ENGINE=InnoDB;
 
+create table if not exists mr_job_conf_template (
+	ts  timestamp default CURRENT_TIMESTAMP,
+    job_id varchar(80),
+	mr_output_key_cls varchar(128) null,
+	mr_runner_cls varchar(128) null,
+	mr_output_value_cls varchar(128) null,
+	mr_input_fmt_cls varchar(128) null,
+	mr_output_fmt_cls varchar(128) null,
+	mr_reducer_cls varchar(128) null,
+	mr_mapper_cls varchar(128) null,
+    primary key(job_id),
+    index (ts)
+) ENGINE=InnoDB;
+

Modified: hadoop/chukwa/trunk/conf/mdl.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/mdl.xml?rev=774556&r1=774555&r2=774556&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/mdl.xml (original)
+++ hadoop/chukwa/trunk/conf/mdl.xml Wed May 13 22:04:05 2009
@@ -1953,5 +1953,16 @@
 <property><name>metric.chukwahttpsender.httptimeoutexception_rate</name><value>httptimeoutexception_rate</value></property>
 <property><name>metric.chukwahttpsender.httpexception</name><value>httpexception</value></property>
 
+<!-- for job conf parameters -->
+<property><name>report.db.name.jobconfdata</name><value>mr_job_conf</value></property>
+<property><name>consolidator.table.mr_job_conf</name><value>5,30,180,720</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.output.key.class</name><value>mr_output_key_cls</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.map.runner.class</name><value>mr_runner_cls</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.output.value.class</name><value>mr_output_value_cls</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.input.format.class</name><value>mr_input_fmt_cls</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.output.format.class</name><value>mr_output_fmt_cls</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.reducer.class</name><value>mr_reducer_cls</value></property>
+<property><name>metric.jobconfdata.job_conf.mapred.mapper.class</name><value>mr_mapper_cls</value></property>
+
 </configuration>
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java?rev=774556&r1=774555&r2=774556&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobConfProcessor.java
Wed May 13 22:04:05 2009
@@ -47,7 +47,8 @@
         if(matcher.matches()) {
           jobID=matcher.group(2);
         }
-    ChukwaRecord record = new ChukwaRecord();
+        ChukwaRecord record = new ChukwaRecord();
+        ChukwaRecord jobConfRecord = new ChukwaRecord();
       DocumentBuilderFactory docBuilderFactory 
         = DocumentBuilderFactory.newInstance();
       //ignore all comments inside the xml file
@@ -98,20 +99,25 @@
                 if(attr.intern()=="mapred.job.queue.name".intern()) {
                     queue=value;
                 }
+                jobConfRecord.add("job_conf." + attr, value);
             }
         }
-            record.add("JOBCONF-JSON", json.toString());
-            record.add("mapred.job.queue.name", queue);
-            record.add("JOBID", "job_"+jobID);
-        buildGenericRecord(record,null, time,"JobData");
-      calendar.setTimeInMillis(time);
-      calendar.set(Calendar.MINUTE, 0);
-      calendar.set(Calendar.SECOND, 0);
-      calendar.set(Calendar.MILLISECOND, 0);      
-      key.setKey(""+ calendar.getTimeInMillis() + "/job_" + jobID + "/" + time);
-                
-            output.collect(key,record);
-            tmp.delete();
+        record.add("JOBCONF-JSON", json.toString());
+        record.add("mapred.job.queue.name", queue);
+        record.add("JOBID", "job_" + jobID);
+        buildGenericRecord(record, null, time, "JobData");
+        calendar.setTimeInMillis(time);
+        calendar.set(Calendar.MINUTE, 0);
+        calendar.set(Calendar.SECOND, 0);
+        calendar.set(Calendar.MILLISECOND, 0);
+        key.setKey("" + calendar.getTimeInMillis() + "/job_" + jobID + "/" + time);
+        output.collect(key, record);
+
+        jobConfRecord.add("JOBID", "job_" + jobID);
+        buildGenericRecord(jobConfRecord, null, time, "JobConfData");
+        output.collect(key, jobConfRecord);
+            
+        tmp.delete();
       } catch(Exception e) {
           e.printStackTrace();  
           throw e;



Mime
View raw message