chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r767720 - in /hadoop/chukwa: branches/chukwa-0.1/conf/database_create_tables.sql branches/chukwa-0.1/conf/mdl.xml trunk/conf/database_create_tables.sql trunk/conf/mdl.xml
Date Wed, 22 Apr 2009 23:27:12 GMT
Author: eyang
Date: Wed Apr 22 23:27:12 2009
New Revision: 767720

URL: http://svn.apache.org/viewvc?rev=767720&view=rev
Log:
CHUKWA-131. Added additional Mapred job/task metrics.  (Eric Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql
    hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml
    hadoop/chukwa/trunk/conf/database_create_tables.sql
    hadoop/chukwa/trunk/conf/mdl.xml

Modified: hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql?rev=767720&r1=767719&r2=767720&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql (original)
+++ hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql Wed Apr 22 23:27:12
2009
@@ -439,10 +439,13 @@
     reduce_input_records bigint default 0,
     reduce_output_records bigint default 0,
     jobconf  text, 
-    finished_maps default 0,
-    finished_reduces default 0,
-    failed_maps default 0,
-    failed_reduces default 0,
+    finished_maps bigint default 0,
+    finished_reduces bigint default 0,
+    failed_maps bigint default 0,
+    failed_reduces bigint default 0,
+    total_maps bigint default 0,
+    total_reduces bigint default 0,
+    reduce_shuffle_bytes bigint default 0,
     primary key(job_id),
     index(submit_time, finish_time, user, queue)
 ) ENGINE=InnoDB;
@@ -471,6 +474,12 @@
     reduce_output_records bigint default 0,
     reduce_input_bytes bigint default 0,
     reduce_output_bytes bigint default 0,
+    type VARCHAR(20),
+    reduce_shuffle_bytes bigint default 0,
+    hostname VARCHAR(80),
+    shuffle_finished timestamp default 0,
+    sort_finished timestamp default 0,
+    spilts bigint default 0,
     primary key(task_id),
     index(start_time, finish_time, job_id)
 ) ENGINE=InnoDB;
@@ -621,4 +630,3 @@
     primary key(timestamp),
      index(timestamp)
 ) ENGINE=InnoDB;
-

Modified: hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml?rev=767720&r1=767719&r2=767720&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml (original)
+++ hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml Wed Apr 22 23:27:12 2009
@@ -1139,26 +1139,6 @@
   <value>NumOfMachines</value>
 </property>
 
-<!-- MSSRGraph -->
-<property>
-  <name>report.db.primary.key.mssrgraph</name>
-  <value>timestamp</value>
-</property>
-
-<property>
-  <name>metric.mssrgraph.jobid</name>
-  <value>job_id</value>
-</property>
-<property>
-  <name>metric.mssrgraph.count</name>
-  <value>count</value>
-</property>
-
-<property>
-  <name>metric.mssrgraph.type</name>
-  <value>type</value>
-</property>
-
 <!-- Map Reduce Job Counters -->
 <property>
   <name>report.db.primary.key.mrjobcounters</name>
@@ -1376,7 +1356,7 @@
 </property>
 
 <property>
-  <name>metric.jobdata.job_status</name>
+  <name>metric.jobdata.job_final_status</name>
   <value>status</value>
 </property>
 
@@ -1420,6 +1400,21 @@
   <value>failed_reduces</value>
 </property>
 
+<property>
+  <name>metric.jobdata.total_maps</name>
+  <value>total_maps</value>
+</property>
+
+<property>
+  <name>metric.jobdata.total_reduces</name>
+  <value>total_reduces</value>
+</property>
+
+<property>
+  <name>metric.jobdata.reduce_shuffle_bytes</name>
+  <value>reduce_shuffle_bytes</value>
+</property>
+
 <!-- for hadoop 0.20 -->
 <property>
   <name>metric.jobdata.counter:filesystemcounters:hdfs_bytes_read</name>
@@ -1658,6 +1653,26 @@
   <value>attempts</value>
 </property>
 
+<property>
+  <name>metric.taskdata.hostname</name>
+  <value>hostname</value>
+</property>
+
+<property>
+  <name>metric.taskdata.shuffle_finished</name>
+  <value>shuffle_finished</value>
+</property>
+
+<property>
+  <name>metric.taskdata.sort_finished</name>
+  <value>sort_finished</value>
+</property>
+
+<property>
+  <name>metric.taskdata.spilts</name>
+  <value>spilts</value>
+</property>
+
 <!-- for hadoop 20 -->
 <property>
   <name>metric.taskdata.counter:filesystemcounters:hdfs_bytes_read</name>
@@ -1743,6 +1758,11 @@
   <name>metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_output_records</name>
   <value>reduce_output_records</value>
 </property>
+
+<property>
+  <name>metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_shuffle_bytes</name>
+  <value>reduce_shuffle_bytes</value>
+</property>
 <!-- end for hadoop 20 -->
 
 <!-- for hadoop 18 -->
@@ -1820,6 +1840,11 @@
   <name>metric.taskdata.counter:hadoop18:map-reduce-framework.reduce-output-records</name>
   <value>reduce_output_records</value>
 </property>
+
+<property>
+  <name>metric.taskdata.counter:hadoop18:map-reduce-framework.reduce_shuffle_bytes</name>
+  <value>reduce_shuffle_bytes</value>
+</property>
 <!-- end for hadoop 18 -->
 
 <property>

Modified: hadoop/chukwa/trunk/conf/database_create_tables.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/database_create_tables.sql?rev=767720&r1=767719&r2=767720&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/database_create_tables.sql (original)
+++ hadoop/chukwa/trunk/conf/database_create_tables.sql Wed Apr 22 23:27:12 2009
@@ -443,6 +443,9 @@
     finished_reduces bigint default 0,
     failed_maps bigint default 0,
     failed_reduces bigint default 0,
+    total_maps bigint default 0,
+    total_reduces bigint default 0,
+    reduce_shuffle_bytes bigint default 0,
     primary key(job_id),
     index(submit_time, finish_time, user, queue)
 ) ENGINE=InnoDB;
@@ -471,6 +474,12 @@
     reduce_output_records bigint default 0,
     reduce_input_bytes bigint default 0,
     reduce_output_bytes bigint default 0,
+    type VARCHAR(20),
+    reduce_shuffle_bytes bigint default 0,
+    hostname VARCHAR(80),
+    shuffle_finished timestamp default 0,
+    sort_finished timestamp default 0,
+    spilts bigint default 0,
     primary key(task_id),
     index(start_time, finish_time, job_id)
 ) ENGINE=InnoDB;

Modified: hadoop/chukwa/trunk/conf/mdl.xml
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/mdl.xml?rev=767720&r1=767719&r2=767720&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/mdl.xml (original)
+++ hadoop/chukwa/trunk/conf/mdl.xml Wed Apr 22 23:27:12 2009
@@ -1139,26 +1139,6 @@
   <value>NumOfMachines</value>
 </property>
 
-<!-- MSSRGraph -->
-<property>
-  <name>report.db.primary.key.mssrgraph</name>
-  <value>timestamp</value>
-</property>
-
-<property>
-  <name>metric.mssrgraph.jobid</name>
-  <value>job_id</value>
-</property>
-<property>
-  <name>metric.mssrgraph.count</name>
-  <value>count</value>
-</property>
-
-<property>
-  <name>metric.mssrgraph.type</name>
-  <value>type</value>
-</property>
-
 <!-- Map Reduce Job Counters -->
 <property>
   <name>report.db.primary.key.mrjobcounters</name>
@@ -1376,7 +1356,7 @@
 </property>
 
 <property>
-  <name>metric.jobdata.job_status</name>
+  <name>metric.jobdata.job_final_status</name>
   <value>status</value>
 </property>
 
@@ -1420,6 +1400,21 @@
   <value>failed_reduces</value>
 </property>
 
+<property>
+  <name>metric.jobdata.total_maps</name>
+  <value>total_maps</value>
+</property>
+
+<property>
+  <name>metric.jobdata.total_reduces</name>
+  <value>total_reduces</value>
+</property>
+
+<property>
+  <name>metric.jobdata.reduce_shuffle_bytes</name>
+  <value>reduce_shuffle_bytes</value>
+</property>
+
 <!-- for hadoop 0.20 -->
 <property>
   <name>metric.jobdata.counter:filesystemcounters:hdfs_bytes_read</name>
@@ -1658,6 +1653,26 @@
   <value>attempts</value>
 </property>
 
+<property>
+  <name>metric.taskdata.hostname</name>
+  <value>hostname</value>
+</property>
+
+<property>
+  <name>metric.taskdata.shuffle_finished</name>
+  <value>shuffle_finished</value>
+</property>
+
+<property>
+  <name>metric.taskdata.sort_finished</name>
+  <value>sort_finished</value>
+</property>
+
+<property>
+  <name>metric.taskdata.spilts</name>
+  <value>spilts</value>
+</property>
+
 <!-- for hadoop 20 -->
 <property>
   <name>metric.taskdata.counter:filesystemcounters:hdfs_bytes_read</name>
@@ -1743,6 +1758,11 @@
   <name>metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_output_records</name>
   <value>reduce_output_records</value>
 </property>
+
+<property>
+  <name>metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_shuffle_bytes</name>
+  <value>reduce_shuffle_bytes</value>
+</property>
 <!-- end for hadoop 20 -->
 
 <!-- for hadoop 18 -->
@@ -1820,6 +1840,11 @@
   <name>metric.taskdata.counter:hadoop18:map-reduce-framework.reduce-output-records</name>
   <value>reduce_output_records</value>
 </property>
+
+<property>
+  <name>metric.taskdata.counter:hadoop18:map-reduce-framework.reduce_shuffle_bytes</name>
+  <value>reduce_shuffle_bytes</value>
+</property>
 <!-- end for hadoop 18 -->
 
 <property>



Mime
View raw message