Return-Path: Delivered-To: apmail-hadoop-chukwa-commits-archive@minotaur.apache.org Received: (qmail 75245 invoked from network); 22 Apr 2009 23:27:37 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 22 Apr 2009 23:27:37 -0000 Received: (qmail 8965 invoked by uid 500); 22 Apr 2009 23:27:36 -0000 Delivered-To: apmail-hadoop-chukwa-commits-archive@hadoop.apache.org Received: (qmail 8954 invoked by uid 500); 22 Apr 2009 23:27:36 -0000 Mailing-List: contact chukwa-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: chukwa-dev@hadoop.apache.org Delivered-To: mailing list chukwa-commits@hadoop.apache.org Received: (qmail 8944 invoked by uid 99); 22 Apr 2009 23:27:36 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Apr 2009 23:27:36 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Apr 2009 23:27:34 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 2B68623888A2; Wed, 22 Apr 2009 23:27:13 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r767720 - in /hadoop/chukwa: branches/chukwa-0.1/conf/database_create_tables.sql branches/chukwa-0.1/conf/mdl.xml trunk/conf/database_create_tables.sql trunk/conf/mdl.xml Date: Wed, 22 Apr 2009 23:27:12 -0000 To: chukwa-commits@hadoop.apache.org From: eyang@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20090422232713.2B68623888A2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: eyang Date: Wed Apr 22 23:27:12 2009 New Revision: 767720 URL: http://svn.apache.org/viewvc?rev=767720&view=rev Log: CHUKWA-131. Added additional Mapred job/task metrics. (Eric Yang) Modified: hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml hadoop/chukwa/trunk/conf/database_create_tables.sql hadoop/chukwa/trunk/conf/mdl.xml Modified: hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql?rev=767720&r1=767719&r2=767720&view=diff ============================================================================== --- hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql (original) +++ hadoop/chukwa/branches/chukwa-0.1/conf/database_create_tables.sql Wed Apr 22 23:27:12 2009 @@ -439,10 +439,13 @@ reduce_input_records bigint default 0, reduce_output_records bigint default 0, jobconf text, - finished_maps default 0, - finished_reduces default 0, - failed_maps default 0, - failed_reduces default 0, + finished_maps bigint default 0, + finished_reduces bigint default 0, + failed_maps bigint default 0, + failed_reduces bigint default 0, + total_maps bigint default 0, + total_reduces bigint default 0, + reduce_shuffle_bytes bigint default 0, primary key(job_id), index(submit_time, finish_time, user, queue) ) ENGINE=InnoDB; @@ -471,6 +474,12 @@ reduce_output_records bigint default 0, reduce_input_bytes bigint default 0, reduce_output_bytes bigint default 0, + type VARCHAR(20), + reduce_shuffle_bytes bigint default 0, + hostname VARCHAR(80), + shuffle_finished timestamp default 0, + sort_finished timestamp default 0, + spilts bigint default 0, primary key(task_id), index(start_time, finish_time, job_id) ) ENGINE=InnoDB; @@ -621,4 +630,3 @@ primary key(timestamp), index(timestamp) ) ENGINE=InnoDB; - Modified: hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml?rev=767720&r1=767719&r2=767720&view=diff ============================================================================== --- hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml (original) +++ hadoop/chukwa/branches/chukwa-0.1/conf/mdl.xml Wed Apr 22 23:27:12 2009 @@ -1139,26 +1139,6 @@ NumOfMachines - - - report.db.primary.key.mssrgraph - timestamp - - - - metric.mssrgraph.jobid - job_id - - - metric.mssrgraph.count - count - - - - metric.mssrgraph.type - type - - report.db.primary.key.mrjobcounters @@ -1376,7 +1356,7 @@ - metric.jobdata.job_status + metric.jobdata.job_final_status status @@ -1420,6 +1400,21 @@ failed_reduces + + metric.jobdata.total_maps + total_maps + + + + metric.jobdata.total_reduces + total_reduces + + + + metric.jobdata.reduce_shuffle_bytes + reduce_shuffle_bytes + + metric.jobdata.counter:filesystemcounters:hdfs_bytes_read @@ -1658,6 +1653,26 @@ attempts + + metric.taskdata.hostname + hostname + + + + metric.taskdata.shuffle_finished + shuffle_finished + + + + metric.taskdata.sort_finished + sort_finished + + + + metric.taskdata.spilts + spilts + + metric.taskdata.counter:filesystemcounters:hdfs_bytes_read @@ -1743,6 +1758,11 @@ metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_output_records reduce_output_records + + + metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_shuffle_bytes + reduce_shuffle_bytes + @@ -1820,6 +1840,11 @@ metric.taskdata.counter:hadoop18:map-reduce-framework.reduce-output-records reduce_output_records + + + metric.taskdata.counter:hadoop18:map-reduce-framework.reduce_shuffle_bytes + reduce_shuffle_bytes + Modified: hadoop/chukwa/trunk/conf/database_create_tables.sql URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/database_create_tables.sql?rev=767720&r1=767719&r2=767720&view=diff ============================================================================== --- hadoop/chukwa/trunk/conf/database_create_tables.sql (original) +++ hadoop/chukwa/trunk/conf/database_create_tables.sql Wed Apr 22 23:27:12 2009 @@ -443,6 +443,9 @@ finished_reduces bigint default 0, failed_maps bigint default 0, failed_reduces bigint default 0, + total_maps bigint default 0, + total_reduces bigint default 0, + reduce_shuffle_bytes bigint default 0, primary key(job_id), index(submit_time, finish_time, user, queue) ) ENGINE=InnoDB; @@ -471,6 +474,12 @@ reduce_output_records bigint default 0, reduce_input_bytes bigint default 0, reduce_output_bytes bigint default 0, + type VARCHAR(20), + reduce_shuffle_bytes bigint default 0, + hostname VARCHAR(80), + shuffle_finished timestamp default 0, + sort_finished timestamp default 0, + spilts bigint default 0, primary key(task_id), index(start_time, finish_time, job_id) ) ENGINE=InnoDB; Modified: hadoop/chukwa/trunk/conf/mdl.xml URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/mdl.xml?rev=767720&r1=767719&r2=767720&view=diff ============================================================================== --- hadoop/chukwa/trunk/conf/mdl.xml (original) +++ hadoop/chukwa/trunk/conf/mdl.xml Wed Apr 22 23:27:12 2009 @@ -1139,26 +1139,6 @@ NumOfMachines - - - report.db.primary.key.mssrgraph - timestamp - - - - metric.mssrgraph.jobid - job_id - - - metric.mssrgraph.count - count - - - - metric.mssrgraph.type - type - - report.db.primary.key.mrjobcounters @@ -1376,7 +1356,7 @@ - metric.jobdata.job_status + metric.jobdata.job_final_status status @@ -1420,6 +1400,21 @@ failed_reduces + + metric.jobdata.total_maps + total_maps + + + + metric.jobdata.total_reduces + total_reduces + + + + metric.jobdata.reduce_shuffle_bytes + reduce_shuffle_bytes + + metric.jobdata.counter:filesystemcounters:hdfs_bytes_read @@ -1658,6 +1653,26 @@ attempts + + metric.taskdata.hostname + hostname + + + + metric.taskdata.shuffle_finished + shuffle_finished + + + + metric.taskdata.sort_finished + sort_finished + + + + metric.taskdata.spilts + spilts + + metric.taskdata.counter:filesystemcounters:hdfs_bytes_read @@ -1743,6 +1758,11 @@ metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_output_records reduce_output_records + + + metric.taskdata.counter:org.apache.hadoop.mapred.task$counter:reduce_shuffle_bytes + reduce_shuffle_bytes + @@ -1820,6 +1840,11 @@ metric.taskdata.counter:hadoop18:map-reduce-framework.reduce-output-records reduce_output_records + + + metric.taskdata.counter:hadoop18:map-reduce-framework.reduce_shuffle_bytes + reduce_shuffle_bytes +