chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r773960 - in /hadoop/chukwa: branches/chukwa-0.1/bin/exec-data-loader.sh branches/chukwa-0.1/bin/systemDataLoader.sh branches/chukwa-0.1/conf/aggregator.sql trunk/bin/exec-data-loader.sh trunk/bin/systemDataLoader.sh trunk/conf/aggregator.sql
Date Tue, 12 May 2009 16:16:07 GMT
Author: eyang
Date: Tue May 12 16:16:07 2009
New Revision: 773960

URL: http://svn.apache.org/viewvc?rev=773960&view=rev
Log:
CHUKWA-226. Changed HDFS usage collection frequency from 10 minutes to 60 minutes. (Cheng
Zhang via Eric Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh
    hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh
    hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql
    hadoop/chukwa/trunk/bin/exec-data-loader.sh
    hadoop/chukwa/trunk/bin/systemDataLoader.sh
    hadoop/chukwa/trunk/conf/aggregator.sql

Modified: hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.1/bin/exec-data-loader.sh Tue May 12 16:16:07 2009
@@ -98,7 +98,7 @@
 fi
 
 if [ "X$PARM" == "XHDFSUsage" ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
 fi
 
 pid=$!

Modified: hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.1/bin/systemDataLoader.sh Tue May 12 16:16:07 2009
@@ -285,7 +285,7 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
 fi
 
 echo "done"

Modified: hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql (original)
+++ hadoop/chukwa/branches/chukwa-0.1/conf/aggregator.sql Tue May 12 16:16:07 2009
@@ -11,7 +11,7 @@
 #replace delayed into [hod_job_digest] (select timestamp,d.hodid,d.userid,[avg(system_metrics)]
from (select a.HodID,b.host as machine,a.userid,a.starttime,a.endtime from [HodJob] a join
[hod_machine] b on (a.HodID = b.HodID) where endtime between '[past_15_minutes]' and '[past_5_minutes]')
as d,[system_metrics] where timestamp between d.starttime and d.endtime and host=d.machine
group by hodid,timestamp);
 replace into [cluster_hadoop_rpc] (select timestamp, count(host), avg(rpc_processing_time_avg_time),
sum(rpc_processing_time_num_ops), avg(rpc_queue_time_avg_time), sum(rpc_queue_time_num_ops),
avg(get_build_version_avg_time), sum(get_build_version_num_ops), avg(get_job_counters_avg_time),
sum(get_job_counters_num_ops), avg(get_job_profile_avg_time), sum(get_job_profile_num_ops),
avg(get_job_status_avg_time), sum(get_job_status_num_ops), avg(get_new_job_id_avg_time), sum(get_new_job_id_num_ops),
avg(get_protocol_version_avg_time), sum(get_protocol_version_num_ops), avg(get_system_dir_avg_time),
sum(get_system_dir_num_ops), avg(get_task_completion_events_avg_time), sum(get_task_completion_events_num_ops),
avg(get_task_diagnostics_avg_time), sum(get_task_diagnostics_num_ops), avg(heartbeat_avg_time),
sum(heartbeat_num_ops), avg(killJob_avg_time), sum(killJob_num_ops), avg(submit_job_avg_time),
sum(submit_job_num_ops) from [hadoop_rpc] where timestamp between '[past_15_minutes]
 ' and '[past_5_minutes]' group by timestamp);
 #replace into [user_util] (select timestamp, j.UserID as user, sum(j.NumOfMachines) as node_total,
sum(cpu_idle_pcnt*j.NumOfMachines) as cpu_unused, sum((cpu_user_pcnt+cpu_system_pcnt)*j.NumOfMachines)
as cpu_used, avg(cpu_user_pcnt+cpu_system_pcnt) as cpu_used_pcnt, sum((100-(sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines)
as disk_unused, sum(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines)
as disk_used, avg(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)) as disk_used_pcnt,
sum((100-eth0_busy_pcnt)*j.NumOfMachines) as network_unused, sum(eth0_busy_pcnt*j.NumOfMachines)
as network_used, avg(eth0_busy_pcnt) as network_used_pcnt, sum((100-mem_used_pcnt)*j.NumOfMachines)
as memory_unused, sum(mem_used_pcnt*j.NumOfMachines) as memory_used, avg(mem_used_pcnt) as
memory_used_pcnt from [hod_job_digest] d,[HodJob] j where (d.HodID = j.HodID) and Timestamp
between '[past_15_minutes]' and '[past_5_minutes]' grou
 p by j.UserID);
-replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes)
as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as
timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts)
as slot_time from [mr_task],  (select job_id,user,queue,launch_time,finish_time from [mr_job]
where finish_time between '[past_20_minutes]' and '[now]') as job  where [mr_task].job_id=job.job_id
group by floor(unix_timestamp(timestamp)/600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user
and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/600),
user);
+replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes)
as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as
timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts)
as slot_time from [mr_task],  (select job_id,user,queue,launch_time,finish_time from [mr_job]
where finish_time between '[past_20_minutes]' and '[now]') as job  where [mr_task].job_id=job.job_id
group by floor(unix_timestamp(timestamp)/3600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user
and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/3600),
user);
 #
 # Down sample metrics for charts
 replace into [system_metrics_month] (select [group_avg(system_metrics)] from [system_metrics_week]
where timestamp between '[past_15_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/300),host);

Modified: hadoop/chukwa/trunk/bin/exec-data-loader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/exec-data-loader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/exec-data-loader.sh (original)
+++ hadoop/chukwa/trunk/bin/exec-data-loader.sh Tue May 12 16:16:07 2009
@@ -98,7 +98,7 @@
 fi
 
 if [ "X$PARM" == "XHDFSUsage" ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${COMMON}:${HADOOP_JAR}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
 fi
 
 pid=$!

Modified: hadoop/chukwa/trunk/bin/systemDataLoader.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/systemDataLoader.sh?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/systemDataLoader.sh (original)
+++ hadoop/chukwa/trunk/bin/systemDataLoader.sh Tue May 12 16:16:07 2009
@@ -285,7 +285,7 @@
 fi
 
 if [ ${EXISTS} -lt 1 ]; then
-  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
+  ${JAVA_HOME}/bin/java $JVM_OPTS -DPERIOD=3600 -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR}
-DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DRECORD_TYPE=HDFSUsage -Dlog4j.configuration=system-data-loader.properties
-classpath ${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${TOOLS}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.inputtools.jplugin.JPluginAgent org.apache.hadoop.chukwa.inputtools.hdfsusage.HDFSUsagePlugin
&
 fi
 
 echo "done"

Modified: hadoop/chukwa/trunk/conf/aggregator.sql
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/conf/aggregator.sql?rev=773960&r1=773959&r2=773960&view=diff
==============================================================================
--- hadoop/chukwa/trunk/conf/aggregator.sql (original)
+++ hadoop/chukwa/trunk/conf/aggregator.sql Tue May 12 16:16:07 2009
@@ -11,7 +11,7 @@
 #replace delayed into [hod_job_digest] (select timestamp,d.hodid,d.userid,[avg(system_metrics)]
from (select a.HodID,b.host as machine,a.userid,a.starttime,a.endtime from [HodJob] a join
[hod_machine] b on (a.HodID = b.HodID) where endtime between '[past_15_minutes]' and '[past_5_minutes]')
as d,[system_metrics] where timestamp between d.starttime and d.endtime and host=d.machine
group by hodid,timestamp);
 replace into [cluster_hadoop_rpc] (select timestamp, count(host), avg(rpc_processing_time_avg_time),
sum(rpc_processing_time_num_ops), avg(rpc_queue_time_avg_time), sum(rpc_queue_time_num_ops),
avg(get_build_version_avg_time), sum(get_build_version_num_ops), avg(get_job_counters_avg_time),
sum(get_job_counters_num_ops), avg(get_job_profile_avg_time), sum(get_job_profile_num_ops),
avg(get_job_status_avg_time), sum(get_job_status_num_ops), avg(get_new_job_id_avg_time), sum(get_new_job_id_num_ops),
avg(get_protocol_version_avg_time), sum(get_protocol_version_num_ops), avg(get_system_dir_avg_time),
sum(get_system_dir_num_ops), avg(get_task_completion_events_avg_time), sum(get_task_completion_events_num_ops),
avg(get_task_diagnostics_avg_time), sum(get_task_diagnostics_num_ops), avg(heartbeat_avg_time),
sum(heartbeat_num_ops), avg(killJob_avg_time), sum(killJob_num_ops), avg(submit_job_avg_time),
sum(submit_job_num_ops) from [hadoop_rpc] where timestamp between '[past_15_minutes]
 ' and '[past_5_minutes]' group by timestamp);
 #replace into [user_util] (select timestamp, j.UserID as user, sum(j.NumOfMachines) as node_total,
sum(cpu_idle_pcnt*j.NumOfMachines) as cpu_unused, sum((cpu_user_pcnt+cpu_system_pcnt)*j.NumOfMachines)
as cpu_used, avg(cpu_user_pcnt+cpu_system_pcnt) as cpu_used_pcnt, sum((100-(sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines)
as disk_unused, sum(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)*j.NumOfMachines)
as disk_used, avg(((sda_busy_pcnt+sdb_busy_pcnt+sdc_busy_pcnt+sdd_busy_pcnt)/4)) as disk_used_pcnt,
sum((100-eth0_busy_pcnt)*j.NumOfMachines) as network_unused, sum(eth0_busy_pcnt*j.NumOfMachines)
as network_used, avg(eth0_busy_pcnt) as network_used_pcnt, sum((100-mem_used_pcnt)*j.NumOfMachines)
as memory_unused, sum(mem_used_pcnt*j.NumOfMachines) as memory_used, avg(mem_used_pcnt) as
memory_used_pcnt from [hod_job_digest] d,[HodJob] j where (d.HodID = j.HodID) and Timestamp
between '[past_15_minutes]' and '[past_5_minutes]' grou
 p by j.UserID);
-replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes)
as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as
timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts)
as slot_time from [mr_task],  (select job_id,user,queue,launch_time,finish_time from [mr_job]
where finish_time between '[past_20_minutes]' and '[now]') as job  where [mr_task].job_id=job.job_id
group by floor(unix_timestamp(timestamp)/600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user
and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/600),
user);
+replace into [util] (select [hdfs_usage].timestamp as timestamp, mr_usage.user, queue, sum(bytes)
as bytes, sum(slot_time)/3600 as slot_hours from [hdfs_usage], (select job.finish_time as
timestamp,job.user,queue,sum(([mr_task].finish_time-[mr_task].start_time)*[mr_task].attempts)
as slot_time from [mr_task],  (select job_id,user,queue,launch_time,finish_time from [mr_job]
where finish_time between '[past_20_minutes]' and '[now]') as job  where [mr_task].job_id=job.job_id
group by floor(unix_timestamp(timestamp)/3600),user,queue) as mr_usage where mr_usage.user=[hdfs_usage].user
and [hdfs_usage].timestamp between '[past_20_minutes]' and '[now]' group by floor(unix_timestamp([hdfs_usage].timestamp)/3600),
user);
 #
 # Down sample metrics for charts
 replace into [system_metrics_month] (select [group_avg(system_metrics)] from [system_metrics_week]
where timestamp between '[past_15_minutes]' and '[now]' group by FLOOR(UNIX_TIMESTAMP(Timestamp)/300),host);



Mime
View raw message