chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r786700 - in /hadoop/chukwa: branches/chukwa-0.2/ branches/chukwa-0.2/bin/ branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/ trunk/ trunk/bin/ trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/
Date Fri, 19 Jun 2009 22:53:53 GMT
Author: eyang
Date: Fri Jun 19 22:53:52 2009
New Revision: 786700

URL: http://svn.apache.org/viewvc?rev=786700&view=rev
Log:
CHUKWA-276. Fix hourly and daily rolling to use a single reducer. (Jerome Boulon via Eric
Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
    hadoop/chukwa/branches/chukwa-0.2/bin/archive.sh
    hadoop/chukwa/branches/chukwa-0.2/bin/dailyRolling.sh
    hadoop/chukwa/branches/chukwa-0.2/bin/hourlyRolling.sh
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/bin/archive.sh
    hadoop/chukwa/trunk/bin/dailyRolling.sh
    hadoop/chukwa/trunk/bin/hourlyRolling.sh
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java

Modified: hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt (original)
+++ hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt Fri Jun 19 22:53:52 2009
@@ -34,6 +34,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-276. Fix hourly and daily rolling to use a single reducer. (Jerome Boulon via
Eric Yang)
+
     CHUKWA-313. Removed the 10 second ack from SeqFileWriter. (Eric Yang)
 
     CHUKWA-5. Adaptors have durable names. (asrabkin)

Modified: hadoop/chukwa/branches/chukwa-0.2/bin/archive.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/bin/archive.sh?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/bin/archive.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.2/bin/archive.sh Fri Jun 19 22:53:52 2009
@@ -38,5 +38,5 @@
 fi
 
 # run ArchiveManager
-${JAVA_HOME}/bin/java -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DAPP=archiveManager
-Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_HOME}/conf:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveManager
+${JAVA_HOME}/bin/java -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DAPP=archiveManager
-Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveManager
 

Modified: hadoop/chukwa/branches/chukwa-0.2/bin/dailyRolling.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/bin/dailyRolling.sh?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/bin/dailyRolling.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.2/bin/dailyRolling.sh Fri Jun 19 22:53:52 2009
@@ -32,5 +32,5 @@
 
 rm -f ${pidFile}
 
-${JAVA_HOME}/bin/java -DAPP=dailyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_PID_DIR=${CHUKWA_PID_DIR}
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling rollInSequence true deleteRawdata
true
+${JAVA_HOME}/bin/java -DAPP=dailyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_PID_DIR=${CHUKWA_PID_DIR}
-classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling rollInSequence true deleteRawdata
true
 

Modified: hadoop/chukwa/branches/chukwa-0.2/bin/hourlyRolling.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/bin/hourlyRolling.sh?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/bin/hourlyRolling.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.2/bin/hourlyRolling.sh Fri Jun 19 22:53:52 2009
@@ -31,5 +31,5 @@
 
 rm -f ${pidFile}
 
-${JAVA_HOME}/bin/java -DAPP=hourlyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_PID_DIR=${CHUKWA_PID_DIR} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling rollInSequence true deleteRawdata
true
+${JAVA_HOME}/bin/java -DAPP=hourlyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_PID_DIR=${CHUKWA_PID_DIR} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}::${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling rollInSequence true deleteRawdata
true
 

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
Fri Jun 19 22:53:52 2009
@@ -293,6 +293,7 @@
     FileInputFormat.setInputPaths(conf, args[0]);
     FileOutputFormat.setOutputPath(conf, new Path(args[1]));
     conf.setJobPriority(JobPriority.LOW);
+    conf.setNumReduceTasks(1);
     JobClient.runJob(conf);
     return 0;
   }

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
Fri Jun 19 22:53:52 2009
@@ -255,6 +255,8 @@
     FileInputFormat.setInputPaths(conf, args[0]);
     FileOutputFormat.setOutputPath(conf, new Path(args[1]));
     conf.setJobPriority(JobPriority.LOW);
+    conf.setNumReduceTasks(1);
+    
     JobClient.runJob(conf);
     return 0;
   }

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
Fri Jun 19 22:53:52 2009
@@ -193,9 +193,7 @@
         boolean rename = fs.rename(recordFile,destFilePath);
         done = true;
         log.info(">>>>>>>>>>>> after Rename" + destFilePath
+ " , rename:"+rename);
-      } else {
-        log.info("increment spill for :" + destFilePath);
-      }
+      } 
       count++;
 
       if (count > 1000) {

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Fri Jun 19 22:53:52 2009
@@ -32,6 +32,8 @@
 
   IMPROVEMENTS
 
+    CHUKWA-276. Fix hourly and daily rolling to use a single reducer. (Jerome Boulon via
Eric Yang)
+
     CHUKWA-313. Removed the 10 second ack from SeqFileWriter. (Eric Yang)
 
     CHUKWA-5. Adaptors have durable names. (asrabkin)

Modified: hadoop/chukwa/trunk/bin/archive.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/archive.sh?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/archive.sh (original)
+++ hadoop/chukwa/trunk/bin/archive.sh Fri Jun 19 22:53:52 2009
@@ -38,5 +38,5 @@
 fi
 
 # run ArchiveManager
-${JAVA_HOME}/bin/java -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DAPP=archiveManager
-Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_HOME}/conf:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveManager
+${JAVA_HOME}/bin/java -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DAPP=archiveManager
-Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.archive.ChukwaArchiveManager
 

Modified: hadoop/chukwa/trunk/bin/dailyRolling.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/dailyRolling.sh?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/dailyRolling.sh (original)
+++ hadoop/chukwa/trunk/bin/dailyRolling.sh Fri Jun 19 22:53:52 2009
@@ -32,5 +32,5 @@
 
 rm -f ${pidFile}
 
-${JAVA_HOME}/bin/java -DAPP=dailyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_PID_DIR=${CHUKWA_PID_DIR}
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling rollInSequence true deleteRawdata
true
+${JAVA_HOME}/bin/java -DAPP=dailyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_PID_DIR=${CHUKWA_PID_DIR}
-classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.demux.DailyChukwaRecordRolling rollInSequence true deleteRawdata
true
 

Modified: hadoop/chukwa/trunk/bin/hourlyRolling.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/bin/hourlyRolling.sh?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/bin/hourlyRolling.sh (original)
+++ hadoop/chukwa/trunk/bin/hourlyRolling.sh Fri Jun 19 22:53:52 2009
@@ -31,5 +31,5 @@
 
 rm -f ${pidFile}
 
-${JAVA_HOME}/bin/java -DAPP=hourlyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_PID_DIR=${CHUKWA_PID_DIR} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_CONF_DIR}
org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling rollInSequence true deleteRawdata
true
+${JAVA_HOME}/bin/java -DAPP=hourlyRolling -Dlog4j.configuration=chukwa-log4j.properties -DCHUKWA_HOME=${CHUKWA_HOME}
-DCHUKWA_PID_DIR=${CHUKWA_PID_DIR} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-classpath ${CHUKWA_CONF_DIR}:${HADOOP_CONF_DIR}:${CLASSPATH}::${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}
org.apache.hadoop.chukwa.extraction.demux.HourlyChukwaRecordRolling rollInSequence true deleteRawdata
true
 

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/DailyChukwaRecordRolling.java
Fri Jun 19 22:53:52 2009
@@ -293,6 +293,7 @@
     FileInputFormat.setInputPaths(conf, args[0]);
     FileOutputFormat.setOutputPath(conf, new Path(args[1]));
     conf.setJobPriority(JobPriority.LOW);
+    conf.setNumReduceTasks(1);
     JobClient.runJob(conf);
     return 0;
   }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/HourlyChukwaRecordRolling.java
Fri Jun 19 22:53:52 2009
@@ -255,6 +255,8 @@
     FileInputFormat.setInputPaths(conf, args[0]);
     FileOutputFormat.setOutputPath(conf, new Path(args[1]));
     conf.setJobPriority(JobPriority.LOW);
+    conf.setNumReduceTasks(1);
+    
     JobClient.runJob(conf);
     return 0;
   }

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java?rev=786700&r1=786699&r2=786700&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/MoveToRepository.java
Fri Jun 19 22:53:52 2009
@@ -193,9 +193,7 @@
         boolean rename = fs.rename(recordFile,destFilePath);
         done = true;
         log.info(">>>>>>>>>>>> after Rename" + destFilePath
+ " , rename:"+rename);
-      } else {
-        log.info("increment spill for :" + destFilePath);
-      }
+      } 
       count++;
 
       if (count > 1000) {



Mime
View raw message