chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r786339 - in /hadoop/chukwa: branches/chukwa-0.2/ branches/chukwa-0.2/bin/ branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/proce...
Date Fri, 19 Jun 2009 00:02:42 GMT
Author: eyang
Date: Fri Jun 19 00:02:41 2009
New Revision: 786339

URL: http://svn.apache.org/viewvc?rev=786339&view=rev
Log:
CHUKWA-314. Added filtering for start_time to remove invalid value. (Eric Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
    hadoop/chukwa/branches/chukwa-0.2/bin/mdl.sh
    hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
    hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
    hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
    hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java

Modified: hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt (original)
+++ hadoop/chukwa/branches/chukwa-0.2/CHANGES.txt Fri Jun 19 00:02:41 2009
@@ -64,6 +64,10 @@
 
   BUG FIXES
 
+    CHUKWA-314. Added filtering for start_time to remove invalid value. (Eric Yang)
+
+    CHUKWA-304. Propagate exception upward for database test cases. (Eric Yang)
+
     CHUKWA-309. Fix MDL configuration for mr_job_conf table. (Cheng Zhang via Eric Yang)
 
     CHUKWA-301. Set TestFileExpirationPolicy controller port to 0 to avoid race condition.
(Jerome Boulon via asrabkin)

Modified: hadoop/chukwa/branches/chukwa-0.2/bin/mdl.sh
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/bin/mdl.sh?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/bin/mdl.sh (original)
+++ hadoop/chukwa/branches/chukwa-0.2/bin/mdl.sh Fri Jun 19 00:02:41 2009
@@ -24,5 +24,5 @@
     exit 1
 fi
 
-${JAVA_HOME}/bin/java -Xms2048M -Xmx3096M -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -Djava.library.path=${JAVA_LIBRARY_PATH}
-DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-DAPP=MDL -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_HOME}/conf
org.apache.hadoop.chukwa.extraction.database.MetricDataLoader $@
+${JAVA_HOME}/bin/java -Xms2048M -Xmx3096M -DDATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml -Djava.library.path=${JAVA_LIBRARY_PATH}
-DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR}
-DAPP=MDL -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools}:${CHUKWA_HOME}/conf
org.apache.hadoop.chukwa.dataloader.MetricDataLoader $@
 

Modified: hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
Fri Jun 19 00:02:41 2009
@@ -173,7 +173,10 @@
 
 				int pos = pairs[i].lastIndexOf('"');
 				String value = pairs[i].substring(0, pos);
-				put(key, value);
+                                if(key.intern()=="START_TIME".intern() && value.intern()=="0".intern())
{
+                                } else {
+				  put(key, value);
+                                }
 				if(i == (pairs.length-1))
 					break;
 				key = pairs[i].substring(pos + 2);
@@ -349,4 +352,4 @@
     }
 	}
 
-}
\ No newline at end of file
+}

Modified: hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
Fri Jun 19 00:02:41 2009
@@ -87,3 +87,4 @@
         at org\.apache\.hadoop\.mapred\.Child\.main(Child\.java:156)
 " .
 MapAttempt TASK_TYPE="CLEANUP" TASKID="task_200904060626_2141_m_000197" TASK_ATTEMPT_ID="attempt_200904060626_2141_m_000197_0"
START_TIME="1239190961843" TRACKER_NAME="tracker_kry3083\.chukwa-example\.com:localhost/127\.0\.0\.1:60970"
HTTP_PORT="50060" .
+Task TASKID="task_200903062215_0577_m_000002" TASK_TYPE="MAP" START_TIME="0" SPLITS="/default-rack/test33\.chukwa-example\.com"
.

Modified: hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.2/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
Fri Jun 19 00:02:41 2009
@@ -90,6 +90,9 @@
 		assertEquals("1", counters.get("Counter:org.apache.hadoop.mapred.JobInProgress$Counter:TOTAL_LAUNCHED_REDUCES").toString());
 		assertEquals("20471", counters.get("Counter:FileSystemCounters:HDFS_BYTES_READ").toString());
 		
+		log = jobLog.getJobLogLine(testLogList.get(90));
+		assertTrue("START_TIME should not exist", log.get("START_TIME")==null);
+
 		log = jobLog.getJobLogLine("");
 		assertTrue(log==null);
 		

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Fri Jun 19 00:02:41 2009
@@ -64,6 +64,12 @@
 
   BUG FIXES
 
+    CHUKWA-304. Propagate exception upward for database test cases. (Eric Yang)
+
+    CHUKWA-314. Added filtering for start_time to remove invalid value. (Eric Yang)
+
+    CHUKWA-304. Propagate exception upward for database test cases. (Eric Yang)
+
     CHUKWA-309. Fix MDL configuration for mr_job_conf table. (Cheng Zhang via Eric Yang)
 
     CHUKWA-301. Set TestFileExpirationPolicy controller port to 0 to avoid race condition.
(Jerome Boulon via asrabkin)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
Fri Jun 19 00:02:41 2009
@@ -173,7 +173,10 @@
 
 				int pos = pairs[i].lastIndexOf('"');
 				String value = pairs[i].substring(0, pos);
-				put(key, value);
+				if(key.intern()=="START_TIME".intern() && value.intern()=="0".intern()) {
+				} else {
+	                          put(key, value);				  
+				}
 				if(i == (pairs.length-1))
 					break;
 				key = pairs[i].substring(pos + 2);
@@ -349,4 +352,4 @@
     }
 	}
 
-}
\ No newline at end of file
+}

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLog.txt
Fri Jun 19 00:02:41 2009
@@ -87,3 +87,4 @@
         at org\.apache\.hadoop\.mapred\.Child\.main(Child\.java:156)
 " .
 MapAttempt TASK_TYPE="CLEANUP" TASKID="task_200904060626_2141_m_000197" TASK_ATTEMPT_ID="attempt_200904060626_2141_m_000197_0"
START_TIME="1239190961843" TRACKER_NAME="tracker_kry3083\.chukwa-example\.com:localhost/127\.0\.0\.1:60970"
HTTP_PORT="50060" .
+Task TASKID="task_200903062215_0577_m_000002" TASK_TYPE="MAP" START_TIME="0" SPLITS="/default-rack/test33\.chukwa-example\.com"
.

Modified: hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java?rev=786339&r1=786338&r2=786339&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
(original)
+++ hadoop/chukwa/trunk/src/test/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/TestJobLogEntry.java
Fri Jun 19 00:02:41 2009
@@ -90,6 +90,9 @@
 		assertEquals("1", counters.get("Counter:org.apache.hadoop.mapred.JobInProgress$Counter:TOTAL_LAUNCHED_REDUCES").toString());
 		assertEquals("20471", counters.get("Counter:FileSystemCounters:HDFS_BYTES_READ").toString());
 		
+		log = jobLog.getJobLogLine(testLogList.get(90));
+		assertTrue("START_TIME should not exist", log.get("START_TIME")==null);
+
 		log = jobLog.getJobLogLine("");
 		assertTrue(log==null);
 		



Mime
View raw message