chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ey...@apache.org
Subject svn commit: r769588 - in /hadoop/chukwa: branches/chukwa-0.1/ branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/ trunk/ trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/
Date Tue, 28 Apr 2009 23:28:51 GMT
Author: eyang
Date: Tue Apr 28 23:28:50 2009
New Revision: 769588

URL: http://svn.apache.org/viewvc?rev=769588&view=rev
Log:
CHUKWA-193. Remove unrecognized tag from Job History Log file parser. (Cheng Zhang via Eric
Yang)

Modified:
    hadoop/chukwa/branches/chukwa-0.1/CHANGES.txt
    hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java

Modified: hadoop/chukwa/branches/chukwa-0.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/CHANGES.txt?rev=769588&r1=769587&r2=769588&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/CHANGES.txt (original)
+++ hadoop/chukwa/branches/chukwa-0.1/CHANGES.txt Tue Apr 28 23:28:50 2009
@@ -2,7 +2,7 @@
 
 Trunk (unrelease changes)
 
-Release 0.1.2 - 2009-04-24
+Release 0.1.2 - 2009-04-28
 
   NEW FEATURES
 
@@ -111,6 +111,8 @@
 
   BUG FIXES
 
+    CHUKWA-193. Remove unrecognized tag from Job History Log file parser. (Cheng Zhang via
Eric Yang)
+
     CHUKWA-189. Added JAVA_LIBRARY_PATH to chukwa-env.sh for enabling compression. (Jerome
Boulon via Eric Yang)
 
     CHUKWA-186. Remove duplicated timestamp from aggregation script. (Eric Yang)

Modified: hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java?rev=769588&r1=769587&r2=769588&view=diff
==============================================================================
--- hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
(original)
+++ hadoop/chukwa/branches/chukwa-0.1/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
Tue Apr 28 23:28:50 2009
@@ -22,6 +22,8 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
@@ -51,6 +53,7 @@
     }
     
     if(recordEntry.startsWith("Job") 
+        || recordEntry.startsWith("Meta") 
         || recordEntry.startsWith("Task") 
         || recordEntry.startsWith("MapAttempt") 
         || recordEntry.startsWith("ReduceAttempt")) 
@@ -66,16 +69,32 @@
     JobLogLine line = new JobLogLine(savedLines);
     return line;
   }
-
+  
 	@Override
 	protected void parse(String recordEntry,
 			OutputCollector<ChukwaRecordKey, ChukwaRecord> output,
 			Reporter reporter) throws Throwable 
 	{
 	  JobLogLine line = getJobLogLine(recordEntry);
-	  if(line == null) {
+	  if(line == null || (!line.getLogType().equals("Meta") 
+	                      && !line.getLogType().equals("JobData") 
+	                      && !line.getLogType().equals("TaskData"))) 
+	  {
 	    return;
 	  }
+	  
+    if(line.getLogType().equals("Meta")) {
+      String streamName = chunk.getStreamName();
+      if(streamName == null) {
+        return;
+      }
+      String jobId = JobLogFileName.getJobIdFromFileName(streamName);
+      if(jobId == null) {
+        return;
+      }
+      line.setLogType("JobData");
+    }
+    
 		key = new ChukwaRecordKey();
 		ChukwaRecord record = new ChukwaRecord();
 		this.buildGenericRecord(record, null, -1l, line.getLogType());
@@ -96,6 +115,9 @@
 	
 	private String getKey(long ts, String jobId) {
 		long unit = 60 * 60 * 1000;
+		if(ts == 0) {
+		  ts = archiveKey.getTimePartition();
+		}
 		long rounded = (ts / unit) * unit;
 		return rounded + "/" + jobId + "/" + ts;
 	}
@@ -180,9 +202,13 @@
 			}
 		}
 
-		public String getLogType() {
-			return logType;
-		}
+    public String getLogType() {
+      return logType;
+    }
+
+    public void setLogType(String logType) {
+      this.logType = logType;
+    }
 
 		public String getJobId() {
 			return jobId;
@@ -308,4 +334,19 @@
 		}
 		return result;
 	}
+
+	private static class JobLogFileName {
+    private static final Pattern pattern = Pattern.compile("job_[0-9]+_[0-9]+");
+    
+    public static String getJobIdFromFileName(String name) {
+      Matcher matcher = pattern.matcher(name);
+      if (matcher.find()) {
+        return matcher.group(0);
+      }
+      else {
+        return null;
+      }
+    }
+	}
+
 }
\ No newline at end of file

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=769588&r1=769587&r2=769588&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Tue Apr 28 23:28:50 2009
@@ -2,7 +2,7 @@
 
 Trunk (unreleased changes)
 
-Release 0.1.2 - 2009-04-24
+Release 0.1.2 - 2009-04-28
 
   INCOMPATIBLE CHANGES
 
@@ -117,6 +117,8 @@
 
   BUG FIXES
 
+    CHUKWA-193. Remove unrecognized tag from Job History Log file parser. (Cheng Zhang via
Eric Yang)
+
     CHUKWA-189. Added JAVA_LIBRARY_PATH to chukwa-env.sh for enabling compression. (Jerome
Boulon via Eric Yang)
 
     CHUKWA-186. Remove duplicated timestamp from aggregation script. (Eric Yang)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java?rev=769588&r1=769587&r2=769588&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
(original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/extraction/demux/processor/mapper/JobLog.java
Tue Apr 28 23:28:50 2009
@@ -22,6 +22,8 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
@@ -51,6 +53,7 @@
     }
     
     if(recordEntry.startsWith("Job") 
+        || recordEntry.startsWith("Meta") 
         || recordEntry.startsWith("Task") 
         || recordEntry.startsWith("MapAttempt") 
         || recordEntry.startsWith("ReduceAttempt")) 
@@ -66,16 +69,32 @@
     JobLogLine line = new JobLogLine(savedLines);
     return line;
   }
-
+  
 	@Override
 	protected void parse(String recordEntry,
 			OutputCollector<ChukwaRecordKey, ChukwaRecord> output,
 			Reporter reporter) throws Throwable 
 	{
 	  JobLogLine line = getJobLogLine(recordEntry);
-	  if(line == null) {
+	  if(line == null || (!line.getLogType().equals("Meta") 
+	                      && !line.getLogType().equals("JobData") 
+	                      && !line.getLogType().equals("TaskData"))) 
+	  {
 	    return;
 	  }
+	  
+    if(line.getLogType().equals("Meta")) {
+      String streamName = chunk.getStreamName();
+      if(streamName == null) {
+        return;
+      }
+      String jobId = JobLogFileName.getJobIdFromFileName(streamName);
+      if(jobId == null) {
+        return;
+      }
+      line.setLogType("JobData");
+    }
+    
 		key = new ChukwaRecordKey();
 		ChukwaRecord record = new ChukwaRecord();
 		this.buildGenericRecord(record, null, -1l, line.getLogType());
@@ -96,6 +115,9 @@
 	
 	private String getKey(long ts, String jobId) {
 		long unit = 60 * 60 * 1000;
+		if(ts == 0) {
+		  ts = archiveKey.getTimePartition();
+		}
 		long rounded = (ts / unit) * unit;
 		return rounded + "/" + jobId + "/" + ts;
 	}
@@ -180,9 +202,13 @@
 			}
 		}
 
-		public String getLogType() {
-			return logType;
-		}
+    public String getLogType() {
+      return logType;
+    }
+
+    public void setLogType(String logType) {
+      this.logType = logType;
+    }
 
 		public String getJobId() {
 			return jobId;
@@ -308,4 +334,19 @@
 		}
 		return result;
 	}
+
+	private static class JobLogFileName {
+    private static final Pattern pattern = Pattern.compile("job_[0-9]+_[0-9]+");
+    
+    public static String getJobIdFromFileName(String name) {
+      Matcher matcher = pattern.matcher(name);
+      if (matcher.find()) {
+        return matcher.group(0);
+      }
+      else {
+        return null;
+      }
+    }
+	}
+
 }
\ No newline at end of file



Mime
View raw message