chukwa-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From asrab...@apache.org
Subject svn commit: r939397 - in /hadoop/chukwa/trunk: CHANGES.txt src/java/org/apache/hadoop/chukwa/util/CreateRecordFile.java
Date Thu, 29 Apr 2010 17:30:42 GMT
Author: asrabkin
Date: Thu Apr 29 17:30:42 2010
New Revision: 939397

URL: http://svn.apache.org/viewvc?rev=939397&view=rev
Log:
Allow CreateRecordFile to read job confs. Contributed by Bill Graham.

Modified:
    hadoop/chukwa/trunk/CHANGES.txt
    hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/CreateRecordFile.java

Modified: hadoop/chukwa/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/CHANGES.txt?rev=939397&r1=939396&r2=939397&view=diff
==============================================================================
--- hadoop/chukwa/trunk/CHANGES.txt (original)
+++ hadoop/chukwa/trunk/CHANGES.txt Thu Apr 29 17:30:42 2010
@@ -12,6 +12,8 @@ Trunk (unreleased changes)
 
   IMPROVEMENTS
 
+   CHUKWA-480.  Allow CreateRecordFile to read job confs. (Bill Graham via asrabkin)
+
     CHUKWA-471. Expose JobConf to Demux Processors. (Jerome Boulon via asrabkin)
 
     CHUKWA-472. Added ability to configure timestamp format for the default Time Series Processor.
 (Bill Graham via Eric Yang)

Modified: hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/CreateRecordFile.java
URL: http://svn.apache.org/viewvc/hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/CreateRecordFile.java?rev=939397&r1=939396&r2=939397&view=diff
==============================================================================
--- hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/CreateRecordFile.java (original)
+++ hadoop/chukwa/trunk/src/java/org/apache/hadoop/chukwa/util/CreateRecordFile.java Thu Apr
29 17:30:42 2010
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.chukwa.extraction.demux.processor.mapper.MapProcessor;
 import org.apache.hadoop.chukwa.extraction.demux.processor.mapper.TsProcessor;
+import org.apache.hadoop.chukwa.extraction.demux.Demux;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.ChunkImpl;
@@ -30,6 +31,8 @@ import org.apache.hadoop.chukwa.ChukwaAr
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reporter;
 
 import java.io.IOException;
 import java.io.File;
@@ -79,7 +82,7 @@ public class CreateRecordFile {
        archiveKey.setStreamName(chunk.getStreamName());
        archiveKey.setSeqId(chunk.getSeqID());
 
-       processor.process(archiveKey, chunk, collector, null);
+       processor.process(archiveKey, chunk, collector, Reporter.NULL);
        seqFileWriter.append(collector.getChukwaRecordKey(),
                             collector.getChukwaRecord());
      }
@@ -119,16 +122,34 @@ public class CreateRecordFile {
      String dataType = "testDataType";
      String streamName = "testStreamName";               
      MapProcessor processor = new TsProcessor();
+     Path confFile = null;
 
      if (args.length > 2) clusterName = args[2];
      if (args.length > 3) dataType = args[3];
      if (args.length > 4) streamName = args[4];
 
      if (args.length > 5) {
-       Class clazz = Class.forName(args[5]);
+       Class clazz = null;
+       try {
+         clazz = Class.forName(args[5]);
+       }
+       catch (ClassNotFoundException e) {
+         try {
+           clazz = Class.forName(
+                 "org.apache.hadoop.chukwa.extraction.demux.processor.mapper." + args[5]);
+         }
+         catch (Exception e2) {
+           throw e;
+         }
+       }
        processor = (MapProcessor)clazz.newInstance();
      }
 
+     if (args.length > 6) {
+       confFile = new Path(args[6]);
+       Demux.jobConf = new JobConf(confFile);
+     }
+
      System.out.println("Creating sequence file using the following input:");
      System.out.println("inputFile  : " + inputFile);
      System.out.println("outputFile : " + outputFile);
@@ -136,6 +157,7 @@ public class CreateRecordFile {
      System.out.println("dataType   : " + dataType);
      System.out.println("streamName : " + streamName);
      System.out.println("processor  : " + processor.getClass().getName());
+     System.out.println("confFile   : " + confFile);
 
      makeTestSequenceFile(inputFile, outputFile, clusterName, dataType, streamName, processor);
 
@@ -143,7 +165,7 @@ public class CreateRecordFile {
    }
 
    public static void usage() {
-     System.out.println("Usage: java " + TempFileUtil.class.toString().split(" ")[1] + "
<inputFile> <outputFile> [clusterName] [dataType] [streamName] [processorClass]");
+     System.out.println("Usage: java " + TempFileUtil.class.toString().split(" ")[1] + "
<inputFile> <outputFile> [<clusterName> <dataType> <streamName>
<processorClass> [confFile]]");
      System.out.println("Description: Takes a plain text input file and generates a Hadoop
sequence file contaning ChukwaRecordKey,ChukwaRecord entries");
      System.out.println("Parameters: inputFile      - Text input file to read");
      System.out.println("            outputFile     - Sequence file to create");
@@ -151,6 +173,7 @@ public class CreateRecordFile {
      System.out.println("            dataType       - Data type to use in the records");
      System.out.println("            streamName     - Stream name to use in the records");
      System.out.println("            processorClass - Processor class to use. Defaults to
TsProcessor");
+     System.out.println("            confFile       - File to use to create the JobConf");
      System.exit(0);
    }
 }



Mime
View raw message