hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r496837 - in /lucene/hadoop/trunk: ./ src/java/org/apache/hadoop/record/compiler/ src/java/org/apache/hadoop/record/compiler/ant/ src/java/org/apache/hadoop/record/compiler/generated/ src/test/ddl/ src/test/org/apache/hadoop/record/test/
Date Tue, 16 Jan 2007 19:46:47 GMT
Author: cutting
Date: Tue Jan 16 11:46:45 2007
New Revision: 496837

URL: http://svn.apache.org/viewvc?view=rev&rev=496837
Log:
HADOOP-852.  Add an ant task to compile record definitions.  Contributed by Milind.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
Removed:
    lucene/hadoop/trunk/src/test/ddl/links.jr
    lucene/hadoop/trunk/src/test/ddl/location.jr
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/RecBuffer.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/RecInt.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/RecRecord0.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/RecRecord1.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/RecString.java
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/build.xml
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JavaGenerator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Tue Jan 16 11:46:45 2007
@@ -11,6 +11,9 @@
     locations on the basis of network topology.
     (Hairong Kuang via cutting)
 
+ 3. HADOOP-852.  Add an ant task to compile record definitions, and
+    use it to compile record unit tests.  (Milind Bhandarkar via cutting)
+
 
 Release 0.10.1 - 2007-01-10
 

Modified: lucene/hadoop/trunk/build.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/build.xml?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/build.xml (original)
+++ lucene/hadoop/trunk/build.xml Tue Jan 16 11:46:45 2007
@@ -37,6 +37,7 @@
 
   <property name="test.src.dir" value="${basedir}/src/test"/>
   <property name="test.build.dir" value="${build.dir}/test"/>
+  <property name="test.generated.dir" value="${test.build.dir}/src"/>
   <property name="test.build.data" value="${test.build.dir}/data"/>
   <property name="test.cache.data" value="${test.build.dir}/cache"/>
   <property name="test.log.dir" value="${test.build.dir}/logs"/>
@@ -143,7 +144,27 @@
       />
   </target>
   
-  <target name="compile-core-classes" depends="init, record-parser">
+  <target name="compile-rcc-compiler" depends="init, record-parser">
+    <javac 
+        encoding="${build.encoding}" 
+        srcdir="${src.dir}"
+        includes="org/apache/hadoop/record/compiler/**/*.java"
+        destdir="${build.classes}"
+        debug="${javac.debug}"
+        optimize="${javac.optimize}"
+        target="${javac.version}"
+        source="${javac.version}"
+        deprecation="${javac.deprecation}">
+        <classpath refid="classpath"/>
+    </javac>
+    
+    <taskdef name="recordcc" classname="org.apache.hadoop.record.compiler.ant.RccTask">
+      <classpath refid="classpath" />
+    </taskdef>
+  </target>
+  
+  
+  <target name="compile-core-classes" depends="init, compile-rcc-compiler">
 
     <copy file="${src.webapps}/datanode/browseDirectory.jsp" todir="${src.webapps}/dfs/"/>
     <jsp-compile
@@ -319,11 +340,29 @@
     </jar>
   </target>
 
-
+  <target name="generate-test-records" depends="compile-rcc-compiler">
+    <recordcc destdir="${test.generated.dir}">
+      <fileset dir="${test.src.dir}"
+	         includes="**/*.jr" />
+    </recordcc>
+  </target>
+  
   <!-- ================================================================== -->
   <!-- Compile test code                                                  --> 
   <!-- ================================================================== -->
-  <target name="compile-core-test" depends="compile-examples">
+  <target name="compile-core-test" depends="compile-examples, generate-test-records">
+    <javac 
+     encoding="${build.encoding}" 
+     srcdir="${test.generated.dir}"
+     includes="org/apache/hadoop/**/*.java"
+     destdir="${test.build.classes}"
+     debug="${javac.debug}"
+     optimize="${javac.optimize}"
+     target="${javac.version}"
+     source="${javac.version}"
+     deprecation="${javac.deprecation}">
+      <classpath refid="test.classpath"/>
+    </javac>
     <javac 
      encoding="${build.encoding}" 
      srcdir="${test.src.dir}"

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Tue Jan
16 11:46:45 2007
@@ -30,7 +30,6 @@
  * @author Milind Bhandarkar
  */
 class CppGenerator {
-    private String mFullName;
     private String mName;
     private ArrayList mInclFiles;
     private ArrayList mRecList;
@@ -40,10 +39,10 @@
      * @param name possibly full pathname to the file
      * @param ilist included files (as JFile)
      * @param rlist List of records defined within this file
+     * @param destDir Output directory
      */
-    CppGenerator(String name, ArrayList ilist, ArrayList rlist) {
-        mFullName = name;
-        mName = (new File(name)).getName();
+    CppGenerator(String name, ArrayList ilist, ArrayList rlist, String destDir) {
+        mName = new File(destDir, (new File(name)).getName()).getAbsolutePath();
         mInclFiles = ilist;
         mRecList = rlist;
     }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java Tue Jan 16 11:46:45
2007
@@ -55,16 +55,17 @@
     /** Generate record code in given language. Language should be all
      *  lowercase.
      */
-    public void genCode(String language) throws IOException {
+    public int genCode(String language, String destDir) throws IOException {
         if ("c++".equals(language)) {
-            CppGenerator gen = new CppGenerator(mName, mInclFiles, mRecords);
+            CppGenerator gen = new CppGenerator(mName, mInclFiles, mRecords, destDir);
             gen.genCode();
         } else if ("java".equals(language)) {
-            JavaGenerator gen = new JavaGenerator(mName, mInclFiles, mRecords);
+            JavaGenerator gen = new JavaGenerator(mName, mInclFiles, mRecords, destDir);
             gen.genCode();
         } else {
-            System.out.println("Cannnot recognize language:"+language);
-            System.exit(1);
+            System.err.println("Cannnot recognize language:"+language);
+            return 1;
         }
+        return 0;
     }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java Tue Jan 16
11:46:45 2007
@@ -237,21 +237,19 @@
         
     }
     
-    public void genJavaCode() throws IOException {
+    public void genJavaCode(String destDir) throws IOException {
         String pkg = getJavaPackage();
         String pkgpath = pkg.replaceAll("\\.", "/");
-        File pkgdir = new File(pkgpath);
+        File pkgdir = new File(destDir, pkgpath);
         if (!pkgdir.exists()) {
             // create the pkg directory
             boolean ret = pkgdir.mkdirs();
             if (!ret) {
-                System.out.println("Cannnot create directory: "+pkgpath);
-                System.exit(1);
+                throw new IOException("Cannnot create directory: "+pkgpath);
             }
         } else if (!pkgdir.isDirectory()) {
             // not a directory
-            System.out.println(pkgpath+" is not a directory.");
-            System.exit(1);
+            throw new IOException(pkgpath+" is not a directory.");
         }
         File jfile = new File(pkgdir, getName()+".java");
         FileWriter jj = new FileWriter(jfile);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JavaGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JavaGenerator.java?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JavaGenerator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JavaGenerator.java Tue
Jan 16 11:46:45 2007
@@ -31,6 +31,7 @@
  */
 class JavaGenerator {
     private String mName;
+    private String destDir;
     private ArrayList mInclFiles;
     private ArrayList mRecList;
     
@@ -39,11 +40,13 @@
      * @param name possibly full pathname to the file
      * @param incl included files (as JFile)
      * @param records List of records defined within this file
+     * @param destDir output directory
      */
-    JavaGenerator(String name, ArrayList incl, ArrayList records) {
+    JavaGenerator(String name, ArrayList incl, ArrayList records, String destDir) {
         mName = name;
         mInclFiles = incl;
         mRecList = records;
+        this.destDir = destDir;
     }
     
     /**
@@ -53,7 +56,7 @@
     void genCode() throws IOException {
         for (Iterator i = mRecList.iterator(); i.hasNext(); ) {
             JRecord rec = (JRecord) i.next();
-            rec.genJavaCode();
+            rec.genJavaCode(destDir);
         }
     }
 }

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java?view=auto&rev=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java Tue Jan
16 11:46:45 2007
@@ -0,0 +1,136 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.record.compiler.ant;
+
+import java.io.File;
+import java.util.Vector;
+import org.apache.hadoop.record.compiler.generated.Rcc;
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.DirectoryScanner;
+import org.apache.tools.ant.Project;
+import org.apache.tools.ant.Task;
+import org.apache.tools.ant.types.FileSet;
+
+/**
+ * Hadoop record compiler ant Task
+ *<p> This task takes the given record definition files and compiles them into
+ * java or c++
+ * files. It is then up to the user to compile the generated files.
+ *
+ * <p> The task requires the <code>file</code> or the nested fileset element
to be
+ * specified. Optional attributes are <code>language</code> (set the output
+ * language, default is "java"),
+ * <code>destdir</code> (name of the destination directory for generated java/c++
+ * code, default is ".") and <code>failonerror</code> (specifies error handling
+ * behavior. default is true).
+ * <p><h4>Usage</h4>
+ * <pre>
+ * &lt;recordcc
+ *       destdir="${basedir}/gensrc"
+ *       language="java"&gt;
+ *   &lt;fileset include="**\/*.jr" /&gt;
+ * &lt;/recordcc&gt;
+ * </pre>
+ *
+ * @author Milind Bhandarkar
+ */
+public class RccTask extends Task {
+  
+  private String language = "java";
+  private File src;
+  private File dest = new File(".");
+  private final Vector<FileSet> filesets = new Vector();
+  private boolean failOnError = true;
+  
+  /** Creates a new instance of RccTask */
+  public RccTask() {
+  }
+  
+  /**
+   * Sets the output language option
+   * @param language "java"/"c++"
+   */
+  public void setLanguage(String language) {
+    this.language = language;
+  }
+  
+  /**
+   * Sets the record definition file attribute
+   * @param file record definition file
+   */
+  public void setFile(File file) {
+    this.src = file;
+  }
+  
+  /**
+   * Given multiple files (via fileset), set the error handling behavior
+   * @param flag true will throw build exception in case of failure (default)
+   */
+  public void setFailonerror(boolean flag) {
+    this.failOnError = flag;
+  }
+  
+  /**
+   * Sets directory where output files will be generated
+   * @param dir output directory
+   */
+  public void setDestdir(File dir) {
+    this.dest = dir;
+  }
+  
+  /**
+   * Adds a fileset that can consist of one or more files
+   * @param set Set of record definition files
+   */
+  public void addFileset(FileSet set) {
+    filesets.add(set);
+  }
+  
+  /**
+   * Invoke the Hadoop record compiler on each record definition file
+   */
+  public void execute() throws BuildException {
+    if (src == null && filesets.size()==0) {
+      throw new BuildException("There must be a file attribute or a fileset child element");
+    }
+    if (src != null) {
+      doCompile(src);
+    }
+    Project myProject = getProject();
+    for (int i = 0; i < filesets.size(); i++) {
+      FileSet fs = filesets.elementAt(i);
+      DirectoryScanner ds = fs.getDirectoryScanner(myProject);
+      File dir = fs.getDir(myProject);
+      String[] srcs = ds.getIncludedFiles();
+      for (int j = 0; j < srcs.length; j++) {
+        doCompile(new File(dir, srcs[j]));
+      }
+    }
+  }
+  
+  private void doCompile(File file) throws BuildException {
+    String[] args = new String[5];
+    args[0] = "--language";
+    args[1] = this.language;
+    args[2] = "--destdir";
+    args[3] = this.dest.getPath();
+    args[4] = file.getPath();
+    int retVal = Rcc.driver(args);
+    if (retVal != 0 && failOnError) {
+      throw new BuildException("Hadoop record compiler returned error code "+retVal);
+    }
+  }
+}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java Tue
Jan 16 11:46:45 2007
@@ -30,62 +30,71 @@
 
 public class Rcc implements RccConstants {
     private static String language = "java";
+    private static String destDir = ".";
     private static ArrayList recFiles = new ArrayList();
     private static JFile curFile;
     private static Hashtable recTab;
-    private static String curDir = System.getProperty("user.dir");
+    private static String curDir = ".";
     private static String curFileName;
     private static String curModuleName;
 
-    public static void main(String args[]) {
+    public static void main(String[] args) {
+        System.exit(driver(args));
+    }
+
+    public static int driver(String[] args) {
         for (int i=0; i<args.length; i++) {
             if ("-l".equalsIgnoreCase(args[i]) ||
                 "--language".equalsIgnoreCase(args[i])) {
                 language = args[i+1].toLowerCase();
                 i++;
+            } else if ("-d".equalsIgnoreCase(args[i]) ||
+                "--destdir".equalsIgnoreCase(args[i])) {
+                destDir = args[i+1];
+                i++;
             } else {
                 recFiles.add(args[i]);
             }
         }
         if (!"c++".equals(language) && !"java".equals(language)) {
-            System.out.println("Cannot recognize language:" + language);
-            System.exit(1);
+            System.err.println("Cannot recognize language:" + language);
+            return 1;
         }
         if (recFiles.size() == 0) {
-            System.out.println("No record files specified. Exiting.");
-            System.exit(1);
+            System.err.println("No record files specified. Exiting.");
+            return 1;
         }
         for (int i=0; i<recFiles.size(); i++) {
             curFileName = (String) recFiles.get(i);
-            File file = new File(curDir, curFileName);
+            File file = new File(curFileName);
             try {
                 FileReader reader = new FileReader(file);
                 Rcc parser = new Rcc(reader);
                 try {
                     recTab = new Hashtable();
                     curFile = parser.Input();
-                    System.out.println((String) recFiles.get(i) +
-                        " Parsed Successfully");
                 } catch (ParseException e) {
-                    System.out.println(e.toString());
-                    System.exit(1);
+                    System.err.println(e.toString());
+                    return 1;
                 }
                 try {
                     reader.close();
                 } catch (IOException e) {
                 }
             } catch (FileNotFoundException e) {
-                System.out.println("File " + (String) recFiles.get(i) +
+                System.err.println("File " + (String) recFiles.get(i) +
                     " Not found.");
-                System.exit(1);
+                return 1;
             }
             try {
-                curFile.genCode(language);
+                int retCode = curFile.genCode(language, destDir);
+                if (retCode != 0) { return retCode; }
             } catch (IOException e) {
-                System.out.println(e.toString());
-                System.exit(1);
+                System.err.println(e.toString());
+                return 1;
             }
         }
+        return 0;
     }
 
   final public JFile Input() throws ParseException {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj?view=diff&rev=496837&r1=496836&r2=496837
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj Tue Jan
16 11:46:45 2007
@@ -34,62 +34,71 @@
 
 public class Rcc {
     private static String language = "java";
+    private static String destDir = ".";
     private static ArrayList recFiles = new ArrayList();
     private static JFile curFile;
     private static Hashtable recTab;
-    private static String curDir = System.getProperty("user.dir");
+    private static String curDir = ".";
     private static String curFileName;
     private static String curModuleName;
 
-    public static void main(String args[]) {
+    public static void main(String[] args) {
+        System.exit(driver(args));
+    }
+ 
+    public static int driver(String[] args) {
         for (int i=0; i<args.length; i++) {
             if ("-l".equalsIgnoreCase(args[i]) ||
                 "--language".equalsIgnoreCase(args[i])) {
                 language = args[i+1].toLowerCase();
                 i++;
+            } else if ("-d".equalsIgnoreCase(args[i]) ||
+                "--destdir".equalsIgnoreCase(args[i])) {
+                destDir = args[i+1];
+                i++;
             } else {
                 recFiles.add(args[i]);
             }
         }
         if (!"c++".equals(language) && !"java".equals(language)) {
-            System.out.println("Cannot recognize language:" + language);
-            System.exit(1);
+            System.err.println("Cannot recognize language:" + language);
+            return 1;
         }
         if (recFiles.size() == 0) {
-            System.out.println("No record files specified. Exiting.");
-            System.exit(1);
+            System.err.println("No record files specified. Exiting.");
+            return 1;
         }
         for (int i=0; i<recFiles.size(); i++) {
             curFileName = (String) recFiles.get(i);
-            File file = new File(curDir, curFileName);
+            File file = new File(curFileName);
             try {
                 FileReader reader = new FileReader(file);
                 Rcc parser = new Rcc(reader);
                 try {
                     recTab = new Hashtable();
                     curFile = parser.Input();
-                    System.out.println((String) recFiles.get(i) +
-                        " Parsed Successfully");
                 } catch (ParseException e) {
-                    System.out.println(e.toString());
-                    System.exit(1);
+                    System.err.println(e.toString());
+                    return 1;
                 }
                 try {
                     reader.close();
                 } catch (IOException e) {
                 }
             } catch (FileNotFoundException e) {
-                System.out.println("File " + (String) recFiles.get(i) +
+                System.err.println("File " + (String) recFiles.get(i) +
                     " Not found.");
-                System.exit(1);
+                return 1;
             }
             try {
-                curFile.genCode(language);
+                int retCode = curFile.genCode(language, destDir);
+                if (retCode != 0) { return retCode; }
             } catch (IOException e) {
-                System.out.println(e.toString());
-                System.exit(1);
+                System.err.println(e.toString());
+                return 1;
             }
         }
+        return 0;
     }
 }
 



Mime
View raw message