hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r894240 - in /hadoop/hive/trunk: CHANGES.txt ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java build-common.xml contrib/build.xml ql/build.xml
Date Mon, 28 Dec 2009 20:04:43 GMT
Author: namit
Date: Mon Dec 28 20:04:42 2009
New Revision: 894240

URL: http://svn.apache.org/viewvc?rev=894240&view=rev
Log:
HIVE-1012. Ability to run testcases via regular expression
(Carl Steinbach via namit)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
    hadoop/hive/trunk/build-common.xml
    hadoop/hive/trunk/contrib/build.xml
    hadoop/hive/trunk/ql/build.xml

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Dec 28 20:04:42 2009
@@ -146,6 +146,9 @@
     HIVE-980. update readme.txt to build hive using eclipse
     (Patrick Angeles via namit)
 
+    HIVE-1012. Ability to run testcases via regular expression
+    (Carl Steinbach via namit)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Mon Dec 28 20:04:42
2009
@@ -18,10 +18,19 @@
 
 package org.apache.hadoop.hive.ant;
 
-import java.io.*;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.StringTokenizer;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.regex.Pattern;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.tools.ant.AntClassLoader;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.Task;
@@ -48,26 +57,44 @@
     }
     
   }
+  
+  public class QFileRegexFilter extends QFileFilter {
+    Pattern filterPattern;
+    
+    public QFileRegexFilter(String filter) {
+      filterPattern = Pattern.compile(filter);
+    }
+    
+    public boolean accept(File filePath) {
+      if (!super.accept(filePath)) {
+        return false;
+      }
+      String testName = StringUtils.chomp(filePath.getName(), ".q");
+      return filterPattern.matcher(testName).matches();
+    }
+  }
 
-  protected String templatePath;
-
-  protected String outputDirectory;
+  private List<String> templatePaths = new ArrayList<String>();
+  
+  private String outputDirectory;
  
-  protected String queryDirectory;
+  private String queryDirectory;
  
-  protected String queryFile;
+  private String queryFile;
+  
+  private String queryFileRegex;
 
-  protected String resultsDirectory;
+  private String resultsDirectory;
 
-  protected String logDirectory;
+  private String logDirectory;
 
-  protected String template;
+  private String template;
 
-  protected String className;
+  private String className;
 
-  protected String logFile;
+  private String logFile;
 
-  protected String clusterMode;
+  private String clusterMode;
 
   public void setClusterMode(String clusterMode) {
     this.clusterMode = clusterMode;
@@ -102,30 +129,21 @@
   }
 
   public void setTemplatePath(String templatePath) throws Exception {
-    StringBuffer resolvedPath = new StringBuffer();
-    StringTokenizer st = new StringTokenizer(templatePath, ",");
-    while (st.hasMoreTokens()) {
-      // resolve relative path from basedir and leave
-      // absolute path untouched.
-      File fullPath = project.resolveFile(st.nextToken());
-      resolvedPath.append(fullPath.getCanonicalPath());
-      if (st.hasMoreTokens()) {
-        resolvedPath.append(",");
-      }
+    templatePaths.clear();
+    for (String relativePath : templatePath.split(",")) {
+      templatePaths.add(project.resolveFile(relativePath).getCanonicalPath());
     }
-    this.templatePath = resolvedPath.toString();
-    System.out.println("Template Path:" + this.templatePath);
+    System.out.println("Template Path:" + getTemplatePath());
   }
 
   public String getTemplatePath() {
-    return templatePath;
+    return StringUtils.join(templatePaths, ",");
   }
 
   public void setOutputDirectory(File outputDirectory) {
     try {
       this.outputDirectory = outputDirectory.getCanonicalPath();
-    }
-    catch (IOException ioe) {
+    } catch (IOException ioe) {
       throw new BuildException(ioe);
     }
   }
@@ -139,7 +157,7 @@
   }
 
   public String getLogDirectory() {
-    return this.logDirectory;
+    return logDirectory;
   }
 
   public void setResultsDirectory(String resultsDirectory) {
@@ -147,7 +165,7 @@
   }
 
   public String getResultsDirectory() {
-    return this.resultsDirectory;
+    return resultsDirectory;
   }
 
   public void setQueryDirectory(String queryDirectory) {
@@ -155,7 +173,7 @@
   }
 
   public String getQueryDirectory() {
-    return this.queryDirectory;
+    return queryDirectory;
   }
 
   public void setQueryFile(String queryFile) {
@@ -163,12 +181,20 @@
   }
 
   public String getQueryFile() {
-    return this.queryFile;
+    return queryFile;
+  }
+  
+  public void setQueryFileRegex(String queryFileRegex) {
+    this.queryFileRegex = queryFileRegex;
+  }
+  
+  public String getQueryFileRegex() {
+    return queryFileRegex;
   }
 
   public void execute() throws BuildException {
 
-    if (templatePath == null) {
+    if (getTemplatePath().equals("")) {
       throw new BuildException("No templatePath attribute specified");
     }
 
@@ -196,7 +222,7 @@
       throw new BuildException("No className specified");
     }
 
-    File [] qFiles = null;
+    List<File> qFiles = new ArrayList<File>();
     File outDir = null;
     File resultsDir = null;
     File logDir = null;
@@ -209,17 +235,20 @@
 
       if (queryFile != null && !queryFile.equals("")) {
         // The user may have passed a list of files - comma seperated
-        String[] queryFiles = queryFile.split(",");
-        qFiles = new File[queryFiles.length];
-
-        for (int i = 0; i < queryFiles.length; i++) {
-          qFiles[i] = inpDir != null ? new File(inpDir, queryFiles[i]) : new File(queryFiles[i]);
+        for (String qFile : queryFile.split(",")) {
+          if (null != inpDir) {
+            qFiles.add(new File(inpDir, qFile));
+          } else {
+            qFiles.add(new File(qFile));
+          }
         }
+      } else if (queryFileRegex != null && !queryFileRegex.equals("")) {
+        qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileRegexFilter(queryFileRegex))));
+      } else {
+        qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileFilter())));
       }
-      else {
-        qFiles = inpDir.listFiles(new QFileFilter());
-        Arrays.sort(qFiles);
-      }
+      
+      Collections.sort(qFiles);
 
       // Make sure the output directory exists, if it doesn't
       // then create it.
@@ -237,15 +266,14 @@
       if (!resultsDir.exists()) {
         throw new BuildException("Results Directory " + resultsDir.getCanonicalPath() + "
does not exist");
       }
-    }
-    catch (Exception e) {
+    } catch (Exception e) {
       throw new BuildException(e);
     }
     
     VelocityEngine ve = new VelocityEngine();
 
     try {
-      ve.setProperty(RuntimeConstants.FILE_RESOURCE_LOADER_PATH, templatePath);
+      ve.setProperty(RuntimeConstants.FILE_RESOURCE_LOADER_PATH, getTemplatePath());
       if (logFile != null) {
         File lf = new File(logFile);
         if (lf.exists()) {
@@ -260,8 +288,9 @@
       ve.init();
       Template t = ve.getTemplate(template);
 
-      if (clusterMode == null) 
+      if (clusterMode == null) {
         clusterMode = new String("");
+      }
 
       // For each of the qFiles generate the test
       VelocityContext ctx = new VelocityContext();
@@ -277,22 +306,17 @@
       writer.close();
 
       System.out.println("Generated " + outFile.getCanonicalPath() + " from template " +
template);
-    }
-    catch(BuildException e) {
+    } catch(BuildException e) {
       throw e;
-    }
-    catch(MethodInvocationException e) {
+    } catch(MethodInvocationException e) {
       throw new BuildException("Exception thrown by '" + e.getReferenceName() + "." +
                                e.getMethodName() +"'",
                                e.getWrappedThrowable());
-    }
-    catch(ParseErrorException e) {
+    } catch(ParseErrorException e) {
       throw new BuildException("Velocity syntax error", e);
-    }
-    catch(ResourceNotFoundException e) {
+    } catch(ResourceNotFoundException e) {
       throw new BuildException("Resource not found", e);
-    }
-    catch(Exception e) {
+    } catch(Exception e) {
       throw new BuildException("Generation failed", e);
     }
   }

Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Mon Dec 28 20:04:42 2009
@@ -236,6 +236,12 @@
       </not>
     </condition>
 
+    <condition property="qfile_regex" value="">
+      <not>
+        <isset property="qfile_regex"/>
+      </not>
+    </condition>
+
     <condition property="overwrite" value="false">
       <not>
         <isset property="overwrite"/>

Modified: hadoop/hive/trunk/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/build.xml?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/build.xml (original)
+++ hadoop/hive/trunk/contrib/build.xml Mon Dec 28 20:04:42 2009
@@ -71,6 +71,7 @@
               templatePath="${ql.test.template.dir}" template="TestParse.vm" 
               queryDirectory="${contrib.test.query.dir}/positive"
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               resultsDirectory="${contrib.test.results.dir}/compiler" className="TestContribParse"
               logFile="${test.log.dir}/testcontribparsegen.log"
               logDirectory="${test.log.dir}/contribpositive"/>
@@ -79,6 +80,7 @@
               templatePath="${ql.test.template.dir}" template="TestParseNegative.vm" 
               queryDirectory="${contrib.test.query.dir}/negative" 
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               resultsDirectory="${contrib.test.results.dir}/compiler/errors" className="TestContribParseNegative"
               logFile="${test.log.dir}/testcontribparseneggen.log"
               logDirectory="${test.log.dir}/contribnegative"/>
@@ -87,6 +89,7 @@
               templatePath="${ql.test.template.dir}" template="TestCliDriver.vm" 
               queryDirectory="${contrib.test.query.dir}/clientpositive" 
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               clusterMode="${clustermode}"
               resultsDirectory="${contrib.test.results.dir}/clientpositive" className="TestContribCliDriver"
               logFile="${test.log.dir}/testcontribclidrivergen.log"
@@ -96,6 +99,7 @@
               templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"

               queryDirectory="${contrib.test.query.dir}/clientnegative" 
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               resultsDirectory="${contrib.test.results.dir}/clientnegative" className="TestContribNegativeCliDriver"
               logFile="${test.log.dir}/testcontribnegclidrivergen.log"
               logDirectory="${test.log.dir}/contribclientnegative"/>

Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=894240&r1=894239&r2=894240&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Mon Dec 28 20:04:42 2009
@@ -66,6 +66,7 @@
               templatePath="${ql.test.template.dir}" template="TestParse.vm" 
               queryDirectory="${ql.test.query.dir}/positive"
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               resultsDirectory="${ql.test.results.dir}/compiler" className="TestParse"
               logFile="${test.log.dir}/testparsegen.log"
               logDirectory="${test.log.dir}/positive"/>
@@ -74,6 +75,7 @@
               templatePath="${ql.test.template.dir}" template="TestParseNegative.vm" 
               queryDirectory="${ql.test.query.dir}/negative" 
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               resultsDirectory="${ql.test.results.dir}/compiler/errors" className="TestParseNegative"
               logFile="${test.log.dir}/testparseneggen.log"
               logDirectory="${test.log.dir}/negative"/>
@@ -82,6 +84,7 @@
               templatePath="${ql.test.template.dir}" template="TestCliDriver.vm" 
               queryDirectory="${ql.test.query.clientpositive.dir}" 
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               clusterMode="${clustermode}"
               resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver"
               logFile="${test.log.dir}/testclidrivergen.log"
@@ -91,6 +94,7 @@
               templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"

               queryDirectory="${ql.test.query.dir}/clientnegative" 
               queryFile="${qfile}"
+              queryFileRegex="${qfile_regex}"
               resultsDirectory="${ql.test.results.dir}/clientnegative" className="TestNegativeCliDriver"
               logFile="${test.log.dir}/testnegclidrivergen.log"
               logDirectory="${test.log.dir}/clientnegative"/>



Mime
View raw message