hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From athu...@apache.org
Subject svn commit: r779275 - in /hadoop/hive/trunk: ./ ant/src/org/apache/hadoop/hive/ant/ ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/templates/
Date Wed, 27 May 2009 18:57:46 GMT
Author: athusoo
Date: Wed May 27 18:57:45 2009
New Revision: 779275

URL: http://svn.apache.org/viewvc?rev=779275&view=rev
Log:
HIVE-204. Provide option to run tests with MiniMRCluster.
(Namit Jain via athusoo)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
    hadoop/hive/trunk/build-common.xml
    hadoop/hive/trunk/build.properties
    hadoop/hive/trunk/ql/build.xml
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
    hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed May 27 18:57:45 2009
@@ -47,6 +47,9 @@
     HIVE-502. Optimize RCFile's performance by removeing all synchronized
     modifiers. (Yongqiang He via zshao)
 
+    HIVE-204. Provide option to run tests with MiniMRCluster.
+    (Namit Jain via athusoo)
+
   OPTIMIZATIONS
 
     HIVE-279. Predicate Pushdown support (Prasad Chakka via athusoo).

Modified: hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Wed May 27 18:57:45
2009
@@ -56,7 +56,7 @@
   protected String queryDirectory;
  
   protected String queryFile;
- 
+
   protected String resultsDirectory;
 
   protected String logDirectory;
@@ -67,6 +67,16 @@
 
   protected String logFile;
 
+  protected String clusterMode;
+
+  public void setClusterMode(String clusterMode) {
+    this.clusterMode = clusterMode;
+  }
+
+  public String getClusterMode() {
+    return clusterMode;
+  }
+
   public void setLogFile(String logFile) {
     this.logFile = logFile;
   }
@@ -198,8 +208,13 @@
       }
 
       if (queryFile != null && !queryFile.equals("")) {
-        qFiles = new File[1];
-        qFiles[0] = inpDir != null ? new File(inpDir, queryFile) : new File(queryFile);
+        // The user may have passed a list of files - comma seperated
+        String[] queryFiles = queryFile.split(",");
+        qFiles = new File[queryFiles.length];
+
+        for (int i = 0; i < queryFiles.length; i++) {
+          qFiles[i] = inpDir != null ? new File(inpDir, queryFiles[i]) : new File(queryFiles[i]);
+        }
       }
       else {
         qFiles = inpDir.listFiles(new QFileFilter());
@@ -245,12 +260,16 @@
       ve.init();
       Template t = ve.getTemplate(template);
 
+      if (clusterMode == null) 
+        clusterMode = new String("");
+
       // For each of the qFiles generate the test
       VelocityContext ctx = new VelocityContext();
       ctx.put("className", className);
       ctx.put("qfiles", qFiles);
       ctx.put("resultsDir", resultsDir);
       ctx.put("logDir", logDir);
+      ctx.put("clusterMode", clusterMode);
 
       File outFile = new File(outDir, className + ".java");
       FileWriter writer = new FileWriter(outFile);

Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Wed May 27 18:57:45 2009
@@ -57,6 +57,7 @@
   <property name="test.timeout" value="5400000"/>
   <property name="test.junit.output.format" value="xml"/>
   <property name="test.junit.output.usefile" value="true"/>
+  <property name="minimr.query.files" value="join1.q,groupby1.q"/>
   <property name="test.silent" value="true"/>
 
   <path id="test.classpath">
@@ -205,21 +206,41 @@
   </target>
 
   <target name="test-conditions">
+        
+    <condition property="qfile" value="${minimr.query.files}">
+      <and>
+        <not>
+          <isset property="qfile"/>
+        </not>
+      
+        <equals arg1="${clustermode}" arg2="miniMR" /> 
+      </and>
+    </condition>
+
     <condition property="qfile" value="">
       <not>
         <isset property="qfile"/>
       </not>
     </condition>
+
     <condition property="overwrite" value="false">
       <not>
         <isset property="overwrite"/>
       </not>
     </condition>
+
     <condition property="standalone" value="false">
       <not>
         <isset property="standalone"/>
       </not>
     </condition>
+
+    <condition property="clustermode" value="">
+      <not>
+        <isset property="clustermode"/>
+      </not>
+    </condition>
+
   </target>
 
   <!-- target to deploy anttasks -->
@@ -253,10 +274,11 @@
       <sysproperty key="test.service.standalone.server" value="${standalone}"/>
       <sysproperty key="log4j.configuration" value="file://${test.data.dir}/conf/hive-log4j.properties"/>
       <sysproperty key="derby.stream.error.file" value="${test.build.dir}/derby.log"/>
-      <sysproperty key="hive.aux.jars.path" value="${test.build.dir}/test-udfs.jar,${test.src.data.dir}/files/TestSerDe.jar"/>
+      <sysproperty key="hive.aux.jars.path" value="file://${test.build.dir}/test-udfs.jar,file://${test.src.data.dir}/files/TestSerDe.jar"/>
       <sysproperty key="ql.test.query.clientpositive.dir" value="${ql.test.query.clientpositive.dir}"/>
       <sysproperty key="ql.test.results.clientpositive.dir" value="${ql.test.results.clientpositive.dir}"/>
       <sysproperty key="test.log.dir" value="${test.log.dir}"/>
+      <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
       <sysproperty key="test.silent" value="${test.silent}"/>
 
       <classpath refid="${test.classpath.id}"/>

Modified: hadoop/hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.properties?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/build.properties (original)
+++ hadoop/hive/trunk/build.properties Wed May 27 18:57:45 2009
@@ -12,3 +12,10 @@
 hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version}
 hadoop.root=${hadoop.root.default}
 hadoop.jar=${hadoop.root}/hadoop-${hadoop.version}-core.jar
+hadoop.test.jar=${hadoop.root}/hadoop-${hadoop.version}-test.jar
+jetty.test.jar=${hadoop.root}/lib/jetty-5.1.4.jar
+servlet.test.jar=${hadoop.root}/lib/servlet-api.jar
+jasper.test.jar=${hadoop.root}/lib/jetty-ext/jasper-runtime.jar
+jasperc.test.jar=${hadoop.root}/lib/jetty-ext/jasper-compiler.jar
+jsp.test.jar=${hadoop.root}/lib/jetty-ext/jsp-api.jar
+common.jar=${hadoop.root}/lib/commons-httpclient-3.0.1.jar

Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Wed May 27 18:57:45 2009
@@ -38,6 +38,13 @@
     <pathelement location="${test.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
     <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
+    <pathelement location="${hadoop.test.jar}"/>
+    <pathelement location="${jetty.test.jar}"/>
+    <pathelement location="${servlet.test.jar}"/>
+    <pathelement location="${jasper.test.jar}"/>
+    <pathelement location="${jasperc.test.jar}"/>
+    <pathelement location="${jsp.test.jar}"/>
+    <pathelement location="${common.jar}"/>
     <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
     <fileset dir="${hadoop.root}/lib" includes="*.jar"/>
     <path refid="classpath"/>
@@ -68,6 +75,7 @@
               templatePath="${ql.test.template.dir}" template="TestCliDriver.vm" 
               queryDirectory="${ql.test.query.clientpositive.dir}" 
               queryFile="${qfile}"
+              clusterMode="${clustermode}"
               resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver"
               logFile="${test.log.dir}/testclidrivergen.log"
               logDirectory="${test.log.dir}/clientpositive"/>

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Wed May 27
18:57:45 2009
@@ -273,13 +273,19 @@
    * @param newPaths
    *          Array of classpath elements
    */
-  private static void addToClassPath(String[] newPaths) throws Exception {
+  private static void addToClassPath(String[] newPaths, boolean local) throws Exception {
     Thread curThread = Thread.currentThread();
     URLClassLoader loader = (URLClassLoader) curThread.getContextClassLoader();
     List<URL> curPath = Arrays.asList(loader.getURLs());
     ArrayList<URL> newPath = new ArrayList<URL>();
 
     for (String onestr : newPaths) {
+      // special processing for hadoop-17. file:// needs to be removed
+      if (local) {
+        if (StringUtils.indexOf(onestr, "file://") == 0)
+          onestr = StringUtils.substring(onestr, 7);
+      }
+
       URL oneurl = (new File(onestr)).toURL();
       if (!curPath.contains(oneurl)) {
         newPath.add(oneurl);
@@ -541,7 +547,7 @@
       String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
       if (StringUtils.isNotBlank(auxJars)) {
         try {
-          addToClassPath(StringUtils.split(auxJars, ","));
+          addToClassPath(StringUtils.split(auxJars, ","), true);
         } catch (Exception e) {
           throw new HiveException(e.getMessage(), e);
         }

Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed May 27 18:57:45
2009
@@ -34,10 +34,13 @@
 import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.lang.reflect.Method;
+import java.lang.reflect.Constructor;
 
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.cli.CliDriver;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -59,6 +62,7 @@
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.MiniMRCluster;
 
 import com.facebook.thrift.protocol.TBinaryProtocol;
 
@@ -81,6 +85,10 @@
   private FileSystem fs;
   private boolean overWrite;
   private CliDriver cliDriver;
+  private MiniMRCluster mr = null;
+  private Object dfs = null;
+  private boolean miniMr = false;
+  private Class<?> dfsClass = null;
   
   public boolean deleteDirectory(File path) {
     if (path.exists()) {
@@ -155,9 +163,50 @@
   }
 
   public QTestUtil(String outDir, String logDir) throws Exception {
+    this(outDir, logDir, false);
+  }
+
+  public QTestUtil(String outDir, String logDir, boolean miniMr) throws Exception {
     this.outDir = outDir;
     this.logDir = logDir;
     conf = new HiveConf(Driver.class);
+    this.miniMr = miniMr;
+    qMap = new TreeMap<String, String>();
+
+    if (miniMr) {
+      dfsClass = null;
+
+      // The path for MiniDFSCluster has changed, so look in both 17 and 19
+      // In hadoop 17, the path is org.apache.hadoop.dfs.MiniDFSCluster, whereas
+      // it is org.apache.hadoop.hdfs.MiniDFSCluster in hadoop 19. Due to this anamonly,
+      // use reflection to invoke the methods.
+      try {
+        dfsClass = Class.forName("org.apache.hadoop.dfs.MiniDFSCluster");
+      } catch (ClassNotFoundException e) {
+        dfsClass = null;
+      }
+
+      if (dfsClass == null) {
+        dfsClass = Class.forName("org.apache.hadoop.hdfs.MiniDFSCluster");
+      }
+
+      Constructor<?> dfsCons = 
+        dfsClass.getDeclaredConstructor(new Class<?>[] {Configuration.class, Integer.TYPE,

+                                            Boolean.TYPE, (new String[] {}).getClass()});
+
+      dfs = dfsCons.newInstance(conf, 4, true, null);
+      Method m = dfsClass.getDeclaredMethod("getFileSystem", new Class[]{});
+      FileSystem fs = (FileSystem)m.invoke(dfs, new Object[] {});
+
+      mr = new MiniMRCluster(4, fs.getUri().toString(), 1);
+      
+      // hive.metastore.warehouse.dir needs to be set relative to the jobtracker
+      String fsName = conf.get("fs.default.name");
+      assert fsName != null;
+      conf.set("hive.metastore.warehouse.dir", fsName.concat("/build/ql/test/data/warehouse/"));
+      
+      conf.set("mapred.job.tracker", "localhost:" + mr.getJobTrackerPort());
+    }    
 
     // System.out.println(conf.toString());
     testFiles = conf.get("test.data.files").replace('\\', '/').replace("c:", "");
@@ -168,12 +217,25 @@
       overWrite = true;
     }
 
-    qMap = new TreeMap<String, String>();
     srcTables = new LinkedList<String>();
     init();
   }
   
+  public void shutdown() throws Exception {
+    cleanUp();
 
+    if (dfs != null) {
+      Method m = dfsClass.getDeclaredMethod("shutdown", new Class[]{});
+      m.invoke(dfs, new Object[]{});
+      dfs = null;
+      dfsClass = null;
+    }
+    
+    if (mr != null) {
+      mr.shutdown();
+      mr = null;
+    }
+  }
 
   public void addFile(String qFile) throws Exception {
 
@@ -367,7 +429,7 @@
   }
 
   public void cliInit(String tname, boolean recreate) throws Exception {
-    if(recreate) {
+    if (miniMr || recreate) {
       cleanUp();
       createSources();
     }

Modified: hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm?rev=779275&r1=779274&r2=779275&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm (original)
+++ hadoop/hive/trunk/ql/src/test/templates/TestCliDriver.vm Wed May 27 18:57:45 2009
@@ -32,7 +32,11 @@
   @Override
   protected void setUp() {
     try {
-      qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()");
+      boolean miniMR = false;
+      if ("$clusterMode".equals("miniMR"))
+        miniMR = true;
+
+      qt = new QTestUtil("$resultsDir.getCanonicalPath()", "$logDir.getCanonicalPath()",
miniMR);
 
 #foreach ($qf in $qfiles)
       qt.addFile("$qf.getCanonicalPath()");
@@ -46,6 +50,19 @@
     }
   }
 
+  @Override
+  protected void tearDown() {
+    try {
+      qt.shutdown();
+    }
+    catch (Exception e) {
+      System.out.println("Exception: " + e.getMessage());
+      e.printStackTrace();
+      System.out.flush();
+      fail("Unexpected exception in tearDown");
+    }
+  }
+
   public static Test suite() {
     TestSuite suite = new TestSuite();
 #foreach ($qf in $qfiles)



Mime
View raw message