hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r989372 - in /hadoop/hive/trunk: CHANGES.txt ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java build-common.xml build.xml data/conf/hive-site.xml ql/build.xml ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
Date Wed, 25 Aug 2010 21:07:55 GMT
Author: namit
Date: Wed Aug 25 21:07:55 2010
New Revision: 989372

URL: http://svn.apache.org/viewvc?rev=989372&view=rev
Log:
HIVE-1523. Enable some tests in miniMR mode
(Joydeep Sen Sarma via namit)

M    ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
M    build.xml
M    data/conf/hive-site.xml
M    build-common.xml
M    CHANGES.txt
M    ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
M    ql/build.xml

Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
    hadoop/hive/trunk/build-common.xml
    hadoop/hive/trunk/build.xml
    hadoop/hive/trunk/data/conf/hive-site.xml
    hadoop/hive/trunk/ql/build.xml
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Wed Aug 25 21:07:55 2010
@@ -105,6 +105,9 @@ Trunk -  Unreleased
     HIVE-1441. Extend ivy offline mode to cover metastore downloads
     (John Sichi via Ning Zhang)
 
+    HIVE-1523. Enable some tests in miniMR mode
+    (Joydeep Sen Sarma via namit)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java (original)
+++ hadoop/hive/trunk/ant/src/org/apache/hadoop/hive/ant/QTestGenTask.java Wed Aug 25 21:07:55
2010
@@ -81,6 +81,8 @@ public class QTestGenTask extends Task {
   private String queryDirectory;
  
   private String queryFile;
+
+  private String excludeQueryFile;
   
   private String queryFileRegex;
 
@@ -193,11 +195,19 @@ public class QTestGenTask extends Task {
   public String getQueryFile() {
     return queryFile;
   }
-  
+
+  public void setExcludeQueryFile(String excludeQueryFile) {
+    this.excludeQueryFile = excludeQueryFile;
+  }
+
+  public String getExcludeQueryFile() {
+    return excludeQueryFile;
+  }
+
   public void setQueryFileRegex(String queryFileRegex) {
     this.queryFileRegex = queryFileRegex;
   }
-  
+
   public String getQueryFileRegex() {
     return queryFileRegex;
   }
@@ -257,6 +267,17 @@ public class QTestGenTask extends Task {
       } else {
         qFiles.addAll(Arrays.asList(inpDir.listFiles(new QFileFilter())));
       }
+
+      if (excludeQueryFile != null && !excludeQueryFile.equals("")) {
+        // Exclude specified query files, comma separated
+        for (String qFile : excludeQueryFile.split(",")) {
+          if (null != inpDir) {
+            qFiles.remove(new File(inpDir, qFile));
+          } else {
+            qFiles.remove(new File(qFile));
+          }
+        }
+      }
       
       Collections.sort(qFiles);
 

Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Wed Aug 25 21:07:55 2010
@@ -50,7 +50,7 @@
   <property name="test.timeout" value="13200000"/>
   <property name="test.junit.output.format" value="xml"/>
   <property name="test.junit.output.usefile" value="true"/>
-  <property name="minimr.query.files" value="join1.q,groupby1.q"/>
+  <property name="minimr.query.files" value="input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q"/>
   <property name="test.silent" value="true"/>
   <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
   <property name="test.serialize.qplan" value="false"/>
@@ -330,16 +330,6 @@
 
   <target name="test-conditions">
 
-    <condition property="qfile" value="${minimr.query.files}">
-      <and>
-        <not>
-          <isset property="qfile"/>
-        </not>
-
-        <equals arg1="${clustermode}" arg2="miniMR" />
-      </and>
-    </condition>
-
     <condition property="qfile" value="">
       <not>
         <isset property="qfile"/>
@@ -399,6 +389,9 @@
     </not>
   </condition>
 
+  <property name="test.warehouse.dir" value="${test.warehouse.scheme}${build.dir}/test/data/warehouse"/>
+  <property name="mapred.job.tracker" value="local"/>
+  <property name="fs.default.name" value="file:///"/>
 
   <!-- target to run the tests -->
   <target name="test"
@@ -425,7 +418,9 @@
       <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
       <sysproperty key="test.silent" value="${test.silent}"/>
       <sysproperty key="test.tmp.dir" value="${build.dir}/tmp"/>
-      <sysproperty key="test.warehouse.dir" value="${test.warehouse.scheme}${build.dir}/test/data/warehouse"/>
+      <sysproperty key="test.warehouse.dir" value="${test.warehouse.dir}"/>
+      <sysproperty key="mapred.job.tracker" value="${mapred.job.tracker}"/>
+      <sysproperty key="fs.default.name" value="${fs.default.name}"/>
       <sysproperty key="build.dir" value="${build.dir}"/>
       <sysproperty key="build.dir.hive" value="${build.dir.hive}"/>
 

Modified: hadoop/hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/build.xml (original)
+++ hadoop/hive/trunk/build.xml Wed Aug 25 21:07:55 2010
@@ -342,16 +342,16 @@
     <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}"/>
 
     <condition property="jetty.jar" value="jetty-6.1.14.jar" else="jetty-5.1.4.jar">
-      <equals arg1="${hadoop.version.ant-internal.prefix}" arg2="0.20" />
+      <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
     </condition>
 
     <condition property="jetty.util.jar" value="jetty-util-6.1.14.jar" else="jetty-ext/jsp-api.jar">
       <!-- hadoop 0.19 and earlier does not have jetty-util.jar, so we just load another
jar -->
-      <equals arg1="${hadoop.version.ant-internal.prefix}" arg2="0.20" />
+      <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
     </condition>
 
     <condition property="servlet-api.jar" value="servlet-api-2.5-6.1.14.jar" else="servlet-api.jar">
-      <equals arg1="${hadoop.version.ant-internal.prefix}" arg2="0.20" />
+      <matches string="${hadoop.version.ant-internal}" pattern="^0\.20\..*" />
     </condition>
 
     <pathconvert property="eclipse.project">

Modified: hadoop/hive/trunk/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/conf/hive-site.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/data/conf/hive-site.xml (original)
+++ hadoop/hive/trunk/data/conf/hive-site.xml Wed Aug 25 21:07:55 2010
@@ -73,18 +73,6 @@
 </property>
 
 <property>
-  <name>fs.default.name</name>
-  <value>file:///</value>
-  <description></description>
-</property>
-
-<property>
-  <name>mapred.job.tracker</name>
-  <value>local</value>
-  <description></description>
-</property>
-
-<property>
   <name>test.log.dir</name>
   <value>${build.dir}/test/logs</value>
   <description></description>

Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Wed Aug 25 21:07:55 2010
@@ -31,6 +31,7 @@
   <property name="thrift.args" value="--gen java:beans --gen php --gen py -I ${basedir}/include
-I ${basedir}/.. -o ${basedir}/src"/>
 
   <import file="../build-common.xml"/>
+  <property name="ql.test.clientpositive.exclude" value="${minimr.query.files}"/>
 
   <path id="test.classpath">
     <pathelement location="${test.build.classes}" />
@@ -48,6 +49,7 @@
     <pathelement location="${common.jar}"/>
     <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
     <fileset dir="${hadoop.root}/lib" includes="*.jar"/>
+    <fileset dir="${hadoop.root}/lib/jsp-2.1" includes="*.jar"/>
     <path refid="classpath"/>
   </path>
 
@@ -86,6 +88,7 @@
               templatePath="${ql.test.template.dir}" template="TestCliDriver.vm" 
               queryDirectory="${ql.test.query.clientpositive.dir}" 
               queryFile="${qfile}"
+              excludeQueryFile="${ql.test.clientpositive.exclude}"
               queryFileRegex="${qfile_regex}"
               clusterMode="${clustermode}"
               resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestCliDriver"
@@ -95,6 +98,18 @@
     />
 
     <qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli" 
+              templatePath="${ql.test.template.dir}" template="TestCliDriver.vm" 
+              queryDirectory="${ql.test.query.clientpositive.dir}" 
+              queryFile="${minimr.query.files}"
+              queryFileRegex="${qfile_regex}"
+              clusterMode="miniMR"
+              resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestMinimrCliDriver"
+              logFile="${test.log.dir}/testminimrclidrivergen.log"
+              logDirectory="${test.log.dir}/clientpositive"
+              hadoopVersion="${hadoopVersion}"
+    />
+
+    <qtestgen outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli" 
               templatePath="${ql.test.template.dir}" template="TestNegativeCliDriver.vm"

               queryDirectory="${ql.test.query.dir}/clientnegative" 
               queryFile="${qfile}"

Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=989372&r1=989371&r2=989372&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Wed Aug 25 21:07:55
2010
@@ -194,14 +194,17 @@ public class QTestUtil {
     return null;
   }
 
-  public void initConf() {
+  public void initConf() throws Exception {
     if (miniMr) {
-      String fsName = conf.get("fs.default.name");
-      assert fsName != null;
-      // hive.metastore.warehouse.dir needs to be set relative to the jobtracker
-      conf.set("hive.metastore.warehouse.dir", fsName
-               .concat("/build/ql/test/data/warehouse/"));
-      conf.set("mapred.job.tracker", "localhost:" + mr.getJobTrackerPort());
+      assert dfs != null;
+      assert mr != null;
+      // set fs.default.name to the uri of mini-dfs
+      conf.setVar(HiveConf.ConfVars.HADOOPFS, dfs.getFileSystem().getUri().toString());
+      // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs
+      conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, 
+                  (new Path(dfs.getFileSystem().getUri().toString(),
+                            "/build/ql/test/data/warehouse/")).toString());
+      conf.setVar(HiveConf.ConfVars.HADOOPJT, "localhost:" + mr.getJobTrackerPort());
     }
   }
 
@@ -330,7 +333,6 @@ public class QTestUtil {
   }
 
   public void cleanUp() throws Exception {
-    String warehousePath = ((new URI(testWarehouse)).getPath());
     // Drop any tables that remain due to unsuccessful runs
     for (String s : new String[] {"src", "src1", "src_json", "src_thrift",
         "src_sequencefile", "srcpart", "srcbucket", "srcbucket2", "dest1",
@@ -338,9 +340,15 @@ public class QTestUtil {
         "dest_g1", "dest_g2", "fetchtask_ioexception"}) {
       db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, s);
     }
-    for (String s : new String[] {"dest4.out", "union.out"}) {
-      deleteDirectory(new File(warehousePath, s));
+
+    // delete any contents in the warehouse dir
+    Path p = new Path(testWarehouse);
+    FileSystem fs = p.getFileSystem(conf);
+    FileStatus [] ls = fs.listStatus(p);
+    for (int i=0; (ls != null) && (i<ls.length); i++) {
+      fs.delete(ls[i].getPath(), true);
     }
+
     FunctionRegistry.unregisterTemporaryUDF("test_udaf");
     FunctionRegistry.unregisterTemporaryUDF("test_error");
     setup.tearDown();
@@ -516,7 +524,7 @@ public class QTestUtil {
   }
 
   public void cliInit(String tname, boolean recreate) throws Exception {
-    if (miniMr || recreate) {
+    if (recreate) {
       cleanUp();
       createSources();
     }
@@ -538,6 +546,7 @@ public class QTestUtil {
       oldSs.out.close();
     }
     SessionState.start(ss);
+
     cliDriver = new CliDriver();
     if (tname.equals("init_file.q")) {
       ss.initFiles.add("../data/scripts/test_init_file.sql");
@@ -859,6 +868,7 @@ public class QTestUtil {
         "diff", "-a",
         "-I", "file:",
         "-I", "pfile:",
+        "-I", "hdfs:",
         "-I", "/tmp/",
         "-I", "invalidscheme:",
         "-I", "lastUpdateTime",



Mime
View raw message