hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1591527 - in /hive/trunk: ./ itests/util/src/main/java/org/apache/hadoop/hive/ql/ ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/org/apache/hadoop/hive/ql/exec/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Date Thu, 01 May 2014 01:08:11 GMT
Author: khorgath
Date: Thu May  1 01:08:11 2014
New Revision: 1591527

URL: http://svn.apache.org/r1591527
Log:
HIVE-6931 : Windows unit test fixes (Jason Dere via Sushanth Sowmyan)

Modified:
    hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    hive/trunk/pom.xml
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
    hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q
    hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
    hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out

Modified: hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/trunk/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Thu May
 1 01:08:11 2014
@@ -258,7 +258,7 @@ public class QTestUtil {
       mr.setupConfiguration(conf);
 
       // set fs.default.name to the uri of mini-dfs
-      String dfsUriString = getHdfsUriString(dfs.getFileSystem().getUri().toString());
+      String dfsUriString = WindowsPathUtil.getHdfsUriString(dfs.getFileSystem().getUri().toString());
       conf.setVar(HiveConf.ConfVars.HADOOPFS, dfsUriString);
       // hive.metastore.warehouse.dir needs to be set relative to the mini-dfs
       conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE,
@@ -269,46 +269,10 @@ public class QTestUtil {
     // Windows paths should be converted after MiniMrShim.setupConfiguration()
     // since setupConfiguration may overwrite configuration values.
     if (Shell.WINDOWS) {
-      convertPathsFromWindowsToHdfs();
+      WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
     }
   }
 
-  private void convertPathsFromWindowsToHdfs() {
-    // Following local paths are used as HDFS paths in unit tests.
-    // It works well in Unix as the path notation in Unix and HDFS is more or less same.
-    // But when it comes to Windows, drive letter separator ':' & backslash '\" are invalid
-    // characters in HDFS so we need to converts these local paths to HDFS paths before using
them
-    // in unit tests.
-
-    // hive.exec.scratchdir needs to be set relative to the mini-dfs
-    String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
-    conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, getHdfsUriString(orgWarehouseDir));
-
-    String orgTestTempDir = System.getProperty("test.tmp.dir");
-    System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir));
-
-    String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);
-    conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));
-
-    if (miniMr) {
-      String orgAuxJarFolder = conf.getAuxJars();
-      conf.setAuxJars(getHdfsUriString("file://" + orgAuxJarFolder));
-    }
-  }
-
-  private String getHdfsUriString(String uriStr) {
-    assert uriStr != null;
-    if(Shell.WINDOWS) {
-      // If the URI conversion is from Windows to HDFS then replace the '\' with '/'
-      // and remove the windows single drive letter & colon from absolute path.
-      return uriStr.replace('\\', '/')
-        .replaceFirst("/[c-zC-Z]:", "/")
-        .replaceFirst("^[c-zC-Z]:", "");
-    }
-
-    return uriStr;
-  }
-
   public enum MiniClusterType {
     mr,
     tez,
@@ -361,7 +325,7 @@ public class QTestUtil {
     if (clusterType != MiniClusterType.none) {
       dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
       FileSystem fs = dfs.getFileSystem();
-      String uriString = getHdfsUriString(fs.getUri().toString());
+      String uriString = WindowsPathUtil.getHdfsUriString(fs.getUri().toString());
       if (clusterType == MiniClusterType.tez) {
         mr = shims.getMiniTezCluster(conf, 4, uriString, 1);
       } else {

Modified: hive/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Thu May  1 01:08:11 2014
@@ -64,6 +64,7 @@
     <test.hive.hadoop.classpath>${maven.test.classpath}</test.hive.hadoop.classpath>
     <test.log4j.scheme>file://</test.log4j.scheme>
     <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir>
+    <test.tmp.dir.uri>file://${test.tmp.dir}</test.tmp.dir.uri>
     <test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir>
     <test.warehouse.scheme>pfile://</test.warehouse.scheme>
 
@@ -757,6 +758,7 @@
             <test.data.files>${basedir}/${hive.path.to.root}/data/files</test.data.files>
             <test.data.dir>${basedir}/${hive.path.to.root}/data/files</test.data.dir>
             <test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
+            <test.tmp.dir.uri>${test.tmp.dir.uri}</test.tmp.dir.uri>
             <test.dfs.mkdir>${test.dfs.mkdir}</test.dfs.mkdir>
             <test.output.overwrite>${test.output.overwrite}</test.output.overwrite>
             <test.warehouse.dir>${test.warehouse.scheme}${test.warehouse.dir}</test.warehouse.dir>
@@ -1047,6 +1049,7 @@
         <hadoop.bin.path>${basedir}/${hive.path.to.root}/testutils/hadoop.cmd</hadoop.bin.path>
         <test.extra.path>;${env.HADOOP_HOME}/bin</test.extra.path>
         <test.hive.hadoop.classpath>${project.build.directory}/deplibs/*</test.hive.hadoop.classpath>
+        <test.tmp.dir.uri>file:///${test.tmp.dir}</test.tmp.dir.uri>
         <test.log4j.scheme>file:/</test.log4j.scheme>
       </properties>
     </profile>

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/WindowsPathUtil.java Thu May  1 01:08:11
2014
@@ -24,17 +24,26 @@ import org.apache.hadoop.util.Shell;
 public class WindowsPathUtil {
 
   public static void convertPathsFromWindowsToHdfs(HiveConf conf){
+    // Following local paths are used as HDFS paths in unit tests.
+    // It works well in Unix as the path notation in Unix and HDFS is more or less same.
+    // But when it comes to Windows, drive letter separator ':' & backslash '\" are invalid
+    // characters in HDFS so we need to converts these local paths to HDFS paths before using
them
+    // in unit tests.
+
     String orgWarehouseDir = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
     conf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, getHdfsUriString(orgWarehouseDir));
 
     String orgTestTempDir = System.getProperty("test.tmp.dir");
     System.setProperty("test.tmp.dir", getHdfsUriString(orgTestTempDir));
 
+    String orgTestWarehouseDir = System.getProperty("test.warehouse.dir");
+    System.setProperty("test.warehouse.dir", getHdfsUriString(orgTestWarehouseDir));
+
     String orgScratchDir = conf.getVar(HiveConf.ConfVars.SCRATCHDIR);
     conf.setVar(HiveConf.ConfVars.SCRATCHDIR, getHdfsUriString(orgScratchDir));
   }
 
-  private static String getHdfsUriString(String uriStr) {
+  public static String getHdfsUriString(String uriStr) {
     assert uriStr != null;
     if(Shell.WINDOWS) {
       // If the URI conversion is from Windows to HDFS then replace the '\' with '/'

Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java Thu May  1 01:08:11
2014
@@ -71,9 +71,9 @@ public class TestExecDriver extends Test
 
   static HiveConf conf;
 
-  private static final String tmpdir = System.getProperty("test.tmp.dir");
+  private static final String tmpdir;
   private static final Log LOG = LogFactory.getLog(TestExecDriver.class);
-  private static final Path tmppath = new Path(tmpdir);
+  private static final Path tmppath;
   private static Hive db;
   private static FileSystem fs;
 
@@ -86,6 +86,8 @@ public class TestExecDriver extends Test
       if (Shell.WINDOWS) {
         WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
       }
+      tmpdir = System.getProperty("test.tmp.dir");
+      tmppath = new Path(tmpdir);
 
       fs = FileSystem.get(conf);
       if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) {

Modified: hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/scriptfile1_win.q Thu May  1 01:08:11 2014
@@ -3,7 +3,7 @@ set hive.input.format=org.apache.hadoop.
 
 CREATE TABLE dest1(key INT, value STRING);
 
-ADD FILE src/test/scripts/testgrep_win.bat;
+ADD FILE ../../ql/src/test/scripts/testgrep_win.bat;
 
 FROM (
   FROM src

Modified: hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
(original)
+++ hive/trunk/ql/src/test/queries/clientpositive/tez_insert_overwrite_local_directory_1.q
Thu May  1 01:08:11 2014
@@ -1,5 +1,5 @@
 insert overwrite local directory '${system:test.tmp.dir}/tez_local_src_table_1'
 select * from src order by key limit 10 ;
-dfs -cat file:${system:test.tmp.dir}/tez_local_src_table_1/000000_0 ;
+dfs -cat ${system:test.tmp.dir.uri}/tez_local_src_table_1/* ;
 
-dfs -rmr file:${system:test.tmp.dir}/tez_local_src_table_1/ ;
+dfs -rmr ${system:test.tmp.dir.uri}/tez_local_src_table_1/ ;

Modified: hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out?rev=1591527&r1=1591526&r2=1591527&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/scriptfile1_win.q.out Thu May  1 01:08:11
2014
@@ -2,10 +2,12 @@ PREHOOK: query: -- INCLUDE_OS_WINDOWS
 
 CREATE TABLE dest1(key INT, value STRING)
 PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
 POSTHOOK: query: -- INCLUDE_OS_WINDOWS
 
 CREATE TABLE dest1(key INT, value STRING)
 POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
 POSTHOOK: Output: default@dest1
 PREHOOK: query: FROM (
   FROM src



Mime
View raw message