hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject svn commit: r1616593 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/ metastore/src/java/org/apache/hadoop/hive/metastore/ ql/src/java/org/apache/hadoop/hive/ql/metadata/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpos...
Date Thu, 07 Aug 2014 20:57:11 GMT
Author: szehon
Date: Thu Aug  7 20:57:10 2014
New Revision: 1616593

URL: http://svn.apache.org/r1616593
Log:
HIVE-7441 : Custom partition scheme gets rewritten with hive scheme upon concatenate (Chaoyu
Tang via Szehon)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q
    hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1616593&r1=1616592&r2=1616593&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Thu Aug  7 20:57:10
2014
@@ -613,4 +613,19 @@ public final class FileUtils {
       return false;
     }
   }
+
+  /**
+   * @param fs1
+   * @param fs2
+   * @return return true if both file system arguments point to same file system
+   */
+  public static boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) {
+    //When file system cache is disabled, you get different FileSystem objects
+    // for same file system, so '==' can't be used in such cases
+    //FileSystem api doesn't have a .equals() function implemented, so using
+    //the uri for comparison. FileSystem already uses uri+Configuration for
+    //equality in its CACHE .
+    //Once equality has been added in HDFS-4321, we should make use of it
+    return fs1.getUri().equals(fs2.getUri());
+  }
 }

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java?rev=1616593&r1=1616592&r2=1616593&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java Thu
Aug  7 20:57:10 2014
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -156,7 +157,7 @@ public class HiveAlterHandler implements
         destPath = new Path(newTblLoc);
         destFs = wh.getFs(destPath);
         // check that src and dest are on the same file system
-        if (! equalsFileSystem(srcFs, destFs)) {
+        if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
           throw new InvalidOperationException("table new location " + destPath
               + " is on a different file system than the old location "
               + srcPath + ". This operation is not supported");
@@ -251,21 +252,6 @@ public class HiveAlterHandler implements
     }
   }
 
-  /**
-   * @param fs1
-   * @param fs2
-   * @return return true if both file system arguments point to same file system
-   */
-  private boolean equalsFileSystem(FileSystem fs1, FileSystem fs2) {
-    //When file system cache is disabled, you get different FileSystem objects
-    // for same file system, so '==' can't be used in such cases
-    //FileSystem api doesn't have a .equals() function implemented, so using
-    //the uri for comparison. FileSystem already uses uri+Configuration for
-    //equality in its CACHE .
-    //Once equality has been added in HDFS-4321, we should make use of it
-    return fs1.getUri().equals(fs2.getUri());
-  }
-
   public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
       final String name, final List<String> part_vals, final Partition new_part)
       throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1616593&r1=1616592&r2=1616593&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Thu Aug  7 20:57:10
2014
@@ -1242,7 +1242,7 @@ public class Hive {
            */
           FileSystem oldPartPathFS = oldPartPath.getFileSystem(getConf());
           FileSystem loadPathFS = loadPath.getFileSystem(getConf());
-          if (oldPartPathFS.equals(loadPathFS)) {
+          if (FileUtils.equalsFileSystem(oldPartPathFS,loadPathFS)) {
             newPartPath = oldPartPath;
           }
         }

Added: hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q?rev=1616593&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/alter_merge_3.q Thu Aug  7 20:57:10 2014
@@ -0,0 +1,23 @@
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+dfs -put ../../data/files/smbbucket_1.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+dfs -put ../../data/files/smbbucket_2.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+dfs -put ../../data/files/smbbucket_3.rc ${system:test.tmp.dir}/alter_merge_location/ds20140804;
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+dfs -put ../../data/files/smbbucket_1.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+dfs -put ../../data/files/smbbucket_2.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+dfs -put ../../data/files/smbbucket_3.rc ${system:test.tmp.dir}/alter_merge_location/ds20140805;
+
+create table src_rc_merge_test_part (key int, value string) partitioned by (ds string) stored
as rcfile;
+
+set fs.hdfs.impl.disable.cache=false;
+alter table src_rc_merge_test_part add partition (ds = '2014-08-04') location '${system:test.tmp.dir}/alter_merge_location/ds20140804';
+alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate;
+select count(1) from src_rc_merge_test_part where ds='2014-08-04';
+
+set fs.hdfs.impl.disable.cache=true;
+alter table src_rc_merge_test_part add partition (ds = '2014-08-05') location '${system:test.tmp.dir}/alter_merge_location/ds20140805';
+alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate;
+select count(1) from src_rc_merge_test_part where ds='2014-08-05';
+
+drop table src_rc_merge_test_part;
\ No newline at end of file

Added: hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out?rev=1616593&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_3.q.out Thu Aug  7 20:57:10
2014
@@ -0,0 +1,71 @@
+PREHOOK: query: create table src_rc_merge_test_part (key int, value string) partitioned by
(ds string) stored as rcfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table src_rc_merge_test_part (key int, value string) partitioned
by (ds string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_rc_merge_test_part
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_rc_merge_test_part
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04
+PREHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate
+PREHOOK: type: ALTER_PARTITION_MERGE
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04
+POSTHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-04') concatenate
+POSTHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-04
+PREHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-04'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-04
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-04'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-04
+#### A masked pattern was here ####
+15
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@src_rc_merge_test_part
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05
+PREHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate
+PREHOOK: type: ALTER_PARTITION_MERGE
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05
+POSTHOOK: query: alter table src_rc_merge_test_part partition (ds = '2014-08-05') concatenate
+POSTHOOK: type: ALTER_PARTITION_MERGE
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part@ds=2014-08-05
+PREHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-05'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-05
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from src_rc_merge_test_part where ds='2014-08-05'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Input: default@src_rc_merge_test_part@ds=2014-08-05
+#### A masked pattern was here ####
+15
+PREHOOK: query: drop table src_rc_merge_test_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_rc_merge_test_part
+PREHOOK: Output: default@src_rc_merge_test_part
+POSTHOOK: query: drop table src_rc_merge_test_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_rc_merge_test_part
+POSTHOOK: Output: default@src_rc_merge_test_part



Mime
View raw message