hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-11820 : export tables with size of >32MB throws java.lang.IllegalArgumentException: Skip CRC is valid only with update options (Takahiko Saito via Ashutosh Chauhan)
Date Sun, 20 Sep 2015 06:51:20 GMT
Repository: hive
Updated Branches:
  refs/heads/master cc78dd5d8 -> 93a662760


HIVE-11820 : export tables with size of >32MB throws java.lang.IllegalArgumentException:
Skip CRC is valid only with update options (Takahiko Saito via Ashutosh Chauhan)

Signed-off-by: Ashutosh Chauhan <hashutosh@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/93a66276
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/93a66276
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/93a66276

Branch: refs/heads/master
Commit: 93a6627606cb2dad0e04c3a885f71c1be405f51d
Parents: cc78dd5
Author: Takahiko Saito <tysaito@gmail.com>
Authored: Wed Sep 16 15:34:00 2015 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Sat Sep 19 23:50:49 2015 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java  | 5 ++++-
 .../main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java   | 4 +++-
 2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/93a66276/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
----------------------------------------------------------------------
diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
index 6c6ccbc..93dcbd3 100644
--- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
+++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
@@ -73,6 +73,8 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.tools.distcp2.DistCp;
 import org.apache.hadoop.tools.distcp2.DistCpOptions;
+import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute;
+
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.VersionInfo;
@@ -672,8 +674,9 @@ public class Hadoop20SShims extends HadoopShimsSecure {
   public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
 
     DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
-    options.setSkipCRC(true);
     options.setSyncFolder(true);
+    options.setSkipCRC(true);
+    options.preserve(FileAttribute.BLOCKSIZE);
     try {
       DistCp distcp = new DistCp(conf, options);
       distcp.execute();

http://git-wip-us.apache.org/repos/asf/hive/blob/93a66276/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 3292cb3..83369ee 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -97,6 +97,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.tools.DistCp;
 import org.apache.hadoop.tools.DistCpOptions;
+import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.tez.test.MiniTezCluster;
@@ -1213,8 +1214,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
   public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
 
     DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
-    options.setSkipCRC(true);
     options.setSyncFolder(true);
+    options.setSkipCRC(true);
+    options.preserve(FileAttribute.BLOCKSIZE);
     try {
       DistCp distcp = new DistCp(conf, options);
       distcp.execute();


Mime
View raw message