hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-17144: export of temporary tables not working and it seems to be using distcp rather than filesystem copy (Anishek Agarwal, reviewed by Daniel Dai)
Date Thu, 03 Aug 2017 07:32:38 GMT
Repository: hive
Updated Branches:
  refs/heads/master 8ff37ad87 -> 68b2f9b2a


HIVE-17144: export of temporary tables not working and it seems to be using distcp rather
than filesystem copy (Anishek Agarwal, reviewed by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/68b2f9b2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/68b2f9b2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/68b2f9b2

Branch: refs/heads/master
Commit: 68b2f9b2a78d9727f034ece8b3d7adc1c29054ad
Parents: 8ff37ad
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Thu Aug 3 00:32:29 2017 -0700
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Thu Aug 3 00:32:29 2017 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/parse/WarehouseInstance.java    | 17 +++++++++++------
 .../hive/ql/parse/repl/dump/TableExport.java       |  2 +-
 .../hive/ql/parse/repl/dump/io/FileOperations.java |  4 +---
 3 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/68b2f9b2/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index 1128eae..8dfab08 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -64,7 +64,7 @@ class WarehouseInstance implements Closeable {
 
   private final static String LISTENER_CLASS = DbNotificationListener.class.getCanonicalName();
 
-  WarehouseInstance(Logger logger, MiniDFSCluster cluster) throws Exception {
+  WarehouseInstance(Logger logger, MiniDFSCluster cluster, boolean hiveInTests) throws Exception
{
     this.logger = logger;
     this.miniDFSCluster = cluster;
     assert miniDFSCluster.isClusterUp();
@@ -73,10 +73,14 @@ class WarehouseInstance implements Closeable {
 
     Path cmRootPath = mkDir(fs, "/cmroot" + uniqueIdentifier);
     this.functionsRoot = mkDir(fs, "/functions" + uniqueIdentifier).toString();
-    initialize(cmRootPath.toString());
+    initialize(cmRootPath.toString(), hiveInTests);
+  }
+
+  WarehouseInstance(Logger logger, MiniDFSCluster cluster) throws Exception {
+    this(logger, cluster, true);
   }
 
-  private void initialize(String cmRoot) throws Exception {
+  private void initialize(String cmRoot, boolean hiveInTest) throws Exception {
     hiveConf = new HiveConf(miniDFSCluster.getConfiguration(0), TestReplicationScenarios.class);
     String metaStoreUri = System.getProperty("test." + HiveConf.ConfVars.METASTOREURIS.varname);
     String hiveWarehouseLocation = System.getProperty("test.warehouse.dir", "/tmp")
@@ -89,6 +93,7 @@ class WarehouseInstance implements Closeable {
       return;
     }
 
+    hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, hiveInTest);
     // turn on db notification listener on meta store
     hiveConf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, LISTENER_CLASS);
     hiveConf.setBoolVar(HiveConf.ConfVars.REPLCMENABLED, true);
@@ -177,8 +182,7 @@ class WarehouseInstance implements Closeable {
   }
 
   WarehouseInstance verify(String data) throws IOException {
-    verifyResults(data == null ? new String[] {} : new String[] { data });
-    return this;
+    return verifyResults(data == null ? new String[] {} : new String[] { data });
   }
 
   /**
@@ -188,7 +192,7 @@ class WarehouseInstance implements Closeable {
    * Unless for Null Values it actually returns in UpperCase and hence explicitly lowering
case
    * before assert.
    */
-  private void verifyResults(String[] data) throws IOException {
+  WarehouseInstance verifyResults(String[] data) throws IOException {
     List<String> results = getOutput();
     logger.info("Expecting {}", StringUtils.join(data, ","));
     logger.info("Got {}", results);
@@ -196,6 +200,7 @@ class WarehouseInstance implements Closeable {
     for (int i = 0; i < data.length; i++) {
       assertEquals(data[i].toLowerCase(), results.get(i).toLowerCase());
     }
+    return this;
   }
 
   List<String> getOutput() throws IOException {

http://git-wip-us.apache.org/repos/asf/hive/blob/68b2f9b2/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 9f22f23..4d23efc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -57,7 +57,7 @@ public class TableExport {
       throws SemanticException {
     this.tableSpec = (tableSpec != null
         && tableSpec.tableHandle.isTemporary()
-        && !replicationSpec.isInReplicationScope())
+        && replicationSpec.isInReplicationScope())
         ? null
         : tableSpec;
     this.replicationSpec = replicationSpec;

http://git-wip-us.apache.org/repos/asf/hive/blob/68b2f9b2/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
index 164ca74..e1e3ae1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.ReplChangeManager;
-import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
 import org.apache.hadoop.hive.ql.parse.EximUtil;
 import org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer;
 import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
@@ -70,8 +69,7 @@ public class FileOperations {
     for (FileStatus fileStatus : fileStatuses) {
       srcPaths.add(fileStatus.getPath());
     }
-
-    ReplCopyTask.doCopy(exportRootDataDir, exportFileSystem, srcPaths, dataFileSystem, hiveConf);
+    new CopyUtils(hiveConf).doCopy(exportRootDataDir, srcPaths);
   }
 
   /**


Mime
View raw message