hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1549982 - /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Date Tue, 10 Dec 2013 21:57:02 GMT
Author: gunther
Date: Tue Dec 10 21:57:02 2013
New Revision: 1549982

URL: http://svn.apache.org/r1549982
Log:
HIVE-6001: Tez: UDFs are not properly localized (Gunther Hagleitner)

Modified:
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1549982&r1=1549981&r2=1549982&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Tue Dec
10 21:57:02 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.plan.Ba
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.ReduceWork;
 import org.apache.hadoop.hive.ql.plan.TezWork.EdgeType;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
 import org.apache.hadoop.hive.shims.Hadoop20Shims.NullOutputCommitter;
@@ -417,10 +418,20 @@ public class DagUtils {
   public static List<LocalResource> localizeTempFiles(Configuration conf) throws IOException,
LoginException {
     List<LocalResource> tmpResources = new ArrayList<LocalResource>();
 
+    String addedFiles = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
+    if (StringUtils.isNotBlank(addedFiles)) {
+      HiveConf.setVar(conf, ConfVars.HIVEADDEDFILES, addedFiles);
+    }
+    String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR);
+    if (StringUtils.isNotBlank(addedJars)) {
+      HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars);
+    }
+    String addedArchives = Utilities.getResourceFiles(conf, SessionState.ResourceType.ARCHIVE);
+    if (StringUtils.isNotBlank(addedArchives)) {
+      HiveConf.setVar(conf, ConfVars.HIVEADDEDARCHIVES, addedArchives);
+    }
+
     String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
-    String addedJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEADDEDJARS);
-    String addedFiles = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEADDEDFILES);
-    String addedArchives = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEADDEDARCHIVES);
 
     // need to localize the additional jars and files
 
@@ -447,6 +458,9 @@ public class DagUtils {
     String allFiles = auxJars + "," + addedJars + "," + addedFiles + "," + addedArchives;
     String[] allFilesArr = allFiles.split(",");
     for (String file : allFilesArr) {
+      if (!StringUtils.isNotBlank(file)) {
+        continue;
+      }
       String hdfsFilePathStr = hdfsDirPathStr + "/" + getResourceBaseName(file);
       LocalResource localResource = localizeResource(new Path(file),
           new Path(hdfsFilePathStr), conf);



Mime
View raw message