hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sunc...@apache.org
Subject hive git commit: HIVE-13278: Avoid FileNotFoundException when map/reduce.xml is not available (Chao Sun, reviewed by Rui Li)
Date Mon, 19 Dec 2016 23:45:09 GMT
Repository: hive
Updated Branches:
  refs/heads/master 268c60ebb -> eb26dffe9


HIVE-13278: Avoid FileNotFoundException when map/reduce.xml is not available (Chao Sun, reviewed
by Rui Li)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eb26dffe
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eb26dffe
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eb26dffe

Branch: refs/heads/master
Commit: eb26dffe9a1dd5ac661cd9930e6c2ed53fecf9f8
Parents: 268c60e
Author: Chao Sun <sunchao@apache.org>
Authored: Thu Dec 15 10:35:27 2016 -0800
Committer: Chao Sun <sunchao@apache.org>
Committed: Mon Dec 19 15:44:46 2016 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/Utilities.java | 18 ++++++++++++++++++
 .../hive/ql/io/orc/TestInputOutputFormat.java     |  1 +
 2 files changed, 19 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/eb26dffe/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 37e4b9b..122df9f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -215,6 +215,8 @@ public final class Utilities {
   public static String REDUCE_PLAN_NAME = "reduce.xml";
   public static String MERGE_PLAN_NAME = "merge.xml";
   public static final String INPUT_NAME = "iocontext.input.name";
+  public static final String HAS_MAP_WORK = "has.map.work";
+  public static final String HAS_REDUCE_WORK = "has.reduce.work";
   public static final String MAPRED_MAPPER_CLASS = "mapred.mapper.class";
   public static final String MAPRED_REDUCER_CLASS = "mapred.reducer.class";
   public static final String HIVE_ADDED_JARS = "hive.added.jars";
@@ -299,6 +301,9 @@ public final class Utilities {
   }
 
   public static MapWork getMapWork(Configuration conf) {
+    if (!conf.getBoolean(HAS_MAP_WORK, false)) {
+      return null;
+    }
     return (MapWork) getBaseWork(conf, MAP_PLAN_NAME);
   }
 
@@ -307,6 +312,9 @@ public final class Utilities {
   }
 
   public static ReduceWork getReduceWork(Configuration conf) {
+    if (!conf.getBoolean(HAS_REDUCE_WORK, false)) {
+      return null;
+    }
     return (ReduceWork) getBaseWork(conf, REDUCE_PLAN_NAME);
   }
 
@@ -359,6 +367,7 @@ public final class Utilities {
    */
   public static void setBaseWork(Configuration conf, String name, BaseWork work) {
     Path path = getPlanPath(conf, name);
+    setHasWork(conf, name);
     gWorkMap.get(conf).put(path, work);
   }
 
@@ -474,6 +483,14 @@ public final class Utilities {
     }
   }
 
+  private static void setHasWork(Configuration conf, String name) {
+    if (MAP_PLAN_NAME.equals(name)) {
+      conf.setBoolean(HAS_MAP_WORK, true);
+    } else if (REDUCE_PLAN_NAME.equals(name)) {
+      conf.setBoolean(HAS_REDUCE_WORK, true);
+    }
+  }
+
   public static void setWorkflowAdjacencies(Configuration conf, QueryPlan plan) {
     try {
       Graph stageGraph = plan.getQueryPlan().getStageGraph();
@@ -536,6 +553,7 @@ public final class Utilities {
       setPlanPath(conf, hiveScratchDir);
 
       Path planPath = getPlanPath(conf, name);
+      setHasWork(conf, name);
 
       OutputStream out = null;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/eb26dffe/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
index 197c1d2..aa23df8 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
@@ -2110,6 +2110,7 @@ public class TestInputOutputFormat {
     localFs.delete(mapXml, true);
     FSDataOutputStream planStream = localFs.create(mapXml);
     SerializationUtilities.serializePlan(mapWork, planStream);
+    conf.setBoolean(Utilities.HAS_MAP_WORK, true);
     planStream.close();
     return conf;
   }


Mime
View raw message