hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From heyongqi...@apache.org
Subject svn commit: r921994 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/test/org/apache/hadoop/hive/ql/ ql/src/test/org/apache/hadoop/hive/ql/udf/ ql/src/test/queries/clientnegative/ ql/src/test/results/clientnegative/
Date Thu, 11 Mar 2010 19:31:10 GMT
Author: heyongqiang
Date: Thu Mar 11 19:31:09 2010
New Revision: 921994

URL: http://svn.apache.org/viewvc?rev=921994&view=rev
Log:
HIVE-1216. Show the row with error in mapper/reducer. (Zheng via He Yongqiang)

Added:
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error.q
    hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error_reduce.q
    hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error.q.out
    hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error_reduce.q.out
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=921994&r1=921993&r2=921994&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Mar 11 19:31:09 2010
@@ -127,6 +127,9 @@ Trunk -  Unreleased
     HIVE-1212. Explicitly say "Hive Internal Error" to ease debugging
     (Zheng Shao via Ning Zhang)
 
+    HIVE-1216. Show the row with error in mapper/reducer
+    (Zheng Shao via He Yongqiang)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java?rev=921994&r1=921993&r2=921994&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecMapper.java Thu Mar 11
19:31:09 2010
@@ -46,6 +46,7 @@ import org.apache.hadoop.mapred.Mapper;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * ExecMapper.
@@ -198,12 +199,12 @@ public class ExecMapper extends MapReduc
       }
     } catch (Throwable e) {
       abort = true;
-      e.printStackTrace();
       if (e instanceof OutOfMemoryError) {
         // Don't create a new object if we are already out of memory
         throw (OutOfMemoryError) e;
       } else {
-        throw new RuntimeException(e.getMessage(), e);
+        l4j.fatal(StringUtils.stringifyException(e));
+        throw new RuntimeException(e);
       }
     }
   }
@@ -277,7 +278,7 @@ public class ExecMapper extends MapReduc
           // Don't create a new object if we are already out of memory
           throw (OutOfMemoryError) e;
         } else {
-          throw new RuntimeException("Map local work failed", e);
+          throw new RuntimeException("Hive Runtime Error: Map local work failed", e);
         }
       }
     }
@@ -342,7 +343,7 @@ public class ExecMapper extends MapReduc
       if (!abort) {
         // signal new failure to map-reduce
         l4j.error("Hit error while closing operators - failing tree");
-        throw new RuntimeException("Error while closing operators", e);
+        throw new RuntimeException("Hive Runtime Error while closing operators", e);
       }
     }
   }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java?rev=921994&r1=921993&r2=921994&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecReducer.java Thu Mar
11 19:31:09 2010
@@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.plan.Ta
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.SerDe;
 import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -46,6 +47,7 @@ import org.apache.hadoop.mapred.OutputCo
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * ExecReducer.
@@ -84,9 +86,11 @@ public class ExecReducer extends MapRedu
   TableDesc keyTableDesc;
   TableDesc[] valueTableDesc;
 
+  ObjectInspector[] rowObjectInspector;
+  
   @Override
   public void configure(JobConf job) {
-    ObjectInspector[] rowObjectInspector = new ObjectInspector[Byte.MAX_VALUE];
+    rowObjectInspector = new ObjectInspector[Byte.MAX_VALUE];
     ObjectInspector[] valueObjectInspector = new ObjectInspector[Byte.MAX_VALUE];
     ObjectInspector keyObjectInspector;
 
@@ -195,7 +199,7 @@ public class ExecReducer extends MapRedu
           keyObject = inputKeyDeserializer.deserialize(keyWritable);
         } catch (Exception e) {
           throw new HiveException(
-              "Unable to deserialize reduce input key from "
+              "Hive Runtime Error: Unable to deserialize reduce input key from "
               + Utilities.formatBinaryString(keyWritable.get(), 0,
               keyWritable.getSize()) + " with properties "
               + keyTableDesc.getProperties(), e);
@@ -215,7 +219,7 @@ public class ExecReducer extends MapRedu
               .deserialize(valueWritable);
         } catch (SerDeException e) {
           throw new HiveException(
-              "Unable to deserialize reduce input value (tag="
+              "Hive Runtime Error: Unable to deserialize reduce input value (tag="
               + tag.get()
               + ") from "
               + Utilities.formatBinaryString(valueWritable.get(), 0,
@@ -236,7 +240,19 @@ public class ExecReducer extends MapRedu
             nextCntr = getNextCntr(cntr);
           }
         }
-        reducer.process(row, tag.get());
+        try {
+          reducer.process(row, tag.get());
+        } catch (Exception e) {
+          String rowString = null;
+          try {
+            rowString = SerDeUtils.getJSONString(row, rowObjectInspector[tag.get()]);
+          } catch (Exception e2) {
+            rowString = "[Error getting row data with exception " + 
+                  StringUtils.stringifyException(e2) + " ]";
+          }
+          throw new HiveException("Hive Runtime Error while processing row (tag="
+              + tag.get() + ") " + rowString, e);
+        }
       }
 
     } catch (Throwable e) {
@@ -245,7 +261,8 @@ public class ExecReducer extends MapRedu
         // Don't create a new object if we are already out of memory
         throw (OutOfMemoryError) e;
       } else {
-        throw new IOException(e);
+        l4j.fatal(StringUtils.stringifyException(e));
+        throw new RuntimeException(e);
       }
     }
   }
@@ -283,12 +300,12 @@ public class ExecReducer extends MapRedu
       reducer.close(abort);
       reportStats rps = new reportStats(rp);
       reducer.preorderMap(rps);
-      return;
+      
     } catch (Exception e) {
       if (!abort) {
         // signal new failure to map-reduce
         l4j.error("Hit error while closing operators - failing tree");
-        throw new RuntimeException("Error while closing operators: "
+        throw new RuntimeException("Hive Runtime Error while closing operators: "
             + e.getMessage(), e);
       }
     }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=921994&r1=921993&r2=921994&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Thu Mar
11 19:31:09 2010
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.plan.Ma
 import org.apache.hadoop.hive.ql.plan.PartitionDesc;
 import org.apache.hadoop.hive.serde2.Deserializer;
 import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -43,6 +44,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * Map operator. This triggers overall map side processing. This is a little
@@ -371,18 +373,48 @@ public class MapOperator extends Operato
   }
 
   public void process(Writable value) throws HiveException {
+    Object row = null;
     try {
       if (!isPartitioned) {
-        Object row = deserializer.deserialize(value);
-        forward(row, rowObjectInspector);
+        row = deserializer.deserialize(value);
       } else {
         rowWithPart[0] = deserializer.deserialize(value);
-        forward(rowWithPart, rowObjectInspector);
       }
-    } catch (SerDeException e) {
+    } catch (Exception e) {
+      // Serialize the row and output.
+      String rawRowString;
+      try {
+        rawRowString = value.toString();
+      } catch (Exception e2) {
+        rawRowString = "[Error getting row data with exception " + 
+            StringUtils.stringifyException(e2) + " ]";
+      }
+      
       // TODO: policy on deserialization errors
       deserialize_error_count.set(deserialize_error_count.get() + 1);
-      throw new HiveException(e);
+      throw new HiveException("Hive Runtime Error while processing writable " + rawRowString,
e);
+    }
+    
+    try {
+      if (!isPartitioned) {
+        forward(row, rowObjectInspector);
+      } else {
+        forward(rowWithPart, rowObjectInspector);
+      }
+    } catch (Exception e) {
+      // Serialize the row and output the error message.
+      String rowString;
+      try {
+        if (!isPartitioned) {
+          rowString = SerDeUtils.getJSONString(row, rowObjectInspector);
+        } else {
+          rowString = SerDeUtils.getJSONString(rowWithPart, rowObjectInspector);
+        }
+      } catch (Exception e2) {
+        rowString = "[Error getting row data with exception " + 
+            StringUtils.stringifyException(e2) + " ]";
+      }
+      throw new HiveException("Hive Runtime Error while processing row " + rowString, e);
     }
   }
 

Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=921994&r1=921993&r2=921994&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Thu Mar 11 19:31:09
2010
@@ -251,6 +251,7 @@ public class QTestUtil {
       deleteDirectory(new File(warehousePath, s));
     }
     FunctionRegistry.unregisterTemporaryUDF("test_udaf");
+    FunctionRegistry.unregisterTemporaryUDF("test_error");
   }
 
   private void runLoadCmd(String loadCmd) throws Exception {

Added: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java?rev=921994&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java (added)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/udf/UDFTestErrorOnFalse.java Thu
Mar 11 19:31:09 2010
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+/**
+ * A UDF for testing, which throws RuntimeException if  the length of a string.
+ */
+public class UDFTestErrorOnFalse extends UDF {
+
+  public int evaluate(Boolean b) {
+    if (b) {
+      return 1;
+    } else {
+      throw new RuntimeException("UDFTestErrorOnFalse got b=false");
+    }
+  }
+}

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error.q?rev=921994&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error.q Thu Mar 11 19:31:09
2010
@@ -0,0 +1,3 @@
+CREATE TEMPORARY FUNCTION test_error AS 'org.apache.hadoop.hive.ql.udf.UDFTestErrorOnFalse';
+
+SELECT test_error(key < 125 OR key > 130) FROM src;

Added: hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error_reduce.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error_reduce.q?rev=921994&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error_reduce.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientnegative/udf_test_error_reduce.q Thu Mar 11
19:31:09 2010
@@ -0,0 +1,11 @@
+CREATE TEMPORARY FUNCTION test_error AS 'org.apache.hadoop.hive.ql.udf.UDFTestErrorOnFalse';
+
+
+SELECT test_error(key < 125 OR key > 130)
+FROM (
+  SELECT *
+  FROM src
+  DISTRIBUTE BY rand()
+) map_output;
+
+

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error.q.out?rev=921994&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error.q.out Thu Mar 11 19:31:09
2010
@@ -0,0 +1,9 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_error AS 'org.apache.hadoop.hive.ql.udf.UDFTestErrorOnFalse'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_error AS 'org.apache.hadoop.hive.ql.udf.UDFTestErrorOnFalse'
+POSTHOOK: type: CREATEFUNCTION
+PREHOOK: query: SELECT test_error(key < 125 OR key > 130) FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-06_00-58-40_004_2624763517220611615/10000
+FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask

Added: hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error_reduce.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error_reduce.q.out?rev=921994&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error_reduce.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientnegative/udf_test_error_reduce.q.out Thu Mar
11 19:31:09 2010
@@ -0,0 +1,14 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION test_error AS 'org.apache.hadoop.hive.ql.udf.UDFTestErrorOnFalse'
+PREHOOK: type: CREATEFUNCTION
+POSTHOOK: query: CREATE TEMPORARY FUNCTION test_error AS 'org.apache.hadoop.hive.ql.udf.UDFTestErrorOnFalse'
+POSTHOOK: type: CREATEFUNCTION
+PREHOOK: query: SELECT test_error(key < 125 OR key > 130)
+FROM (
+  SELECT *
+  FROM src
+  DISTRIBUTE BY rand()
+) map_output
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: file:/data/users/zshao/hadoop_hive_trunk/build/ql/scratchdir/hive_2010-03-05_23-12-16_809_4809554819212794550/10000
+FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.MapRedTask



Mime
View raw message