hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1504983 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec: JoinOperator.java persistence/AbstractRowContainer.java persistence/MapJoinObjectValue.java persistence/MapJoinRowContainer.java persistence/RowContainer.java
Date Fri, 19 Jul 2013 19:57:05 GMT
Author: hashutosh
Date: Fri Jul 19 19:57:04 2013
New Revision: 1504983

URL: http://svn.apache.org/r1504983
Log:
HIVE-4730 : Join on more than 2^31 records on single reducer failed (wrong results) (Navis
via Ashutosh Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Fri Jul 19 19:57:04
2013
@@ -87,7 +87,7 @@ public class JoinOperator extends Common
       }
 
       // number of rows for the key in the given table
-      int sz = storage[alias].size();
+      long sz = storage[alias].size();
       StructObjectInspector soi = (StructObjectInspector) inputObjInspectors[tag];
       StructField sf = soi.getStructFieldRef(Utilities.ReduceField.KEY
           .toString());

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/AbstractRowContainer.java
Fri Jul 19 19:57:04 2013
@@ -38,7 +38,7 @@ public abstract class AbstractRowContain
    * @return number of elements in the RowContainer
    */
 
-  public abstract int size();
+  public abstract long size();
 
   /**
    * Remove all elements in the RowContainer.

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinObjectValue.java
Fri Jul 19 19:57:04 2013
@@ -140,7 +140,7 @@ public class MapJoinObjectValue implemen
 
       // Different processing for key and value
       MapJoinRowContainer<Object[]> v = obj;
-      out.writeInt(v.size());
+      out.writeInt((int)v.size());
       if (v.size() > 0) {
         Object[] row = v.first();
         out.writeInt(row.length);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinRowContainer.java
Fri Jul 19 19:57:04 2013
@@ -65,7 +65,7 @@ public class MapJoinRowContainer<Row> ex
    * @return number of elements in the RowContainer
    */
   @Override
-  public int size() {
+  public long size() {
     return list.size();
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java?rev=1504983&r1=1504982&r2=1504983&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java Fri
Jul 19 19:57:04 2013
@@ -86,7 +86,7 @@ public class RowContainer<Row extends Li
   private int blockSize; // number of objects in the block before it is spilled
   // to disk
   private int numFlushedBlocks; // total # of blocks
-  private int size; // total # of elements in the RowContainer
+  private long size;    // total # of elements in the RowContainer
   private File tmpFile; // temporary file holding the spilled blocks
   Path tempOutPath = null;
   private File parentFile;
@@ -283,7 +283,7 @@ public class RowContainer<Row extends Li
     }
   }
 
-  ArrayList<Object> row = new ArrayList<Object>(2);
+  private final ArrayList<Object> row = new ArrayList<Object>(2);
 
   private void spillBlock(Row[] block, int length) throws HiveException {
     try {
@@ -360,7 +360,7 @@ public class RowContainer<Row extends Li
    * @return number of elements in the RowContainer
    */
   @Override
-  public int size() {
+  public long size() {
     return size;
   }
 



Mime
View raw message