hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject svn commit: r1608618 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc: ColumnStatisticsImpl.java StringRedBlackTree.java WriterImpl.java
Date Mon, 07 Jul 2014 21:38:27 GMT
Author: gopalv
Date: Mon Jul  7 21:38:26 2014
New Revision: 1608618

URL: http://svn.apache.org/r1608618
Log:
HIVE-7144 : Use Text writables directly in ORC dictionaries to avoid String allocations (Gopal
V, reviewed by Gunther Hagleitner)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java?rev=1608618&r1=1608617&r2=1608618&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java Mon
Jul  7 21:38:26 2014
@@ -22,6 +22,7 @@ import org.apache.hadoop.hive.serde2.io.
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
 
 class ColumnStatisticsImpl implements ColumnStatistics {
 
@@ -335,8 +336,8 @@ class ColumnStatisticsImpl implements Co
 
   protected static final class StringStatisticsImpl extends ColumnStatisticsImpl
       implements StringColumnStatistics {
-    private String minimum = null;
-    private String maximum = null;
+    private Text minimum = null;
+    private Text maximum = null;
     private long sum = 0;
 
     StringStatisticsImpl() {
@@ -346,10 +347,10 @@ class ColumnStatisticsImpl implements Co
       super(stats);
       OrcProto.StringStatistics str = stats.getStringStatistics();
       if (str.hasMaximum()) {
-        maximum = str.getMaximum();
+        maximum = new Text(str.getMaximum());
       }
       if (str.hasMinimum()) {
-        minimum = str.getMinimum();
+        minimum = new Text(str.getMinimum());
       }
       if(str.hasSum()) {
         sum = str.getSum();
@@ -365,16 +366,15 @@ class ColumnStatisticsImpl implements Co
     }
 
     @Override
-    void updateString(String value) {
+    void updateString(Text value) {
       if (minimum == null) {
-        minimum = value;
-        maximum = value;
+        maximum = minimum = new Text(value.copyBytes());
       } else if (minimum.compareTo(value) > 0) {
-        minimum = value;
+        minimum = new Text(value.copyBytes());
       } else if (maximum.compareTo(value) < 0) {
-        maximum = value;
+        maximum = new Text(value.copyBytes());
       }
-      sum += value.length();
+      sum += value.getLength();
     }
 
     @Override
@@ -382,13 +382,18 @@ class ColumnStatisticsImpl implements Co
       super.merge(other);
       StringStatisticsImpl str = (StringStatisticsImpl) other;
       if (minimum == null) {
-        minimum = str.minimum;
-        maximum = str.maximum;
+        if(str.minimum != null) {
+          maximum = new Text(str.getMaximum());
+          minimum = new Text(str.getMinimum());
+        } else {
+          /* both are empty */
+          maximum = minimum = null;
+        }
       } else if (str.minimum != null) {
         if (minimum.compareTo(str.minimum) > 0) {
-          minimum = str.minimum;
+          minimum = new Text(str.getMinimum());
         } else if (maximum.compareTo(str.maximum) < 0) {
-          maximum = str.maximum;
+          maximum = new Text(str.getMaximum());
         }
       }
       sum += str.sum;
@@ -400,8 +405,8 @@ class ColumnStatisticsImpl implements Co
       OrcProto.StringStatistics.Builder str =
         OrcProto.StringStatistics.newBuilder();
       if (getNumberOfValues() != 0) {
-        str.setMinimum(minimum);
-        str.setMaximum(maximum);
+        str.setMinimum(getMinimum());
+        str.setMaximum(getMaximum());
         str.setSum(sum);
       }
       result.setStringStatistics(str);
@@ -410,12 +415,12 @@ class ColumnStatisticsImpl implements Co
 
     @Override
     public String getMinimum() {
-      return minimum;
+      return minimum == null ? null : minimum.toString();
     }
 
     @Override
     public String getMaximum() {
-      return maximum;
+      return maximum == null ? null : maximum.toString();
     }
 
     @Override
@@ -428,9 +433,9 @@ class ColumnStatisticsImpl implements Co
       StringBuilder buf = new StringBuilder(super.toString());
       if (getNumberOfValues() != 0) {
         buf.append(" min: ");
-        buf.append(minimum);
+        buf.append(getMinimum());
         buf.append(" max: ");
-        buf.append(maximum);
+        buf.append(getMaximum());
         buf.append(" sum: ");
         buf.append(sum);
       }
@@ -733,7 +738,7 @@ class ColumnStatisticsImpl implements Co
     throw new UnsupportedOperationException("Can't update double");
   }
 
-  void updateString(String value) {
+  void updateString(Text value) {
     throw new UnsupportedOperationException("Can't update string");
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java?rev=1608618&r1=1608617&r2=1608618&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/StringRedBlackTree.java Mon Jul
 7 21:38:26 2014
@@ -38,7 +38,11 @@ class StringRedBlackTree extends RedBlac
 
   public int add(String value) {
     newKey.set(value);
-    // if the key is new, add it to our byteArray and store the offset & length
+    return addNewKey();
+  }
+
+  private int addNewKey() {
+    // if the newKey is actually new, add it to our byteArray and store the offset &
length
     if (add()) {
       int len = newKey.getLength();
       keyOffsets.add(byteArray.add(newKey.getBytes(), 0, len));
@@ -46,6 +50,11 @@ class StringRedBlackTree extends RedBlac
     return lastAdd;
   }
 
+  public int add(Text value) {
+    newKey.set(value);
+    return addNewKey();
+  }
+
   @Override
   protected int compareValue(int position) {
     int start = keyOffsets.get(position);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1608618&r1=1608617&r2=1608618&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Mon Jul  7 21:38:26
2014
@@ -1032,20 +1032,20 @@ class WriterImpl implements Writer, Memo
     }
 
     /**
-     * Method to retrieve string values from the value object, which can be overridden
+     * Method to retrieve text values from the value object, which can be overridden
      * by subclasses.
      * @param obj  value
-     * @return String value from obj
+     * @return Text text value from obj
      */
-    String getStringValue(Object obj) {
-      return ((StringObjectInspector) inspector).getPrimitiveJavaObject(obj);
+    Text getTextValue(Object obj) {
+      return ((StringObjectInspector) inspector).getPrimitiveWritableObject(obj);
     }
 
     @Override
     void write(Object obj) throws IOException {
       super.write(obj);
       if (obj != null) {
-        String val = getStringValue(obj);
+        Text val = getTextValue(obj);
         rows.add(dictionary.add(val));
         indexStatistics.updateString(val);
       }
@@ -1194,9 +1194,9 @@ class WriterImpl implements Writer, Memo
      * Override base class implementation to support char values.
      */
     @Override
-    String getStringValue(Object obj) {
+    Text getTextValue(Object obj) {
       return (((HiveCharObjectInspector) inspector)
-          .getPrimitiveJavaObject(obj)).getValue();
+          .getPrimitiveWritableObject(obj)).getTextValue();
     }
   }
 
@@ -1216,9 +1216,9 @@ class WriterImpl implements Writer, Memo
      * Override base class implementation to support varchar values.
      */
     @Override
-    String getStringValue(Object obj) {
+    Text getTextValue(Object obj) {
       return (((HiveVarcharObjectInspector) inspector)
-          .getPrimitiveJavaObject(obj)).getValue();
+          .getPrimitiveWritableObject(obj)).getTextValue();
     }
   }
 



Mime
View raw message