hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject svn commit: r1624354 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ ql/src/test/resources/ ql/src/test/results/clientpositive/ ql/src/test/results/clientpositive/tez/
Date Thu, 11 Sep 2014 18:17:32 GMT
Author: prasanthj
Date: Thu Sep 11 18:17:31 2014
New Revision: 1624354

URL: http://svn.apache.org/r1624354
Log:
HIVE-7859: Tune zlib compression in ORC to account for the encoding strategy (Gopal V reviewed by Prasanth J)

Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
    hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
    hive/trunk/ql/src/test/resources/orc-file-dump.out
    hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out
    hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
    hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out
    hive/trunk/ql/src/test/results/clientpositive/orc_merge3.q.out
    hive/trunk/ql/src/test/results/clientpositive/orc_merge4.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/orc_analyze.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge3.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge4.q.out
    hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
    hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Sep 11 18:17:31 2014
@@ -859,6 +859,10 @@ public class HiveConf extends Configurat
         "only affect the light weight encoding for integers. This flag will not\n" +
         "change the compression level of higher level compression codec (like ZLIB)."),
 
+    HIVE_ORC_COMPRESSION_STRATEGY("hive.exec.orc.compression.strategy", "SPEED", new StringSet("SPEED", "COMPRESSION"),
+         "Define the compression strategy to use while writing data. \n" +
+         "This changes the compression level of higher level compression codec (like ZLIB)."),
+
     HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS("hive.orc.splits.include.file.footer", false,
         "If turned on splits generated by orc will include metadata about the stripes in the file. This\n" +
         "data is read remotely (from the client or HS2 machine) and sent to all the tasks."),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/CompressionCodec.java Thu Sep 11 18:17:31 2014
@@ -19,8 +19,20 @@ package org.apache.hadoop.hive.ql.io.orc
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.EnumSet;
 
 interface CompressionCodec {
+
+  public enum Modifier {
+    /* speed/compression tradeoffs */
+    FASTEST,
+    FAST,
+    DEFAULT,
+    /* data sensitivity modifiers */
+    TEXT,
+    BINARY
+  };
+
   /**
    * Compress the in buffer to the out buffer.
    * @param in the bytes to compress
@@ -39,4 +51,17 @@ interface CompressionCodec {
    * @throws IOException
    */
   void decompress(ByteBuffer in, ByteBuffer out) throws IOException;
+
+  /**
+   * Produce a modified compression codec if the underlying algorithm allows
+   * modification.
+   *
+   * This does not modify the current object, but returns a new object if
+   * modifications are possible. Returns the same object if no modifications
+   * are possible.
+   * @param modifiers compression modifiers
+   * @return codec for use after optional modification
+   */
+  CompressionCodec modify(EnumSet<Modifier> modifiers);
+
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java Thu Sep 11 18:17:31 2014
@@ -101,6 +101,10 @@ public final class OrcFile {
     SPEED, COMPRESSION;
   }
 
+  public static enum CompressionStrategy {
+    SPEED, COMPRESSION;
+  }
+
   // Note : these string definitions for table properties are deprecated,
   // and retained only for backward compatibility, please do not add to
   // them, add to OrcTableProperties below instead
@@ -230,6 +234,7 @@ public final class OrcFile {
     private Version versionValue;
     private WriterCallback callback;
     private EncodingStrategy encodingStrategy;
+    private CompressionStrategy compressionStrategy;
     private float paddingTolerance;
 
     WriterOptions(Configuration conf) {
@@ -254,6 +259,15 @@ public final class OrcFile {
       } else {
         encodingStrategy = EncodingStrategy.valueOf(enString);
       }
+
+      String compString = conf
+          .get(HiveConf.ConfVars.HIVE_ORC_COMPRESSION_STRATEGY.varname);
+      if (compString == null) {
+        compressionStrategy = CompressionStrategy.SPEED;
+      } else {
+        compressionStrategy = CompressionStrategy.valueOf(compString);
+      }
+
       paddingTolerance =
           conf.getFloat(HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.varname,
               HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.defaultFloatVal);
@@ -403,7 +417,8 @@ public final class OrcFile {
                           opts.bufferSizeValue, opts.rowIndexStrideValue,
                           opts.memoryManagerValue, opts.blockPaddingValue,
                           opts.versionValue, opts.callback,
-                          opts.encodingStrategy, opts.paddingTolerance,
+                          opts.encodingStrategy, opts.compressionStrategy,
+                          opts.paddingTolerance,
                           opts.blockSizeValue);
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/SnappyCodec.java Thu Sep 11 18:17:31 2014
@@ -25,6 +25,7 @@ import org.iq80.snappy.Snappy;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.EnumSet;
 
 class SnappyCodec implements CompressionCodec, DirectDecompressionCodec {
 
@@ -99,4 +100,10 @@ class SnappyCodec implements Compression
     decompressShim.decompress(in, out);
     out.flip(); // flip for read
   }
+
+  @Override
+  public CompressionCodec modify(EnumSet<Modifier> modifiers) {
+    // snappy allows no modifications
+    return this;
+  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java Thu Sep 11 18:17:31 2014
@@ -24,6 +24,7 @@ import java.lang.management.ManagementFa
 import java.nio.ByteBuffer;
 import java.sql.Timestamp;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -37,6 +38,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.IOConstants;
+import org.apache.hadoop.hive.ql.io.orc.CompressionCodec.Modifier;
+import org.apache.hadoop.hive.ql.io.orc.OrcFile.CompressionStrategy;
 import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
 import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics;
@@ -139,6 +142,7 @@ class WriterImpl implements Writer, Memo
   private final OrcFile.WriterCallback callback;
   private final OrcFile.WriterContext callbackContext;
   private final OrcFile.EncodingStrategy encodingStrategy;
+  private final OrcFile.CompressionStrategy compressionStrategy;
 
   WriterImpl(FileSystem fs,
              Path path,
@@ -153,6 +157,7 @@ class WriterImpl implements Writer, Memo
              OrcFile.Version version,
              OrcFile.WriterCallback callback,
              OrcFile.EncodingStrategy encodingStrategy,
+             CompressionStrategy compressionStrategy,
              float paddingTolerance,
              long blockSizeValue) throws IOException {
     this.fs = fs;
@@ -174,6 +179,7 @@ class WriterImpl implements Writer, Memo
     this.defaultStripeSize = stripeSize;
     this.version = version;
     this.encodingStrategy = encodingStrategy;
+    this.compressionStrategy = compressionStrategy;
     this.addBlockPadding = addBlockPadding;
     this.blockSize = blockSizeValue;
     this.paddingTolerance = paddingTolerance;
@@ -447,10 +453,35 @@ class WriterImpl implements Writer, Memo
     public OutStream createStream(int column,
                                   OrcProto.Stream.Kind kind
                                   ) throws IOException {
-      StreamName name = new StreamName(column, kind);
+      final StreamName name = new StreamName(column, kind);
+      final EnumSet<CompressionCodec.Modifier> modifiers;
+
+      switch (kind) {
+      case DATA:
+      case DICTIONARY_DATA:
+        if (getCompressionStrategy() == CompressionStrategy.SPEED) {
+          modifiers = EnumSet.of(Modifier.FAST, Modifier.TEXT);
+        } else {
+          modifiers = EnumSet.of(Modifier.DEFAULT, Modifier.TEXT);
+        }
+        break;
+      case LENGTH:
+      case DICTIONARY_COUNT:
+      case PRESENT:
+      case ROW_INDEX:
+      case SECONDARY:
+        // easily compressed using the fastest modes
+        modifiers = EnumSet.of(Modifier.FASTEST, Modifier.BINARY);
+        break;
+      default:
+        modifiers = null;
+        break;
+      }
+
       BufferedStream result = streams.get(name);
       if (result == null) {
-        result = new BufferedStream(name.toString(), bufferSize, codec);
+        result = new BufferedStream(name.toString(), bufferSize,
+            codec == null ? codec : codec.modify(modifiers));
         streams.put(name, result);
       }
       return result.outStream;
@@ -496,6 +527,14 @@ class WriterImpl implements Writer, Memo
     }
 
     /**
+     * Get the compression strategy to use.
+     * @return compression strategy
+     */
+    public CompressionStrategy getCompressionStrategy() {
+      return compressionStrategy;
+    }
+
+    /**
      * Get the writer's configuration.
      * @return configuration
      */

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ZlibCodec.java Thu Sep 11 18:17:31 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.io.orc
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.EnumSet;
 import java.util.zip.DataFormatException;
 import java.util.zip.Deflater;
 import java.util.zip.Inflater;
@@ -32,10 +33,24 @@ class ZlibCodec implements CompressionCo
 
   private Boolean direct = null;
 
+  private final int level;
+  private final int strategy;
+
+  public ZlibCodec() {
+    level = Deflater.DEFAULT_COMPRESSION;
+    strategy = Deflater.DEFAULT_STRATEGY;
+  }
+
+  private ZlibCodec(int level, int strategy) {
+    this.level = level;
+    this.strategy = strategy;
+  }
+
   @Override
   public boolean compress(ByteBuffer in, ByteBuffer out,
                           ByteBuffer overflow) throws IOException {
-    Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true);
+    Deflater deflater = new Deflater(level, true);
+    deflater.setStrategy(strategy);
     int length = in.remaining();
     deflater.setInput(in.array(), in.arrayOffset() + in.position(), length);
     deflater.finish();
@@ -113,4 +128,37 @@ class ZlibCodec implements CompressionCo
     decompressShim.decompress(in, out);
     out.flip(); // flip for read
   }
+
+  @Override
+  public CompressionCodec modify(EnumSet<Modifier> modifiers) {
+    int l = this.level;
+    int s = this.strategy;
+
+    for (Modifier m : modifiers) {
+      switch (m) {
+      case BINARY:
+        /* filtered == less LZ77, more huffman */
+        s = Deflater.FILTERED;
+        break;
+      case TEXT:
+        s = Deflater.DEFAULT_STRATEGY;
+        break;
+      case FASTEST:
+        // deflate_fast looking for 8 byte patterns
+        l = Deflater.BEST_SPEED;
+        break;
+      case FAST:
+        // deflate_fast looking for 16 byte patterns
+        l = Deflater.BEST_SPEED + 1;
+        break;
+      case DEFAULT:
+        // deflate_slow looking for 128 byte patterns
+        l = Deflater.DEFAULT_COMPRESSION;
+        break;
+      default:
+        break;
+      }
+    }
+    return new ZlibCodec(l, s);
+  }
 }

Modified: hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out (original)
+++ hive/trunk/ql/src/test/resources/orc-file-dump-dictionary-threshold.out Thu Sep 11 18:17:31 2014
@@ -38,15 +38,15 @@ File Statistics:
   Column 3: count: 21000 min: Darkness,-230 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 sum: 6910238
 
 Stripes:
-  Stripe: offset: 3 data: 144733 rows: 5000 tail: 68 index: 693
+  Stripe: offset: 3 data: 151109 rows: 5000 tail: 68 index: 704
     Stream: column 0 section ROW_INDEX start: 3 length 15
     Stream: column 1 section ROW_INDEX start: 18 length 156
-    Stream: column 2 section ROW_INDEX start: 174 length 170
-    Stream: column 3 section ROW_INDEX start: 344 length 352
-    Stream: column 1 section DATA start: 696 length 20029
-    Stream: column 2 section DATA start: 20725 length 40035
-    Stream: column 3 section DATA start: 60760 length 80382
-    Stream: column 3 section LENGTH start: 141142 length 4287
+    Stream: column 2 section ROW_INDEX start: 174 length 172
+    Stream: column 3 section ROW_INDEX start: 346 length 361
+    Stream: column 1 section DATA start: 707 length 20029
+    Stream: column 2 section DATA start: 20736 length 40035
+    Stream: column 3 section DATA start: 60771 length 86757
+    Stream: column 3 section LENGTH start: 147528 length 4288
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -65,19 +65,19 @@ Stripes:
       Entry 4: count: 1000 min: -9216505819108477308 max: 9196474183833079923 positions: 20006,8686,416
     Row group index column 3:
       Entry 0: count: 1000 min: Darkness,-230 max: worst-54-290-346-648-908-996 positions: 0,0,0,0,0
-      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966 positions: 2083,8442,0,696,18
-      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660 positions: 11712,4780,0,1555,14
-      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788 positions: 28297,228,0,2373,90
-      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744 positions: 49599,5096,0,3355,108
-  Stripe: offset: 145497 data: 321684 rows: 5000 tail: 68 index: 907
-    Stream: column 0 section ROW_INDEX start: 145497 length 15
-    Stream: column 1 section ROW_INDEX start: 145512 length 150
-    Stream: column 2 section ROW_INDEX start: 145662 length 167
-    Stream: column 3 section ROW_INDEX start: 145829 length 575
-    Stream: column 1 section DATA start: 146404 length 20029
-    Stream: column 2 section DATA start: 166433 length 40035
-    Stream: column 3 section DATA start: 206468 length 256119
-    Stream: column 3 section LENGTH start: 462587 length 5501
+      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966 positions: 2777,8442,0,696,18
+      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660 positions: 13595,4780,0,1555,14
+      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788 positions: 31432,228,0,2373,90
+      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744 positions: 54111,5096,0,3355,108
+  Stripe: offset: 151884 data: 336358 rows: 5000 tail: 69 index: 941
+    Stream: column 0 section ROW_INDEX start: 151884 length 15
+    Stream: column 1 section ROW_INDEX start: 151899 length 150
+    Stream: column 2 section ROW_INDEX start: 152049 length 167
+    Stream: column 3 section ROW_INDEX start: 152216 length 609
+    Stream: column 1 section DATA start: 152825 length 20029
+    Stream: column 2 section DATA start: 172854 length 40035
+    Stream: column 3 section DATA start: 212889 length 270789
+    Stream: column 3 section LENGTH start: 483678 length 5505
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -96,19 +96,19 @@ Stripes:
       Entry 4: count: 1000 min: -9201438531577205959 max: 9212462124593119846 positions: 20006,8686,416
     Row group index column 3:
       Entry 0: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726 positions: 0,0,0,0,0
-      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994 positions: 35030,6320,0,967,90
-      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988 positions: 76249,9756,0,1945,222
-      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984 positions: 129633,4496,0,3268,48
-      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938 positions: 187250,6590,0,4064,342
-  Stripe: offset: 468156 data: 531773 rows: 5000 tail: 69 index: 1101
-    Stream: column 0 section ROW_INDEX start: 468156 length 15
-    Stream: column 1 section ROW_INDEX start: 468171 length 159
-    Stream: column 2 section ROW_INDEX start: 468330 length 169
-    Stream: column 3 section ROW_INDEX start: 468499 length 758
-    Stream: column 1 section DATA start: 469257 length 20029
-    Stream: column 2 section DATA start: 489286 length 40035
-    Stream: column 3 section DATA start: 529321 length 466002
-    Stream: column 3 section LENGTH start: 995323 length 5707
+      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994 positions: 37112,6320,0,967,90
+      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988 positions: 80822,9756,0,1945,222
+      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984 positions: 137149,4496,0,3268,48
+      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938 positions: 197972,6590,0,4064,342
+  Stripe: offset: 489252 data: 558031 rows: 5000 tail: 69 index: 1169
+    Stream: column 0 section ROW_INDEX start: 489252 length 15
+    Stream: column 1 section ROW_INDEX start: 489267 length 159
+    Stream: column 2 section ROW_INDEX start: 489426 length 169
+    Stream: column 3 section ROW_INDEX start: 489595 length 826
+    Stream: column 1 section DATA start: 490421 length 20029
+    Stream: column 2 section DATA start: 510450 length 40035
+    Stream: column 3 section DATA start: 550485 length 492258
+    Stream: column 3 section LENGTH start: 1042743 length 5709
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -127,19 +127,19 @@ Stripes:
       Entry 4: count: 1000 min: -9215483580266514322 max: 9220102792864959501 positions: 20006,8686,416
     Row group index column 3:
       Entry 0: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876 positions: 0,0,0,0,0
-      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964 positions: 76243,3880,0,1097,28
-      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976 positions: 161945,3422,0,2077,16
 2
-      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13
 246-13502-13766 positions: 254465,9960,0,3369,16
-      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12
 782-12790-12802-12976-13216-13246-13502-13766-14454-14974 positions: 357930,1620,0,4041,470
-  Stripe: offset: 1001099 data: 751374 rows: 5000 tail: 69 index: 1278
-    Stream: column 0 section ROW_INDEX start: 1001099 length 15
-    Stream: column 1 section ROW_INDEX start: 1001114 length 149
-    Stream: column 2 section ROW_INDEX start: 1001263 length 170
-    Stream: column 3 section ROW_INDEX start: 1001433 length 944
-    Stream: column 1 section DATA start: 1002377 length 20029
-    Stream: column 2 section DATA start: 1022406 length 40035
-    Stream: column 3 section DATA start: 1062441 length 685567
-    Stream: column 3 section LENGTH start: 1748008 length 5743
+      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964 positions: 80352,3880,0,1097,28
+      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976 positions: 170641,3422,0,2077,16
 2
+      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13
 246-13502-13766 positions: 268420,9960,0,3369,16
+      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12
 782-12790-12802-12976-13216-13246-13502-13766-14454-14974 positions: 377916,1620,0,4041,470
+  Stripe: offset: 1048521 data: 792850 rows: 5000 tail: 69 index: 1351
+    Stream: column 0 section ROW_INDEX start: 1048521 length 15
+    Stream: column 1 section ROW_INDEX start: 1048536 length 149
+    Stream: column 2 section ROW_INDEX start: 1048685 length 170
+    Stream: column 3 section ROW_INDEX start: 1048855 length 1017
+    Stream: column 1 section DATA start: 1049872 length 20029
+    Stream: column 2 section DATA start: 1069901 length 40035
+    Stream: column 3 section DATA start: 1109936 length 727038
+    Stream: column 3 section LENGTH start: 1836974 length 5748
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -158,19 +158,19 @@ Stripes:
       Entry 4: count: 1000 min: -9197393848859294562 max: 9208134757538374043 positions: 20006,8686,416
     Row group index column 3:
       Entry 0: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12
 096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610 positions: 0,0,0,0,0
-      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11
 722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936 positions: 120094,2916,0,1077,140
-      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9
 938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878 positions: 248557,206,0,1926,462
-      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-93
 44-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788 positions: 384576,8480,0,3444,250
-      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8
 620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904 positions: 530195,3058,0,4643,292
-  Stripe: offset: 1753820 data: 177935 rows: 1000 tail: 67 index: 813
-    Stream: column 0 section ROW_INDEX start: 1753820 length 10
-    Stream: column 1 section ROW_INDEX start: 1753830 length 36
-    Stream: column 2 section ROW_INDEX start: 1753866 length 39
-    Stream: column 3 section ROW_INDEX start: 1753905 length 728
-    Stream: column 1 section DATA start: 1754633 length 4007
-    Stream: column 2 section DATA start: 1758640 length 8007
-    Stream: column 3 section DATA start: 1766647 length 164661
-    Stream: column 3 section LENGTH start: 1931308 length 1260
+      Entry 1: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11
 722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936 positions: 126968,2916,0,1077,140
+      Entry 2: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9
 938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878 positions: 263111,206,0,1926,462
+      Entry 3: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-93
 44-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788 positions: 407371,8480,0,3444,250
+      Entry 4: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-7960-7988-8232-8256-8390-8416-8478-8
 620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904 positions: 562094,3058,0,4643,292
+  Stripe: offset: 1842791 data: 188033 rows: 1000 tail: 67 index: 832
+    Stream: column 0 section ROW_INDEX start: 1842791 length 10
+    Stream: column 1 section ROW_INDEX start: 1842801 length 36
+    Stream: column 2 section ROW_INDEX start: 1842837 length 39
+    Stream: column 3 section ROW_INDEX start: 1842876 length 747
+    Stream: column 1 section DATA start: 1843623 length 4007
+    Stream: column 2 section DATA start: 1847630 length 8007
+    Stream: column 3 section DATA start: 1855637 length 174759
+    Stream: column 3 section LENGTH start: 2030396 length 1260
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -182,6 +182,6 @@ Stripes:
     Row group index column 3:
       Entry 0: count: 1000 min: Darkness,-230-368-488-586-862-930-1686-2044-2636-2652-2872-3108-3162-3192-3404-3442-3508-3542-3550-3712-3980-4146-4204-4336-4390-4418-4424-4490-4512-4650-4768-4924-4950-5210-5524-5630-5678-5710-5758-5952-6238-6252-6300-6366-6668-6712-6926-6942-7100-7194-7802-8030-8452-8608-8640-8862-8868-9134-9234-9412-9602-9608-9642-9678-9740-9780-10426-10510-10514-10706-10814-10870-10942-11028-11244-11326-11462-11496-11656-11830-12022-12178-12418-12832-13304-13448-13590-13618-13908-14188-14246-14340-14364-14394-14762-14850-14964-15048-15494-15674-15726-16006-16056-16180-16304-16332-16452-16598-16730-16810-16994-17210-17268-17786-17962-18214-18444-18446-18724-18912-18952-19164-19348-19400-19546-19776-19896-20084 max: worst-54-290-346-648-908-996-1038-1080-1560-1584-1620-1744-1770-1798-1852-1966-2162-2244-2286-2296-2534-2660-3114-3676-3788-4068-4150-4706-4744-5350-5420-5582-5696-5726-6006-6020-6024-6098-6184-6568-6636-6802-6994-7004-7318-7498-7758-7780-7798-7920-7952-
 7960-7988-8232-8256-8390-8416-8478-8620-8840-8984-9038-9128-9236-9248-9344-9594-9650-9714-9928-9938-10178-10368-10414-10502-10732-10876-11008-11158-11410-11722-11836-11964-12054-12096-12126-12136-12202-12246-12298-12616-12774-12782-12790-12802-12976-13216-13246-13502-13766-14454-14974-15004-15124-15252-15294-15356-15530-15610-16316-16936-17024-17122-17214-17310-17528-17682-17742-17870-17878-18010-18410-18524-18788-19204-19254-19518-19596-19786-19874-19904-20390-20752-20936 positions: 0,0,0,0,0
 
-File length: 1934469 bytes
+File length: 2033559 bytes
 Padding length: 0 bytes
 Padding ratio: 0%

Modified: hive/trunk/ql/src/test/resources/orc-file-dump.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/resources/orc-file-dump.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/resources/orc-file-dump.out (original)
+++ hive/trunk/ql/src/test/resources/orc-file-dump.out Thu Sep 11 18:17:31 2014
@@ -38,16 +38,16 @@ File Statistics:
   Column 3: count: 21000 min: Darkness, max: worst sum: 81761
 
 Stripes:
-  Stripe: offset: 3 data: 63766 rows: 5000 tail: 79 index: 426
+  Stripe: offset: 3 data: 63766 rows: 5000 tail: 79 index: 428
     Stream: column 0 section ROW_INDEX start: 3 length 15
     Stream: column 1 section ROW_INDEX start: 18 length 158
-    Stream: column 2 section ROW_INDEX start: 176 length 170
-    Stream: column 3 section ROW_INDEX start: 346 length 83
-    Stream: column 1 section DATA start: 429 length 20029
-    Stream: column 2 section DATA start: 20458 length 40035
-    Stream: column 3 section DATA start: 60493 length 3544
-    Stream: column 3 section LENGTH start: 64037 length 25
-    Stream: column 3 section DICTIONARY_DATA start: 64062 length 133
+    Stream: column 2 section ROW_INDEX start: 176 length 171
+    Stream: column 3 section ROW_INDEX start: 347 length 84
+    Stream: column 1 section DATA start: 431 length 20029
+    Stream: column 2 section DATA start: 20460 length 40035
+    Stream: column 3 section DATA start: 60495 length 3544
+    Stream: column 3 section LENGTH start: 64039 length 25
+    Stream: column 3 section DICTIONARY_DATA start: 64064 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -70,16 +70,16 @@ Stripes:
       Entry 2: count: 1000 min: Darkness, max: worst positions: 0,1531,3
       Entry 3: count: 1000 min: Darkness, max: worst positions: 0,2282,32
       Entry 4: count: 1000 min: Darkness, max: worst positions: 0,3034,45
-  Stripe: offset: 64274 data: 63755 rows: 5000 tail: 79 index: 421
-    Stream: column 0 section ROW_INDEX start: 64274 length 15
-    Stream: column 1 section ROW_INDEX start: 64289 length 157
-    Stream: column 2 section ROW_INDEX start: 64446 length 169
-    Stream: column 3 section ROW_INDEX start: 64615 length 80
-    Stream: column 1 section DATA start: 64695 length 20029
-    Stream: column 2 section DATA start: 84724 length 40035
-    Stream: column 3 section DATA start: 124759 length 3533
-    Stream: column 3 section LENGTH start: 128292 length 25
-    Stream: column 3 section DICTIONARY_DATA start: 128317 length 133
+  Stripe: offset: 64276 data: 63755 rows: 5000 tail: 79 index: 421
+    Stream: column 0 section ROW_INDEX start: 64276 length 15
+    Stream: column 1 section ROW_INDEX start: 64291 length 157
+    Stream: column 2 section ROW_INDEX start: 64448 length 169
+    Stream: column 3 section ROW_INDEX start: 64617 length 80
+    Stream: column 1 section DATA start: 64697 length 20029
+    Stream: column 2 section DATA start: 84726 length 40035
+    Stream: column 3 section DATA start: 124761 length 3533
+    Stream: column 3 section LENGTH start: 128294 length 25
+    Stream: column 3 section DICTIONARY_DATA start: 128319 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -102,16 +102,16 @@ Stripes:
       Entry 2: count: 1000 min: Darkness, max: worst positions: 0,1472,70
       Entry 3: count: 1000 min: Darkness, max: worst positions: 0,2250,43
       Entry 4: count: 1000 min: Darkness, max: worst positions: 0,2979,88
-  Stripe: offset: 128529 data: 63766 rows: 5000 tail: 79 index: 421
-    Stream: column 0 section ROW_INDEX start: 128529 length 15
-    Stream: column 1 section ROW_INDEX start: 128544 length 153
-    Stream: column 2 section ROW_INDEX start: 128697 length 169
-    Stream: column 3 section ROW_INDEX start: 128866 length 84
-    Stream: column 1 section DATA start: 128950 length 20029
-    Stream: column 2 section DATA start: 148979 length 40035
-    Stream: column 3 section DATA start: 189014 length 3544
-    Stream: column 3 section LENGTH start: 192558 length 25
-    Stream: column 3 section DICTIONARY_DATA start: 192583 length 133
+  Stripe: offset: 128531 data: 63766 rows: 5000 tail: 79 index: 422
+    Stream: column 0 section ROW_INDEX start: 128531 length 15
+    Stream: column 1 section ROW_INDEX start: 128546 length 153
+    Stream: column 2 section ROW_INDEX start: 128699 length 169
+    Stream: column 3 section ROW_INDEX start: 128868 length 85
+    Stream: column 1 section DATA start: 128953 length 20029
+    Stream: column 2 section DATA start: 148982 length 40035
+    Stream: column 3 section DATA start: 189017 length 3544
+    Stream: column 3 section LENGTH start: 192561 length 25
+    Stream: column 3 section DICTIONARY_DATA start: 192586 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -134,16 +134,16 @@ Stripes:
       Entry 2: count: 1000 min: Darkness, max: worst positions: 0,1469,69
       Entry 3: count: 1000 min: Darkness, max: worst positions: 0,2133,194
       Entry 4: count: 1000 min: Darkness, max: worst positions: 0,3005,43
-  Stripe: offset: 192795 data: 63796 rows: 5000 tail: 79 index: 425
-    Stream: column 0 section ROW_INDEX start: 192795 length 15
-    Stream: column 1 section ROW_INDEX start: 192810 length 156
-    Stream: column 2 section ROW_INDEX start: 192966 length 168
-    Stream: column 3 section ROW_INDEX start: 193134 length 86
-    Stream: column 1 section DATA start: 193220 length 20029
-    Stream: column 2 section DATA start: 213249 length 40035
-    Stream: column 3 section DATA start: 253284 length 3574
-    Stream: column 3 section LENGTH start: 256858 length 25
-    Stream: column 3 section DICTIONARY_DATA start: 256883 length 133
+  Stripe: offset: 192798 data: 63796 rows: 5000 tail: 79 index: 425
+    Stream: column 0 section ROW_INDEX start: 192798 length 15
+    Stream: column 1 section ROW_INDEX start: 192813 length 156
+    Stream: column 2 section ROW_INDEX start: 192969 length 168
+    Stream: column 3 section ROW_INDEX start: 193137 length 86
+    Stream: column 1 section DATA start: 193223 length 20029
+    Stream: column 2 section DATA start: 213252 length 40035
+    Stream: column 3 section DATA start: 253287 length 3574
+    Stream: column 3 section LENGTH start: 256861 length 25
+    Stream: column 3 section DICTIONARY_DATA start: 256886 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -166,16 +166,16 @@ Stripes:
       Entry 2: count: 1000 min: Darkness, max: worst positions: 0,1485,52
       Entry 3: count: 1000 min: Darkness, max: worst positions: 0,2196,104
       Entry 4: count: 1000 min: Darkness, max: worst positions: 0,2934,131
-  Stripe: offset: 257095 data: 12940 rows: 1000 tail: 71 index: 123
-    Stream: column 0 section ROW_INDEX start: 257095 length 10
-    Stream: column 1 section ROW_INDEX start: 257105 length 36
-    Stream: column 2 section ROW_INDEX start: 257141 length 39
-    Stream: column 3 section ROW_INDEX start: 257180 length 38
-    Stream: column 1 section DATA start: 257218 length 4007
-    Stream: column 2 section DATA start: 261225 length 8007
-    Stream: column 3 section DATA start: 269232 length 768
-    Stream: column 3 section LENGTH start: 270000 length 25
-    Stream: column 3 section DICTIONARY_DATA start: 270025 length 133
+  Stripe: offset: 257098 data: 12940 rows: 1000 tail: 71 index: 123
+    Stream: column 0 section ROW_INDEX start: 257098 length 10
+    Stream: column 1 section ROW_INDEX start: 257108 length 36
+    Stream: column 2 section ROW_INDEX start: 257144 length 39
+    Stream: column 3 section ROW_INDEX start: 257183 length 38
+    Stream: column 1 section DATA start: 257221 length 4007
+    Stream: column 2 section DATA start: 261228 length 8007
+    Stream: column 3 section DATA start: 269235 length 768
+    Stream: column 3 section LENGTH start: 270003 length 25
+    Stream: column 3 section DICTIONARY_DATA start: 270028 length 133
     Encoding column 0: DIRECT
     Encoding column 1: DIRECT_V2
     Encoding column 2: DIRECT_V2
@@ -187,6 +187,6 @@ Stripes:
     Row group index column 3:
       Entry 0: count: 1000 min: Darkness, max: worst positions: 0,0,0
 
-File length: 270756 bytes
+File length: 270759 bytes
 Padding length: 0 bytes
 Padding ratio: 0%

Modified: hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_orc.q.out Thu Sep 11 18:17:31 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
 partitioned:false
 partitionColumns:
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test
@@ -91,9 +91,9 @@ columns:struct columns { i32 key, string
 partitioned:false
 partitionColumns:
 totalNumberFiles:1
-totalFileSize:7059
-maxFileSize:7059
-minFileSize:7059
+totalFileSize:7167
+maxFileSize:7167
+minFileSize:7167
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test
@@ -171,9 +171,9 @@ columns:struct columns { i32 key, string
 partitioned:true
 partitionColumns:struct partition_columns { string ds}
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test_part
@@ -218,9 +218,9 @@ columns:struct columns { i32 key, string
 partitioned:true
 partitionColumns:struct partition_columns { string ds}
 totalNumberFiles:1
-totalFileSize:7059
-maxFileSize:7059
-minFileSize:7059
+totalFileSize:7167
+maxFileSize:7167
+minFileSize:7167
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test_part

Modified: hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/alter_merge_stats_orc.q.out Thu Sep 11 18:17:31 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
 partitioned:false
 partitionColumns:
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: desc extended src_orc_merge_test_stat
@@ -92,7 +92,7 @@ Table Parameters:	 	 
 	numFiles            	3                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7380                
+	totalSize           	7488                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -142,7 +142,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7059                
+	totalSize           	7167                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -212,9 +212,9 @@ columns:struct columns { i32 key, string
 partitioned:true
 partitionColumns:struct partition_columns { string ds}
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011')
@@ -245,7 +245,7 @@ Partition Parameters:	 	 
 	numFiles            	3                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	7380                
+	totalSize           	7488                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -294,7 +294,7 @@ Partition Parameters:	 	 
 	numFiles            	3                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7380                
+	totalSize           	7488                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -351,7 +351,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7059                
+	totalSize           	7167                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/orc_analyze.q.out Thu Sep 11 18:17:31 2014
@@ -105,7 +105,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	100                 
 	rawDataSize         	52600               
-	totalSize           	3098                
+	totalSize           	3123                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -195,7 +195,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	100                 
 	rawDataSize         	52600               
-	totalSize           	3098                
+	totalSize           	3123                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -309,7 +309,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	21950               
-	totalSize           	2016                
+	totalSize           	2024                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -354,7 +354,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	22050               
-	totalSize           	2036                
+	totalSize           	2043                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -456,7 +456,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	21950               
-	totalSize           	2016                
+	totalSize           	2024                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -501,7 +501,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	22050               
-	totalSize           	2036                
+	totalSize           	2043                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -621,7 +621,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	21980               
-	totalSize           	4955                
+	totalSize           	4963                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -666,7 +666,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22048               
-	totalSize           	5046                
+	totalSize           	5051                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -774,7 +774,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	21980               
-	totalSize           	4955                
+	totalSize           	4963                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -819,7 +819,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22048               
-	totalSize           	5046                
+	totalSize           	5051                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -984,7 +984,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	21950               
-	totalSize           	2016                
+	totalSize           	2024                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/orc_merge3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/orc_merge3.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/orc_merge3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/orc_merge3.q.out Thu Sep 11 18:17:31 2014
@@ -161,7 +161,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	1000                
 	rawDataSize         	94000               
-	totalSize           	4762                
+	totalSize           	4834                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/orc_merge4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/orc_merge4.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/orc_merge4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/orc_merge4.q.out Thu Sep 11 18:17:31 2014
@@ -64,7 +64,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	2460                
+	totalSize           	2496                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -129,7 +129,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	2460                
+	totalSize           	2496                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -170,7 +170,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	2460                
+	totalSize           	2496                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_orc.q.out Thu Sep 11 18:17:31 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
 partitioned:false
 partitionColumns:
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test
@@ -91,9 +91,9 @@ columns:struct columns { i32 key, string
 partitioned:false
 partitionColumns:
 totalNumberFiles:1
-totalFileSize:7059
-maxFileSize:7059
-minFileSize:7059
+totalFileSize:7167
+maxFileSize:7167
+minFileSize:7167
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test
@@ -171,9 +171,9 @@ columns:struct columns { i32 key, string
 partitioned:true
 partitionColumns:struct partition_columns { string ds}
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test_part
@@ -218,9 +218,9 @@ columns:struct columns { i32 key, string
 partitioned:true
 partitionColumns:struct partition_columns { string ds}
 totalNumberFiles:1
-totalFileSize:7059
-maxFileSize:7059
-minFileSize:7059
+totalFileSize:7167
+maxFileSize:7167
+minFileSize:7167
 #### A masked pattern was here ####
 
 PREHOOK: query: select count(1) from src_orc_merge_test_part

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/alter_merge_stats_orc.q.out Thu Sep 11 18:17:31 2014
@@ -48,9 +48,9 @@ columns:struct columns { i32 key, string
 partitioned:false
 partitionColumns:
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: desc extended src_orc_merge_test_stat
@@ -92,7 +92,7 @@ Table Parameters:	 	 
 	numFiles            	3                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7380                
+	totalSize           	7488                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -142,7 +142,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7059                
+	totalSize           	7167                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -212,9 +212,9 @@ columns:struct columns { i32 key, string
 partitioned:true
 partitionColumns:struct partition_columns { string ds}
 totalNumberFiles:3
-totalFileSize:7380
-maxFileSize:2460
-minFileSize:2460
+totalFileSize:7488
+maxFileSize:2496
+minFileSize:2496
 #### A masked pattern was here ####
 
 PREHOOK: query: desc formatted src_orc_merge_test_part_stat partition (ds='2011')
@@ -245,7 +245,7 @@ Partition Parameters:	 	 
 	numFiles            	3                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	7380                
+	totalSize           	7488                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -294,7 +294,7 @@ Partition Parameters:	 	 
 	numFiles            	3                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7380                
+	totalSize           	7488                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -351,7 +351,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	1500                
 	rawDataSize         	141000              
-	totalSize           	7059                
+	totalSize           	7167                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/orc_analyze.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/orc_analyze.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/orc_analyze.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/orc_analyze.q.out Thu Sep 11 18:17:31 2014
@@ -105,7 +105,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	100                 
 	rawDataSize         	52600               
-	totalSize           	3098                
+	totalSize           	3123                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -195,7 +195,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	100                 
 	rawDataSize         	52600               
-	totalSize           	3098                
+	totalSize           	3123                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -309,7 +309,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	21950               
-	totalSize           	2016                
+	totalSize           	2024                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -354,7 +354,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	22050               
-	totalSize           	2036                
+	totalSize           	2043                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -456,7 +456,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	21950               
-	totalSize           	2016                
+	totalSize           	2024                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -501,7 +501,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	22050               
-	totalSize           	2036                
+	totalSize           	2043                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -621,7 +621,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	21980               
-	totalSize           	4955                
+	totalSize           	4963                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -666,7 +666,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22048               
-	totalSize           	5046                
+	totalSize           	5051                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -774,7 +774,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	21980               
-	totalSize           	4955                
+	totalSize           	4963                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -819,7 +819,7 @@ Partition Parameters:	 	 
 	numFiles            	4                   
 	numRows             	50                  
 	rawDataSize         	22048               
-	totalSize           	5046                
+	totalSize           	5051                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -984,7 +984,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	50                  
 	rawDataSize         	21950               
-	totalSize           	2016                
+	totalSize           	2024                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge3.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge3.q.out Thu Sep 11 18:17:31 2014
@@ -138,7 +138,7 @@ Table Parameters:	 	 
 	numFiles            	1                   
 	numRows             	1000                
 	rawDataSize         	94000               
-	totalSize           	2502                
+	totalSize           	2538                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge4.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/tez/orc_merge4.q.out Thu Sep 11 18:17:31 2014
@@ -64,7 +64,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	2460                
+	totalSize           	2496                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -129,7 +129,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	2460                
+	totalSize           	2496                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 
@@ -170,7 +170,7 @@ Partition Parameters:	 	 
 	numFiles            	1                   
 	numRows             	500                 
 	rawDataSize         	47000               
-	totalSize           	2460                
+	totalSize           	2496                
 #### A masked pattern was here ####
 	 	 
 # Storage Information	 	 

Modified: hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out Thu Sep 11 18:17:31 2014 differ

Modified: hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out?rev=1624354&r1=1624353&r2=1624354&view=diff
==============================================================================
Files hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out (original) and hive/trunk/ql/src/test/results/clientpositive/vectorized_ptf.q.out Thu Sep 11 18:17:31 2014 differ



Mime
View raw message