hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject svn commit: r1663385 - in /hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode: OrcColumnVectorProducer.java OrcEncodedDataConsumer.java
Date Mon, 02 Mar 2015 18:16:25 GMT
Author: prasanthj
Date: Mon Mar  2 18:16:25 2015
New Revision: 1663385

URL: http://svn.apache.org/r1663385
Log:
HIVE-9827: LLAP: Make stripe level column readers thread safe (Prasanth Jayachandran)

Modified:
    hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
    hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java?rev=1663385&r1=1663384&r2=1663385&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
(original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java
Mon Mar  2 18:16:25 2015
@@ -57,8 +57,7 @@ public class OrcColumnVectorProducer ext
   private final OrcEncodedDataProducer _edp;
   private final OrcMetadataCache _metadataCache;
   private boolean _skipCorrupt;
-  private int _previousStripeIndex;
- 
+
   public OrcColumnVectorProducer(ExecutorService executor, OrcEncodedDataProducer edp,
       Configuration conf) {
     super(executor);
@@ -69,7 +68,6 @@ public class OrcColumnVectorProducer ext
     this._edp = edp;
     this._metadataCache = OrcMetadataCache.getInstance();
     this._skipCorrupt = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_SKIP_CORRUPT_DATA);
-    this._previousStripeIndex = -1;
   }
 
   @Override
@@ -84,10 +82,10 @@ public class OrcColumnVectorProducer ext
     OrcEncodedDataConsumer oedc = (OrcEncodedDataConsumer)context;
     String fileName = batch.batchKey.file;
     int currentStripeIndex = batch.batchKey.stripeIx;
-    if (_previousStripeIndex == -1) {
-      _previousStripeIndex = currentStripeIndex;
+    if (oedc.getPreviousStripeIndex() == -1) {
+      oedc.setPreviousStripeIndex(currentStripeIndex);
     }
-    boolean sameStripe = currentStripeIndex == _previousStripeIndex;
+    boolean sameStripe = currentStripeIndex == oedc.getPreviousStripeIndex();
 
     // OrcEncodedDataProducer should have just loaded cache entries from this file.
     // The default LRU algorithm shouldn't have dropped the entries. To make it
@@ -119,7 +117,7 @@ public class OrcColumnVectorProducer ext
         repositionInStreams(oedc.getColumnReaders(), batch, sameStripe, numCols, fileMetadata,
             stripeMetadata);
       }
-      _previousStripeIndex = currentStripeIndex;
+      oedc.setPreviousStripeIndex(currentStripeIndex);
 
       for (int i = 0; i < maxBatchesRG; i++) {
         ColumnVectorBatch cvb = new ColumnVectorBatch(batch.columnIxs.length);

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java?rev=1663385&r1=1663384&r2=1663385&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
(original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java
Mon Mar  2 18:16:25 2015
@@ -18,18 +18,26 @@
 package org.apache.hadoop.hive.llap.io.decode;
 
 import org.apache.hadoop.hive.llap.Consumer;
-import org.apache.hadoop.hive.llap.ConsumerFeedback;
-import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch.StreamBuffer;
 import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch;
 import org.apache.hadoop.hive.llap.io.api.orc.OrcBatchKey;
 import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl;
 
 public class OrcEncodedDataConsumer extends EncodedDataConsumer<OrcBatchKey> {
+  private int previousStripeIndex;
   private RecordReaderImpl.TreeReader[] columnReaders;
 
   public OrcEncodedDataConsumer(
       ColumnVectorProducer<OrcBatchKey> cvp, Consumer<ColumnVectorBatch> consumer,
int colCount) {
     super(cvp, consumer, colCount);
+    this.previousStripeIndex = -1;
+  }
+
+  public int getPreviousStripeIndex() {
+    return previousStripeIndex;
+  }
+
+  public void setPreviousStripeIndex(int previousStripeIndex) {
+    this.previousStripeIndex = previousStripeIndex;
   }
 
   public void setColumnReaders(RecordReaderImpl.TreeReader[] columnReaders) {



Mime
View raw message