drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From meh...@apache.org
Subject [5/5] drill git commit: DRILL-3345: TestWindowFrame fails to properly check cases involving multiple batches
Date Thu, 25 Jun 2015 05:30:39 GMT
DRILL-3345: TestWindowFrame fails to properly check cases involving multiple batches


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3f0d9221
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3f0d9221
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3f0d9221

Branch: refs/heads/master
Commit: 3f0d9221d3f96c20db10e868cc33c2e972318ba6
Parents: 00aa01f
Author: adeneche <adeneche@gmail.com>
Authored: Wed Jun 24 08:58:39 2015 -0700
Committer: Mehant Baid <mehantr@gmail.com>
Committed: Wed Jun 24 16:54:36 2015 -0700

----------------------------------------------------------------------
 .../impl/window/DefaultFrameTemplate.java       |  6 +++---
 .../exec/physical/impl/xsort/MSortTemplate.java | 21 +++++++++++---------
 .../exec/record/selection/SelectionVector4.java | 16 +++++++++++----
 3 files changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/3f0d9221/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
index ada068b..535deaa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/DefaultFrameTemplate.java
@@ -242,11 +242,11 @@ public abstract class DefaultFrameTemplate implements WindowFramer {
     final int lastSize = last.getRecordCount();
 
     if (!isSamePartition(currentSize - 1, current, lastSize - 1, last)
-        || !isPeer(currentSize - 1, current, lastSize - 1, last)) {
-      logger.trace("frame changed, we are ready to process first saved batch");
+        /*|| !isPeer(currentSize - 1, current, lastSize - 1, last)*/) {
+      logger.trace("partition changed, we are ready to process first saved batch");
       return true;
     } else {
-      logger.trace("frame didn't change, fetch next batch");
+      logger.trace("partition didn't change, fetch next batch");
       return false;
     }
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/3f0d9221/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
index 6686fbe..37529ff 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/MSortTemplate.java
@@ -47,6 +47,11 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
   private Queue<Integer> newRunStarts;
   private FragmentContext context;
 
+  /**
+   * This is only useful for debugging and/or unit testing. Controls the maximum size of
batches exposed to downstream
+   */
+  private int desiredRecordBatchCount;
+
   @Override
   public void setup(final FragmentContext context, final BufferAllocator allocator, final
SelectionVector4 vector4, final VectorContainer hyperBatch) throws SchemaChangeException{
     // we pass in the local hyperBatch since that is where we'll be reading data.
@@ -71,15 +76,13 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
     }
     final DrillBuf drillBuf = allocator.buffer(4 * totalCount);
 
-    // This is only useful for debugging: change the maximum size of batches exposed to downstream
-    // when we don't spill to disk
-    int MSORT_BATCH_MAXSIZE;
     try {
-      MSORT_BATCH_MAXSIZE = context.getConfig().getInt(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE);
+      desiredRecordBatchCount = context.getConfig().getInt(ExecConstants.EXTERNAL_SORT_MSORT_MAX_BATCHSIZE);
     } catch(ConfigException.Missing e) {
-      MSORT_BATCH_MAXSIZE = Character.MAX_VALUE;
+      // value not found, use default value instead
+      desiredRecordBatchCount = Character.MAX_VALUE;
     }
-    aux = new SelectionVector4(drillBuf, totalCount, MSORT_BATCH_MAXSIZE);
+    aux = new SelectionVector4(drillBuf, totalCount, desiredRecordBatchCount);
   }
 
   /**
@@ -150,11 +153,11 @@ public abstract class MSortTemplate implements MSorter, IndexedSortable{
       if (outIndex < vector4.getTotalCount()) {
         copyRun(outIndex, vector4.getTotalCount());
       }
-      final SelectionVector4 tmp = aux.createNewWrapperCurrent();
+      final SelectionVector4 tmp = aux.createNewWrapperCurrent(desiredRecordBatchCount);
       aux.clear();
-      aux = this.vector4.createNewWrapperCurrent();
+      aux = this.vector4.createNewWrapperCurrent(desiredRecordBatchCount);
       vector4.clear();
-      this.vector4 = tmp.createNewWrapperCurrent();
+      this.vector4 = tmp.createNewWrapperCurrent(desiredRecordBatchCount);
       tmp.clear();
       runStarts = newRunStarts;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/3f0d9221/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
index 69bc78f..8db0437 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector4.java
@@ -23,7 +23,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.record.DeadBuf;
 
 public class SelectionVector4 {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector4.class);
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector4.class);
 
   private ByteBuf data;
   private int recordCount;
@@ -67,13 +67,13 @@ public class SelectionVector4 {
 
   /**
    * Caution: This method shares the underlying buffer between this vector and the newly
created one.
+   * @param batchRecordCount this will be used when creating the new vector
    * @return Newly created single batch SelectionVector4.
-   * @throws SchemaChangeException
    */
-  public SelectionVector4 createNewWrapperCurrent() {
+  public SelectionVector4 createNewWrapperCurrent(int batchRecordCount) {
     try {
       data.retain();
-      SelectionVector4 sv4 = new SelectionVector4(data, recordCount, length);
+      SelectionVector4 sv4 = new SelectionVector4(data, recordCount, batchRecordCount);
       sv4.start = this.start;
       return sv4;
     } catch (SchemaChangeException e) {
@@ -81,6 +81,14 @@ public class SelectionVector4 {
     }
   }
 
+  /**
+   * Caution: This method shares the underlying buffer between this vector and the newly
created one.
+   * @return Newly created single batch SelectionVector4.
+   */
+  public SelectionVector4 createNewWrapperCurrent() {
+    return createNewWrapperCurrent(length);
+  }
+
   public boolean next() {
 //    logger.debug("Next called. Start: {}, Length: {}, recordCount: " + recordCount, start,
length);
 


Mime
View raw message