drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From adene...@apache.org
Subject [1/5] drill git commit: DRILL-3555: Changing defaults for planner.memory.max_query_memory_per_node causes queries with window function to fail
Date Tue, 08 Sep 2015 23:25:21 GMT
Repository: drill
Updated Branches:
  refs/heads/master 38e660e13 -> 2a1918471


DRILL-3555: Changing defaults for planner.memory.max_query_memory_per_node causes queries
with window function to fail

this closes #137


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7a864a43
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7a864a43
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7a864a43

Branch: refs/heads/master
Commit: 7a864a43919a46b16a2f59964e7288e30d96eb2c
Parents: 38e660e
Author: adeneche <adeneche@gmail.com>
Authored: Thu Aug 27 12:57:23 2015 -0700
Committer: adeneche <adeneche@gmail.com>
Committed: Tue Sep 8 15:35:09 2015 -0700

----------------------------------------------------------------------
 .../exec/physical/impl/TopN/TopNBatch.java      |  3 +--
 .../OrderedPartitionRecordBatch.java            |  5 ++---
 .../exec/physical/impl/sort/SortBatch.java      |  6 ++----
 .../impl/sort/SortRecordBatchBuilder.java       | 20 ++++----------------
 .../physical/impl/xsort/ExternalSortBatch.java  |  3 +--
 5 files changed, 10 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
index 10f1d7f..3ef6bfe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/TopN/TopNBatch.java
@@ -67,7 +67,6 @@ import com.sun.codemodel.JExpr;
 public class TopNBatch extends AbstractRecordBatch<TopN> {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopNBatch.class);
 
-  private static final long MAX_SORT_BYTES = 1L * 1024 * 1024 * 1024;
   private  final int batchPurgeThreshold;
 
   public final MappingSet MAIN_MAPPING = new MappingSet( (String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP,
ClassGenerator.DEFAULT_SCALAR_MAP);
@@ -285,7 +284,7 @@ public class TopNBatch extends AbstractRecordBatch<TopN> {
       }
       copier.setupRemover(context, batch, newBatch);
     }
-    SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator(),
MAX_SORT_BYTES);
+    SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator());
     try {
       do {
         int count = selectionVector4.getCount();

http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index 1286fe1..0050b45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -109,7 +109,6 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
   public final MappingSet partitionMapping = new MappingSet("partitionIndex", null, "partitionVectors",
null,
       ClassGenerator.DEFAULT_CONSTANT_MAP, ClassGenerator.DEFAULT_SCALAR_MAP);
 
-  private static long MAX_SORT_BYTES = 8l * 1024 * 1024 * 1024;
   private final int recordsToSample; // How many records must be received before analyzing
   private final int samplingFactor; // Will collect samplingFactor * number of partitions
to send to distributed cache
   private final float completionFactor; // What fraction of fragments must be completed before
attempting to build
@@ -171,7 +170,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
 
     // Start collecting batches until recordsToSample records have been collected
 
-    SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator(),
MAX_SORT_BYTES);
+    SortRecordBatchBuilder builder = new SortRecordBatchBuilder(oContext.getAllocator());
     WritableBatch batch = null;
     CachedVectorContainer sampleToSave = null;
     VectorContainer containerToCache = new VectorContainer();
@@ -344,7 +343,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
 
     // Get all samples from distributed map
 
-    SortRecordBatchBuilder containerBuilder = new SortRecordBatchBuilder(context.getAllocator(),
MAX_SORT_BYTES);
+    SortRecordBatchBuilder containerBuilder = new SortRecordBatchBuilder(context.getAllocator());
     final VectorContainer allSamplesContainer = new VectorContainer();
     final VectorContainer candidatePartitionTable = new VectorContainer();
     CachedVectorContainer wrap = null;

http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index dea6ba8..407f05d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -54,8 +54,6 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
   public final MappingSet leftMapping = new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_CONSTANT_MAP,
ClassGenerator.DEFAULT_SCALAR_MAP);
   public final MappingSet rightMapping = new MappingSet("rightIndex", null, ClassGenerator.DEFAULT_CONSTANT_MAP,
ClassGenerator.DEFAULT_SCALAR_MAP);
 
-  private static long MAX_SORT_BYTES = 8l * 1024 * 1024 * 1024;
-
   private final RecordBatch incoming;
   private final SortRecordBatchBuilder builder;
   private Sorter sorter;
@@ -64,7 +62,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
   public SortBatch(Sort popConfig, FragmentContext context, RecordBatch incoming) throws
OutOfMemoryException {
     super(popConfig, context);
     this.incoming = incoming;
-    this.builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
+    this.builder = new SortRecordBatchBuilder(oContext.getAllocator());
   }
 
   @Override
@@ -122,7 +120,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
         case OK:
           if (!builder.add(incoming)) {
             throw new UnsupportedOperationException("Sort doesn't currently support doing
an external sort.");
-          };
+          }
           break;
         default:
           throw new UnsupportedOperationException();

http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
index 00f1992..edbb62f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortRecordBatchBuilder.java
@@ -45,15 +45,12 @@ public class SortRecordBatchBuilder implements AutoCloseable {
   private final ArrayListMultimap<BatchSchema, RecordBatchData> batches = ArrayListMultimap.create();
 
   private int recordCount;
-  private long runningBytes;
   private long runningBatches;
-  private final long maxBytes;
   private SelectionVector4 sv4;
   final PreAllocator svAllocator;
   private boolean svAllocatorUsed = false;
 
-  public SortRecordBatchBuilder(BufferAllocator a, long maxBytes) {
-    this.maxBytes = maxBytes;
+  public SortRecordBatchBuilder(BufferAllocator a) {
     this.svAllocator = a.getNewPreAllocator();
   }
 
@@ -84,10 +81,7 @@ public class SortRecordBatchBuilder implements AutoCloseable {
     if (batchBytes == 0 && batches.size() > 0) {
       return true;
     }
-    if (batchBytes + runningBytes > maxBytes) {
-      return false; // enough data memory.
-    }
-    if (runningBatches+1 > Character.MAX_VALUE) {
+    if (runningBatches >= Character.MAX_VALUE) {
       return false; // allowed in batch.
     }
     if (!svAllocator.preAllocate(batch.getRecordCount()*4)) {
@@ -96,7 +90,7 @@ public class SortRecordBatchBuilder implements AutoCloseable {
 
 
     RecordBatchData bd = new RecordBatchData(batch);
-    runningBytes += batchBytes;
+    runningBatches++;
     batches.put(batch.getSchema(), bd);
     recordCount += bd.getRecordCount();
     return true;
@@ -108,12 +102,6 @@ public class SortRecordBatchBuilder implements AutoCloseable {
       return;
     }
 
-    if(batchBytes + runningBytes > maxBytes) {
-      final String errMsg = String.format("Adding this batch causes the total size to exceed
max allowed size. " +
-          "Current runningBytes %d, Incoming batchBytes %d. maxBytes %d", runningBytes, batchBytes,
maxBytes);
-      logger.error(errMsg);
-      throw new DrillRuntimeException(errMsg);
-    }
     if(runningBatches >= Character.MAX_VALUE) {
       final String errMsg = String.format("Tried to add more than %d number of batches.",
Character.MAX_VALUE);
       logger.error(errMsg);
@@ -135,7 +123,7 @@ public class SortRecordBatchBuilder implements AutoCloseable {
       }
       return;
     }
-    runningBytes += batchBytes;
+    runningBatches++;
     batches.put(rbd.getContainer().getSchema(), rbd);
     recordCount += rbd.getRecordCount();
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/7a864a43/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index d9866bb..ed32f43 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -83,7 +83,6 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort>
{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExternalSortBatch.class);
   private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(ExternalSortBatch.class);
 
-  private static final long MAX_SORT_BYTES = 1L * 1024 * 1024 * 1024;
   private static final GeneratorMapping COPIER_MAPPING = new GeneratorMapping("doSetup",
"doCopy", null, null);
   private static final MappingSet MAIN_MAPPING = new MappingSet( (String) null, null, ClassGenerator.DEFAULT_SCALAR_MAP,
ClassGenerator.DEFAULT_SCALAR_MAP);
   private static final MappingSet LEFT_MAPPING = new MappingSet("leftIndex", null, ClassGenerator.DEFAULT_SCALAR_MAP,
ClassGenerator.DEFAULT_SCALAR_MAP);
@@ -390,7 +389,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort>
{
           builder.clear();
           builder.close();
         }
-        builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);
+        builder = new SortRecordBatchBuilder(oContext.getAllocator());
 
         for (BatchGroup group : batchGroups) {
           RecordBatchData rbd = new RecordBatchData(group.getContainer());


Mime
View raw message