drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject drill git commit: DRILL-3307: Query with window function runs out of memory
Date Sat, 27 Jun 2015 21:03:02 GMT
Repository: drill
Updated Branches:
  refs/heads/master 8d7458958 -> e85cdc54c


DRILL-3307: Query with window function runs out of memory


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e85cdc54
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e85cdc54
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e85cdc54

Branch: refs/heads/master
Commit: e85cdc54cc21e0be2bf7169868ec00b8ef7c1f22
Parents: 8d74589
Author: Steven Phillips <smp@apache.org>
Authored: Fri Jun 26 16:15:21 2015 -0700
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Sat Jun 27 11:58:46 2015 -0700

----------------------------------------------------------------------
 .../exec/physical/impl/xsort/ExternalSortBatch.java      | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/e85cdc54/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 02a1c08..6da5582 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -406,7 +406,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort>
{
         batchGroups.addAll(spilledBatchGroups);
         logger.warn("Starting to merge. {} batch groups. Current allocated memory: {}", batchGroups.size(),
oContext.getAllocator().getAllocatedMemory());
         VectorContainer hyperBatch = constructHyperBatch(batchGroups);
-        createCopier(hyperBatch, batchGroups, container);
+        createCopier(hyperBatch, batchGroups, container, false);
 
         int estimatedRecordSize = 0;
         for (VectorWrapper w : batchGroups.get(0)) {
@@ -474,7 +474,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort>
{
     }
     int targetRecordCount = Math.max(1, 250 * 1000 / estimatedRecordSize);
     VectorContainer hyperBatch = constructHyperBatch(batchGroupList);
-    createCopier(hyperBatch, batchGroupList, outputContainer);
+    createCopier(hyperBatch, batchGroupList, outputContainer, true);
 
     int count = copier.next(targetRecordCount);
     assert count > 0;
@@ -673,7 +673,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort>
{
     g.getEvalBlock()._return(JExpr.lit(0));
   }
 
-  private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList,
VectorContainer outputContainer) throws SchemaChangeException {
+  private void createCopier(VectorAccessible batch, List<BatchGroup> batchGroupList,
VectorContainer outputContainer, boolean spilling) throws SchemaChangeException {
     try {
       if (copier == null) {
         CodeGenerator<PriorityQueueCopier> cg = CodeGenerator.get(PriorityQueueCopier.TEMPLATE_DEFINITION,
context.getFunctionRegistry());
@@ -689,11 +689,12 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort>
{
         copier.cleanup();
       }
 
+      BufferAllocator allocator = spilling ? copierAllocator : oContext.getAllocator();
       for (VectorWrapper<?> i : batch) {
-        ValueVector v = TypeHelper.getNewVector(i.getField(), copierAllocator);
+        ValueVector v = TypeHelper.getNewVector(i.getField(), allocator);
         outputContainer.add(v);
       }
-      copier.setup(context, copierAllocator, batch, batchGroupList, outputContainer);
+      copier.setup(context, allocator, batch, batchGroupList, outputContainer);
     } catch (ClassTransformationException e) {
       throw new RuntimeException(e);
     } catch (IOException e) {


Mime
View raw message