drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [8/8] drill git commit: DRILL-3053: add unchecked exception injection site in ChildAllocator
Date Thu, 14 May 2015 04:42:13 GMT
DRILL-3053: add unchecked exception injection site in ChildAllocator


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/583ca4a9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/583ca4a9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/583ca4a9

Branch: refs/heads/master
Commit: 583ca4a95df2c45b5ba20b517cb1aeed48c7548e
Parents: f63dac9
Author: adeneche <adeneche@gmail.com>
Authored: Wed May 13 09:45:23 2015 -0700
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Wed May 13 19:36:30 2015 -0700

----------------------------------------------------------------------
 .../exceptions/DrillRuntimeException.java       |  2 +-
 .../drill/common/exceptions/UserException.java  |  1 +
 .../drill/exec/memory/TopLevelAllocator.java    | 25 ++++--
 .../exec/testing/ExecutionControlsInjector.java | 18 +++++
 .../apache/drill/TestAllocationException.java   | 82 ++++++++++++++++++++
 5 files changed, 122 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/583ca4a9/common/src/main/java/org/apache/drill/common/exceptions/DrillRuntimeException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/DrillRuntimeException.java
b/common/src/main/java/org/apache/drill/common/exceptions/DrillRuntimeException.java
index abc7065..8786994 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/DrillRuntimeException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/DrillRuntimeException.java
@@ -18,7 +18,7 @@
 package org.apache.drill.common.exceptions;
 
 public class DrillRuntimeException extends RuntimeException {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuntimeException.class);
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuntimeException.class);
   private static final long serialVersionUID = -3796081521525479249L;
 
   public DrillRuntimeException() {

http://git-wip-us.apache.org/repos/asf/drill/blob/583ca4a9/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/UserException.java b/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
index a67cb3f..6f28a2b 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/UserException.java
@@ -37,6 +37,7 @@ import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
  * @see org.apache.drill.exec.proto.UserBitShared.DrillPBError.ErrorType
  */
 public class UserException extends DrillRuntimeException {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(UserException.class);
 
   public static final String MEMORY_ERROR_MSG = "One or more nodes ran out of memory while
executing the query.";
 

http://git-wip-us.apache.org/repos/asf/drill/blob/583ca4a9/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
index 9670c7e..f6a37e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/memory/TopLevelAllocator.java
@@ -31,11 +31,15 @@ import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.testing.ExecutionControlsInjector;
 import org.apache.drill.exec.util.AssertionUtil;
 import org.apache.drill.exec.util.Pointer;
 
 public class TopLevelAllocator implements BufferAllocator {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopLevelAllocator.class);
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TopLevelAllocator.class);
+
+  private static final ExecutionControlsInjector injector = ExecutionControlsInjector.getInjector(TopLevelAllocator.class);
+  public static final String CHILD_BUFFER_INJECTION_SITE = "child.buffer";
 
   public static long MAXIMUM_DIRECT_MEMORY;
 
@@ -127,7 +131,7 @@ public class TopLevelAllocator implements BufferAllocator {
               .format(
                   "You attempted to create a new child allocator with initial reservation
%d but only %d bytes of memory were available.",
                   initialReservation, acct.getCapacity() - acct.getAllocation()));
-    };
+    }
     logger.debug("New child allocator with initial reservation {}", initialReservation);
     ChildAllocator allocator = new ChildAllocator(context, acct, maximumReservation, initialReservation,
childrenMap, applyFragmentLimit);
     if(ENABLE_ACCOUNTING){
@@ -180,7 +184,7 @@ public class TopLevelAllocator implements BufferAllocator {
 
 
 
-  private class ChildAllocator implements BufferAllocator{
+  private class ChildAllocator implements BufferAllocator {
     private final DrillBuf empty;
     private Accountor childAcct;
     private Map<ChildAllocator, StackTraceElement[]> children = new HashMap<>();
@@ -221,13 +225,24 @@ public class TopLevelAllocator implements BufferAllocator {
 
     @Override
     public DrillBuf buffer(int size, int max) {
+      if (ENABLE_ACCOUNTING) {
+        try {
+          injector.injectUnchecked(fragmentContext, CHILD_BUFFER_INJECTION_SITE);
+        } catch (NullPointerException e) {
+          // This is an unusual way to use exception injection. If we inject a NullPointerException
into this site
+          // it will actually cause this method to return null, simulating a "normal" failure
to allocate memory
+          // this can be useful to check if the caller will properly handle nulls
+          return null;
+        }
+      }
+
       if (size == 0) {
         return empty;
       }
       if(!childAcct.reserve(size)) {
         logger.warn("Unable to allocate buffer of size {} due to memory limit. Current allocation:
{}", size, getAllocatedMemory(), new Exception());
         return null;
-      };
+      }
 
       UnsafeDirectLittleEndian buffer = innerAllocator.directBuffer(size, max);
       DrillBuf wrapped = new DrillBuf(this, childAcct, buffer);
@@ -253,7 +268,7 @@ public class TopLevelAllocator implements BufferAllocator {
                 .format(
                     "You attempted to create a new child allocator with initial reservation
%d but only %d bytes of memory were available.",
                     initialReservation, childAcct.getAvailable()));
-      };
+      }
       logger.debug("New child allocator with initial reservation {}", initialReservation);
       ChildAllocator newChildAllocator = new ChildAllocator(context, childAcct, maximumReservation,
initialReservation, null, applyFragmentLimit);
       this.children.put(newChildAllocator, Thread.currentThread().getStackTrace());

http://git-wip-us.apache.org/repos/asf/drill/blob/583ca4a9/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControlsInjector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControlsInjector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControlsInjector.java
index 387d300..e3a4ba6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControlsInjector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControlsInjector.java
@@ -17,6 +17,7 @@
  */
 package org.apache.drill.exec.testing;
 
+import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.util.AssertionUtil;
 import org.slf4j.Logger;
 
@@ -83,6 +84,23 @@ public class ExecutionControlsInjector {
   }
 
   /**
+   * Inject (throw) an unchecked exception at this point, if the fragmentContext is not null,
+   * an injection is specified, and it is time for it to be thrown.
+   * <p/>
+   * <p>Implementors use this in their code at a site where they want to simulate an
exception
+   * during testing.
+   *
+   * @param fragmentContext   fragmentContext used to retrieve the controls, can be null
+   * @param desc              the site description
+   *                          throws the exception specified by the injection, if it is time
+   */
+  public void injectUnchecked(final FragmentContext fragmentContext, final String desc) {
+    if (fragmentContext != null) {
+      injectUnchecked(fragmentContext.getExecutionControls(), desc);
+    }
+  }
+
+  /**
    * Inject (throw) a checked exception at this point, if an injection is specified, and
it is time
    * for it to be thrown.
    * <p/>

http://git-wip-us.apache.org/repos/asf/drill/blob/583ca4a9/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
new file mode 100644
index 0000000..051ad4e
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestAllocationException.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
+import org.apache.drill.exec.testing.ControlsInjectionUtil;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import org.junit.Test;
+
+/**
+ * Run several tpch queries and inject an OutOfMemoryException in ScanBatch that will cause
an OUT_OF_MEMORY outcome to
+ * be propagated downstream. Make sure the proper "memory error" message is sent to the client.
+ */
+public class TestAllocationException extends BaseTestQuery {
+
+  private static final String SINGLE_MODE = "ALTER SESSION SET `planner.disable_exchanges`
= true";
+
+  private void testWithException(final String fileName) throws Exception {
+    testWithException(fileName, OutOfMemoryRuntimeException.class);
+  }
+
+  private void testWithException(final String fileName, Class<? extends Throwable>
exceptionClass) throws Exception{
+    test(SINGLE_MODE);
+
+    CoordinationProtos.DrillbitEndpoint endpoint = bits[0].getContext().getEndpoint();
+
+    String controlsString = "{\"injections\":[{"
+      + "\"address\":\"" + endpoint.getAddress() + "\","
+      + "\"port\":\"" + endpoint.getUserPort() + "\","
+      + "\"type\":\"exception\","
+      + "\"siteClass\":\"" + TopLevelAllocator.class.getName() + "\","
+      + "\"desc\":\"" + TopLevelAllocator.CHILD_BUFFER_INJECTION_SITE + "\","
+      + "\"nSkip\":200,"
+      + "\"nFire\":1,"
+      + "\"exceptionClass\":\"" + exceptionClass.getName() + "\""
+      + "}]}";
+    ControlsInjectionUtil.setControls(client, controlsString);
+
+    String query = getFile(fileName);
+
+    try {
+      test(query);
+      fail("The query should have failed!");
+    } catch(UserException uex) {
+      DrillPBError error = uex.getOrCreatePBError(false);
+      assertEquals(DrillPBError.ErrorType.RESOURCE, error.getErrorType());
+      assertTrue("Error message isn't related to memory error",
+        uex.getMessage().contains(UserException.MEMORY_ERROR_MSG));
+    }
+  }
+
+  @Test
+  public void testWithNull() throws Exception{
+    testWithException("queries/tpch/01.sql");
+  }
+
+  @Test
+  public void testWithOOM() throws Exception{
+    testWithException("queries/tpch/03.sql", NullPointerException.class);
+  }
+}


Mime
View raw message