drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From adene...@apache.org
Subject [1/2] drill git commit: DRILL-3927: use OutOfMemoryException in more places Some code cleanup required for the upcoming introduction of the rewritten direct memory allocator. Chiefly the introduction of OutOfMemoryException handling in a few spots, the u
Date Thu, 15 Oct 2015 13:13:11 GMT
Repository: drill
Updated Branches:
  refs/heads/master 202a8e154 -> 12fb9ac11


DRILL-3927: use OutOfMemoryException in more places
Some code cleanup required for the upcoming introduction of the rewritten
direct memory allocator. Chiefly the introduction of OutOfMemoryException
handling in a few spots, the use of DrillAutoCloseables for some allocator
close() calls, and some other minor cleanup.
- removed the exclusion of exec/work from the jdbc-all jar, because it now
  depends on OutOfMemoryException, and that is derived from
  FragmentSetupException and Foreman Exception

this closes #199


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8c098355
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8c098355
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8c098355

Branch: refs/heads/master
Commit: 8c098355f2cb9c0139a039d17835a9c253962504
Parents: 202a8e1
Author: Chris Westin <cwestin@yahoo.com>
Authored: Fri Oct 9 14:08:09 2015 -0700
Committer: adeneche <adeneche@gmail.com>
Committed: Wed Oct 14 18:01:23 2015 -0700

----------------------------------------------------------------------
 .../apache/drill/common/DeferredException.java  | 40 +++++++++++++++-
 .../drill/common/DrillAutoCloseables.java       | 50 ++++++++++++++++++++
 .../apache/drill/common/config/DrillConfig.java | 46 ++++++------------
 .../apache/drill/exec/client/DrillClient.java   | 45 ++++++++++++------
 .../exec/client/PrintingResultsListener.java    | 24 +++++-----
 .../drill/exec/physical/impl/BaseRootExec.java  | 43 +++++++++++++++--
 .../drill/exec/server/BootStrapContext.java     | 16 +++----
 .../exec/work/foreman/ForemanException.java     | 24 +++-------
 .../java/org/apache/drill/QueryTestUtil.java    |  3 +-
 .../drill/TestTpchDistributedConcurrent.java    | 10 ++++
 .../fn/impl/TestByteComparisonFunctions.java    |  3 +-
 .../apache/drill/exec/memory/TestEndianess.java |  3 +-
 .../drill/exec/server/DrillClientFactory.java   | 15 +++---
 exec/jdbc-all/pom.xml                           |  9 ++--
 .../drill/jdbc/impl/DrillConnectionImpl.java    | 34 ++++++++-----
 15 files changed, 246 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/common/src/main/java/org/apache/drill/common/DeferredException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/DeferredException.java b/common/src/main/java/org/apache/drill/common/DeferredException.java
index c7111a9..721a217 100644
--- a/common/src/main/java/org/apache/drill/common/DeferredException.java
+++ b/common/src/main/java/org/apache/drill/common/DeferredException.java
@@ -18,6 +18,7 @@
 package org.apache.drill.common;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
 
 /**
  * Collects one or more exceptions that may occur, using
@@ -30,8 +31,35 @@ import com.google.common.base.Preconditions;
  * <p>This class is thread safe.
  */
 public class DeferredException implements AutoCloseable {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DeferredException.class);
+
   private Exception exception = null;
   private boolean isClosed = false;
+  private final Supplier<Exception> exceptionSupplier;
+
+  /**
+   * Constructor.
+   */
+  public DeferredException() {
+    this(null);
+  }
+
+  /**
+   * Constructor. This constructor accepts a Supplier that can be used
+   * to create the root exception iff any other exceptions are added. For
+   * example, in a series of resources closures in a close(), if any of
+   * the individual closures fails, the root exception should come from
+   * the current class, not from the first subordinate close() to fail.
+   * This can be used to provide an exception in that case which will be
+   * the root exception; the subordinate failed close() will be added to
+   * that exception as a suppressed exception.
+   *
+   * @param exceptionSupplier lazily supplies what will be the root exception
+   *   if any exceptions are added
+   */
+  public DeferredException(Supplier<Exception> exceptionSupplier) {
+    this.exceptionSupplier = exceptionSupplier;
+  }
 
   /**
    * Add an exception. If this is the first exception added, it will be the one
@@ -47,7 +75,15 @@ public class DeferredException implements AutoCloseable {
       Preconditions.checkState(!isClosed);
 
       if (this.exception == null) {
-        this.exception = exception;
+        if (exceptionSupplier == null) {
+          this.exception = exception;
+        } else {
+          this.exception = exceptionSupplier.get();
+          if (this.exception == null) {
+            this.exception = new RuntimeException("Missing root exception");
+          }
+          this.exception.addSuppressed(exception);
+        }
       } else {
         this.exception.addSuppressed(exception);
       }
@@ -93,7 +129,7 @@ public class DeferredException implements AutoCloseable {
    *
    * @throws Exception
    */
-  public synchronized void throwAndClear() throws Exception{
+  public synchronized void throwAndClear() throws Exception {
     final Exception e = getAndClear();
     if (e != null) {
       throw e;

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/common/src/main/java/org/apache/drill/common/DrillAutoCloseables.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/DrillAutoCloseables.java b/common/src/main/java/org/apache/drill/common/DrillAutoCloseables.java
new file mode 100644
index 0000000..11fb9a8
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/DrillAutoCloseables.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+/**
+ * Provides functionality comparable to Guava's Closeables for AutoCloseables.
+ */
+public class DrillAutoCloseables {
+  /**
+   * Constructor. Prevents construction for class of static utilities.
+   */
+  private DrillAutoCloseables() {
+  }
+
+  /**
+   * close() an {@see java.lang.AutoCloseable} without throwing a (checked)
+   * {@see java.lang.Exception}. This wraps the close() call with a
+   * try-catch that will rethrow an Exception wrapped with a
+   * {@see java.lang.RuntimeException}, providing a way to call close()
+   * without having to do the try-catch everywhere or propagate the Exception.
+   *
+   * @param closeable the AutoCloseable to close; may be null
+   * @throws RuntimeException if an Exception occurs; the Exception is
+   *   wrapped by the RuntimeException
+   */
+  public static void closeNoChecked(final AutoCloseable autoCloseable) {
+    if (autoCloseable != null) {
+      try {
+        autoCloseable.close();
+      } catch(final Exception e) {
+        throw new RuntimeException("Exception while closing", e);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index f6d10ea..d747b67 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -23,8 +23,6 @@ import java.net.URL;
 import java.util.Collection;
 import java.util.List;
 import java.util.Properties;
-import java.util.Queue;
-import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.apache.drill.common.exceptions.DrillConfigurationException;
 import org.apache.drill.common.expression.LogicalExpression;
@@ -58,9 +56,6 @@ public final class DrillConfig extends NestedConfig{
   @SuppressWarnings("restriction")
   private static final long MAX_DIRECT_MEMORY = sun.misc.VM.maxDirectMemory();
 
-  @SuppressWarnings("unchecked")
-  private volatile List<Queue<Object>> sinkQueues = new CopyOnWriteArrayList<Queue<Object>>(new
Queue[1]);
-
   @VisibleForTesting
   public DrillConfig(Config config, boolean enableServerConfigs) {
     super(config);
@@ -226,49 +221,41 @@ public final class DrillConfig extends NestedConfig{
     return new DrillConfig(effectiveConfig.resolve(), enableServerConfigs);
   }
 
-  public <T> Class<T> getClassAt(String location, Class<T> clazz) throws
DrillConfigurationException{
-    String className = this.getString(location);
+  public <T> Class<T> getClassAt(String location, Class<T> clazz) throws
DrillConfigurationException {
+    final String className = getString(location);
     if (className == null) {
-      throw new DrillConfigurationException(String.format("No class defined at location '%s'.
 Expected a definition of the class []", location, clazz.getCanonicalName()));
+      throw new DrillConfigurationException(String.format(
+          "No class defined at location '%s'. Expected a definition of the class []",
+          location, clazz.getCanonicalName()));
     }
-    try{
-      Class<?> c = Class.forName(className);
+
+    try {
+      final Class<?> c = Class.forName(className);
       if (clazz.isAssignableFrom(c)) {
-        @SuppressWarnings("unchecked") Class<T> t = (Class<T>) c;
+        @SuppressWarnings("unchecked")
+        final Class<T> t = (Class<T>) c;
         return t;
-      } else {
-        throw new DrillConfigurationException(String.format("The class [%s] listed at location
'%s' should be of type [%s].  It isn't.", className, location, clazz.getCanonicalName()));
       }
+
+      throw new DrillConfigurationException(String.format("The class [%s] listed at location
'%s' should be of type [%s].  It isn't.", className, location, clazz.getCanonicalName()));
     } catch (Exception ex) {
       if (ex instanceof DrillConfigurationException) {
         throw (DrillConfigurationException) ex;
       }
       throw new DrillConfigurationException(String.format("Failure while initializing class
[%s] described at configuration value '%s'.", className, location), ex);
     }
-
   }
 
   public <T> T getInstanceOf(String location, Class<T> clazz) throws DrillConfigurationException{
-    Class<T> c = getClassAt(location, clazz);
+    final Class<T> c = getClassAt(location, clazz);
     try {
-      T t = c.newInstance();
+      final T t = c.newInstance();
       return t;
     } catch (Exception ex) {
       throw new DrillConfigurationException(String.format("Failure while instantiating class
[%s] located at '%s.", clazz.getCanonicalName(), location), ex);
     }
   }
 
-  public void setSinkQueues(int number, Queue<Object> queue) {
-    sinkQueues.set(number, queue);
-  }
-
-  public Queue<Object> getQueue(int number) {
-    if (sinkQueues.size() <= number || number < 0 || sinkQueues == null) {
-      throw new IllegalArgumentException(String.format("Queue %d is not available.", number));
-    }
-    return sinkQueues.get(number);
-  }
-
   public ObjectMapper getMapper() {
     return mapper;
   }
@@ -278,11 +265,6 @@ public final class DrillConfig extends NestedConfig{
     return this.root().render();
   }
 
-  public static void main(String[] args)  throws Exception {
-    //"-XX:MaxDirectMemorySize"
-    DrillConfig config = DrillConfig.create();
-  }
-
   public static long getMaxDirectMemory() {
     return MAX_DIRECT_MEMORY;
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 911828a..bf6c8be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -31,12 +31,14 @@ import java.util.List;
 import java.util.Properties;
 import java.util.Vector;
 
+import org.apache.drill.common.DrillAutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.coord.ClusterCoordinator;
 import org.apache.drill.exec.coord.zk.ZKClusterCoordinator;
 import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryException;
 import org.apache.drill.exec.memory.RootAllocatorFactory;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.proto.GeneralRPCProtos.Ack;
@@ -71,7 +73,7 @@ import com.google.common.util.concurrent.SettableFuture;
 public class DrillClient implements Closeable, ConnectionThrottle {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillClient.class);
 
-  DrillConfig config;
+  private final DrillConfig config;
   private UserClient client;
   private UserProperties props = null;
   private volatile ClusterCoordinator clusterCoordinator;
@@ -85,35 +87,39 @@ public class DrillClient implements Closeable, ConnectionThrottle {
   private final boolean isDirectConnection; // true if the connection bypasses zookeeper
and connects directly to a drillbit
   private EventLoopGroup eventLoopGroup;
 
-  public DrillClient() {
+  public DrillClient() throws OutOfMemoryException {
     this(DrillConfig.create(), false);
   }
 
-  public DrillClient(boolean isDirect) {
+  public DrillClient(boolean isDirect) throws OutOfMemoryException {
     this(DrillConfig.create(), isDirect);
   }
 
-  public DrillClient(String fileName) {
+  public DrillClient(String fileName) throws OutOfMemoryException {
     this(DrillConfig.create(fileName), false);
   }
 
-  public DrillClient(DrillConfig config) {
+  public DrillClient(DrillConfig config) throws OutOfMemoryException {
     this(config, null, false);
   }
 
-  public DrillClient(DrillConfig config, boolean isDirect) {
+  public DrillClient(DrillConfig config, boolean isDirect)
+      throws OutOfMemoryException {
     this(config, null, isDirect);
   }
 
-  public DrillClient(DrillConfig config, ClusterCoordinator coordinator) {
+  public DrillClient(DrillConfig config, ClusterCoordinator coordinator)
+    throws OutOfMemoryException {
     this(config, coordinator, null, false);
   }
 
-  public DrillClient(DrillConfig config, ClusterCoordinator coordinator, boolean isDirect)
{
+  public DrillClient(DrillConfig config, ClusterCoordinator coordinator, boolean isDirect)
+    throws OutOfMemoryException {
     this(config, coordinator, null, isDirect);
   }
 
-  public DrillClient(DrillConfig config, ClusterCoordinator coordinator, BufferAllocator
allocator) {
+  public DrillClient(DrillConfig config, ClusterCoordinator coordinator, BufferAllocator
allocator)
+      throws OutOfMemoryException {
     this(config, coordinator, allocator, false);
   }
 
@@ -258,13 +264,16 @@ public class DrillClient implements Closeable, ConnectionThrottle {
       this.client.close();
     }
     if (this.ownsAllocator && allocator != null) {
-      allocator.close();
+      DrillAutoCloseables.closeNoChecked(allocator);
     }
     if (ownsZkConnection) {
-      try {
-        this.clusterCoordinator.close();
-      } catch (IOException e) {
-        logger.warn("Error while closing Cluster Coordinator.", e);
+      if (clusterCoordinator != null) {
+        try {
+          clusterCoordinator.close();
+          clusterCoordinator = null;
+        } catch (IOException e) {
+          logger.warn("Error while closing Cluster Coordinator.", e);
+        }
       }
     }
     if (eventLoopGroup != null) {
@@ -383,6 +392,14 @@ public class DrillClient implements Closeable, ConnectionThrottle {
       try {
         return future.get();
       } catch (Throwable t) {
+        /*
+         * Since we're not going to return the result to the caller
+         * to clean up, we have to do it.
+         */
+        for(final QueryDataBatch queryDataBatch : results) {
+          queryDataBatch.release();
+        }
+
         throw RpcException.mapException(t);
       }
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
index 212e8b0..61d9ed6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
@@ -23,6 +23,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.drill.common.DrillAutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.client.QuerySubmitter.Format;
@@ -31,6 +32,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.memory.RootAllocatorFactory;
 import org.apache.drill.exec.proto.UserBitShared.QueryData;
 import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserBitShared.QueryData;
 import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
 import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.rpc.user.ConnectionThrottle;
@@ -42,16 +44,15 @@ import com.google.common.base.Stopwatch;
 
 public class PrintingResultsListener implements UserResultsListener {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PrintingResultsListener.class);
-
-  AtomicInteger count = new AtomicInteger();
-  private CountDownLatch latch = new CountDownLatch(1);
-  RecordBatchLoader loader;
-  Format format;
-  int    columnWidth;
-  BufferAllocator allocator;
-  volatile UserException exception;
-  QueryId queryId;
-  Stopwatch w = new Stopwatch();
+  private final AtomicInteger count = new AtomicInteger();
+  private final CountDownLatch latch = new CountDownLatch(1);
+  private RecordBatchLoader loader;
+  private Format format;
+  private final int    columnWidth;
+  private final BufferAllocator allocator;
+  private volatile UserException exception;
+  private QueryId queryId;
+  private final Stopwatch w = new Stopwatch();
 
   public PrintingResultsListener(DrillConfig config, Format format, int columnWidth) {
     this.allocator = RootAllocatorFactory.newRoot(config);
@@ -70,7 +71,7 @@ public class PrintingResultsListener implements UserResultsListener {
 
   @Override
   public void queryCompleted(QueryState state) {
-    allocator.close();
+    DrillAutoCloseables.closeNoChecked(allocator);
     latch.countDown();
     System.out.println("Total rows returned : " + count.get() + ".  Returned in " + w.elapsed(TimeUnit.MILLISECONDS)
         + "ms.");
@@ -125,5 +126,4 @@ public class PrintingResultsListener implements UserResultsListener {
     w.start();
     this.queryId = queryId;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
index accce43..9301650 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/BaseRootExec.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.physical.impl;
 
 import java.util.List;
 
+import org.apache.drill.common.DeferredException;
 import org.apache.drill.exec.memory.OutOfMemoryException;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.ops.OpProfileDef;
@@ -30,6 +31,8 @@ import org.apache.drill.exec.record.CloseableRecordBatch;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.record.RecordBatch.IterOutcome;
 
+import com.google.common.base.Supplier;
+
 public abstract class BaseRootExec implements RootExec {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseRootExec.class);
 
@@ -59,6 +62,24 @@ public abstract class BaseRootExec implements RootExec {
 
   void setOperators(List<CloseableRecordBatch> operators) {
     this.operators = operators;
+
+    if (logger.isDebugEnabled()) {
+      final StringBuilder sb = new StringBuilder();
+      sb.append("BaseRootExec(");
+      sb.append(Integer.toString(System.identityHashCode(this)));
+      sb.append(") operators: ");
+      for(final CloseableRecordBatch crb : operators) {
+        sb.append(crb.getClass().getName());
+        sb.append(' ');
+        sb.append(Integer.toString(System.identityHashCode(crb)));
+        sb.append(", ");
+      }
+
+      // Cut off the last trailing comma and space
+      sb.setLength(sb.length() - 2);
+
+      logger.debug(sb.toString());
+    }
   }
 
   @Override
@@ -117,13 +138,25 @@ public abstract class BaseRootExec implements RootExec {
 
     // close all operators.
     if (operators != null) {
-      for (CloseableRecordBatch b : operators) {
-        try {
-          b.close();
-        } catch (Exception e) {
-          fragmentContext.fail(e);
+      final DeferredException df = new DeferredException(new Supplier<Exception>()
{
+        @Override
+        public Exception get() {
+          return new RuntimeException("Error closing operators");
+        }
+      });
+
+      for (final CloseableRecordBatch crb : operators) {
+        df.suppressingClose(crb);
+        if (logger.isDebugEnabled()) {
+          logger.debug(String.format("closed operator %d", System.identityHashCode(crb)));
         }
       }
+
+      try {
+        df.close();
+      } catch (Exception e) {
+        fragmentContext.fail(e);
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
index 4530dba..903402c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/BootStrapContext.java
@@ -19,8 +19,7 @@ package org.apache.drill.exec.server;
 
 import io.netty.channel.EventLoopGroup;
 
-import java.io.Closeable;
-
+import org.apache.drill.common.DrillAutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.memory.BufferAllocator;
@@ -30,8 +29,7 @@ import org.apache.drill.exec.rpc.TransportCheck;
 
 import com.codahale.metrics.MetricRegistry;
 
-
-public class BootStrapContext implements Closeable {
+public class BootStrapContext implements AutoCloseable {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BootStrapContext.class);
 
   private final DrillConfig config;
@@ -42,10 +40,10 @@ public class BootStrapContext implements Closeable {
 
   public BootStrapContext(DrillConfig config) {
     this.config = config;
-    this.loop = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS),
"BitServer-");
-    this.loop2 = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS),
"BitClient-");
-    this.metrics = DrillMetrics.getInstance();
-    this.allocator = RootAllocatorFactory.newRoot(config);
+    loop = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS),
"BitServer-");
+    loop2 = TransportCheck.createEventLoopGroup(config.getInt(ExecConstants.BIT_SERVER_RPC_THREADS),
"BitClient-");
+    metrics = DrillMetrics.getInstance();
+    allocator = RootAllocatorFactory.newRoot(config);
   }
 
   public DrillConfig getConfig() {
@@ -76,6 +74,6 @@ public class BootStrapContext implements Closeable {
       logger.warn("failure resetting metrics.", e);
     }
     loop.shutdownGracefully();
-    allocator.close();
+    DrillAutoCloseables.closeNoChecked(allocator);
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/ForemanException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/ForemanException.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/ForemanException.java
index 32a99ad..c9ed31f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/ForemanException.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/ForemanException.java
@@ -17,41 +17,29 @@
  */
 package org.apache.drill.exec.work.foreman;
 
-import java.lang.reflect.InvocationTargetException;
-
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 
 public class ForemanException extends ExecutionSetupException {
   private static final long serialVersionUID = -6943409010231014085L;
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ForemanException.class);
-
-  public static ForemanException fromThrowable(String message, Throwable cause) {
-    Throwable t = cause instanceof InvocationTargetException
-        ? ((InvocationTargetException)cause).getTargetException() : cause;
-    if (t instanceof ForemanException) {
-      return ((ForemanException) t);
-    }
-    return new ForemanException(message, t);
-  }
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ForemanException.class);
 
   public ForemanException() {
-    super();
   }
 
-  public ForemanException(String message, Throwable cause, boolean enableSuppression, boolean
writableStackTrace) {
+  public ForemanException(final String message, final Throwable cause, final boolean enableSuppression,
+      final boolean writableStackTrace) {
     super(message, cause, enableSuppression, writableStackTrace);
   }
 
-  public ForemanException(String message, Throwable cause) {
+  public ForemanException(final String message, final Throwable cause) {
     super(message, cause);
   }
 
-  public ForemanException(String message) {
+  public ForemanException(final String message) {
     super(message);
   }
 
-  public ForemanException(Throwable cause) {
+  public ForemanException(final Throwable cause) {
     super(cause);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index e218d6c..7053d93 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.client.PrintingResultsListener;
 import org.apache.drill.exec.client.QuerySubmitter.Format;
+import org.apache.drill.exec.memory.OutOfMemoryException;
 import org.apache.drill.exec.proto.UserBitShared.QueryType;
 import org.apache.drill.exec.rpc.RpcException;
 import org.apache.drill.exec.rpc.user.QueryDataBatch;
@@ -56,7 +57,7 @@ public class QueryTestUtil {
    * @throws RpcException if there is a problem setting up the client
    */
   public static DrillClient createClient(final DrillConfig drillConfig, final RemoteServiceSet
remoteServiceSet,
-      final int maxWidth, final Properties props) throws RpcException {
+      final int maxWidth, final Properties props) throws RpcException, OutOfMemoryException
{
     final DrillClient drillClient = new DrillClient(drillConfig, remoteServiceSet.getCoordinator());
     drillClient.connect(props);
 

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
index f6a1f56..f7843f5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestTpchDistributedConcurrent.java
@@ -201,6 +201,16 @@ public class TestTpchDistributedConcurrent extends BaseTestQuery {
     // Stop the querySubmitter thread.
     querySubmitter.interrupt();
 
+    if (interruptedException != null) {
+      final StackTraceElement[] ste = interruptedException.getStackTrace();
+      final StringBuilder sb = new StringBuilder();
+      for(StackTraceElement s : ste) {
+        sb.append(s.toString());
+        sb.append('\n');
+      }
+      System.out.println("interruptedException: " + interruptedException.getMessage() +
+          " from \n" + sb.toString());
+    }
     assertNull("Query error caused interruption", interruptedException);
 
     final int nListeners = listeners.size();

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
index d8ce7c2..6ff2d24 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.fn.impl;
 
 import static org.junit.Assert.assertTrue;
 
+import org.apache.drill.common.DrillAutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.ExecTest;
 import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers;
@@ -55,7 +56,7 @@ public class TestByteComparisonFunctions extends ExecTest {
     helloLong.buffer.release();
     goodbye.buffer.release();
     goodbyeLong.buffer.release();
-    allocator.close();
+    DrillAutoCloseables.closeNoChecked(allocator);
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
index f6fe79e..bb1e718 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/memory/TestEndianess.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.memory;
 import static org.junit.Assert.assertEquals;
 import io.netty.buffer.ByteBuf;
 
+import org.apache.drill.common.DrillAutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.ExecTest;
 import org.junit.Test;
@@ -37,6 +38,6 @@ public class TestEndianess extends ExecTest {
     assertEquals(b.getByte(2), 0);
     assertEquals(b.getByte(3), 0);
     b.release();
-    a.close();
+    DrillAutoCloseables.closeNoChecked(a);
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
index 4230518..a32322b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/DrillClientFactory.java
@@ -18,12 +18,11 @@
 package org.apache.drill.exec.server;
 
 import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.memory.OutOfMemoryException;
 import org.glassfish.hk2.api.Factory;
 
-public class DrillClientFactory implements Factory<DrillClient>{
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillClientFactory.class);
-
-
+public class DrillClientFactory implements Factory<DrillClient> {
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillClientFactory.class);
 
   @Override
   public void dispose(DrillClient arg0) {
@@ -31,8 +30,10 @@ public class DrillClientFactory implements Factory<DrillClient>{
 
   @Override
   public DrillClient provide() {
-    return new DrillClient();
+    try {
+      return new DrillClient();
+    } catch(OutOfMemoryException e) {
+      throw new RuntimeException(e);
+    }
   }
-
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/jdbc-all/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 68ce38a..f7b889c 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -210,7 +210,7 @@
 
   <build>
     <plugins>
-     
+
       <plugin>
         <artifactId>maven-antrun-plugin</artifactId>
         <executions>
@@ -283,7 +283,7 @@
           <createDependencyReducedPom>true</createDependencyReducedPom>
           <dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>
           <minimizeJar>false</minimizeJar>
-          
+
           <artifactSet>
             <includes>
               <include>*:*</include>
@@ -325,7 +325,7 @@
             <!-- Relocate Drill classes to minimize classloader hell. -->
             <relocation><pattern>org.apache.drill.exec.</pattern><shadedPattern>oadd.org.apache.drill.exec.</shadedPattern></relocation>
             <relocation><pattern>org.apache.drill.common.</pattern><shadedPattern>oadd.org.apache.drill.common.</shadedPattern></relocation>
-            
+
             <!-- Move dependencies out of path -->
             <relocation><pattern>antlr.</pattern><shadedPattern>oadd.antlr.</shadedPattern></relocation>
             <relocation><pattern>antlr.</pattern><shadedPattern>oadd.antlr.</shadedPattern></relocation>
@@ -420,7 +420,6 @@
                <exclude>org/apache/drill/exec/server/rest/**</exclude>
                <exclude>org/apache/drill/exec/rpc/data/**</exclude>
                <exclude>org/apache/drill/exec/rpc/control/**</exclude>
-               <exclude>org/apache/drill/exec/work/**</exclude>
              </excludes>
            </filter>
          </filters>
@@ -505,7 +504,7 @@
         </plugins>
       </build>
     </profile>
-    <!-- mondrian data includes 10s of MBs of JSON file if you want to include 
+    <!-- mondrian data includes 10s of MBs of JSON file if you want to include
          them run maven with -Pwith-mondrian-data -->
     <profile>
       <id>with-mondrian-data</id>

http://git-wip-us.apache.org/repos/asf/drill/blob/8c098355/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
index b3fa0ea..f8e5d8e 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillConnectionImpl.java
@@ -17,8 +17,6 @@
  */
 package org.apache.drill.jdbc.impl;
 
-import java.io.IOException;
-import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
@@ -39,6 +37,7 @@ import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryException;
 import org.apache.drill.exec.memory.RootAllocatorFactory;
 import org.apache.drill.exec.rpc.RpcException;
 import org.apache.drill.exec.server.Drillbit;
@@ -50,7 +49,7 @@ import org.apache.drill.jdbc.DrillConnection;
 import org.apache.drill.jdbc.DrillConnectionConfig;
 import org.apache.drill.jdbc.InvalidParameterSqlException;
 import org.apache.drill.jdbc.JdbcApiSqlException;
-
+import org.slf4j.Logger;
 
 /**
  * Drill's implementation of {@link Connection}.
@@ -133,6 +132,8 @@ class DrillConnectionImpl extends AvaticaConnection
         this.client = new DrillClient();
         this.client.connect(config.getZookeeperConnectionString(), info);
       }
+    } catch (OutOfMemoryException e) {
+      throw new SQLException("Failure creating root allocator", e);
     } catch (RpcException e) {
       // (Include cause exception's text in wrapping exception's text so
       // it's more likely to get to user (e.g., via SQLLine), and use
@@ -369,24 +370,33 @@ class DrillConnectionImpl extends AvaticaConnection
     return factory;
   }
 
+  private static void closeOrWarn(final AutoCloseable autoCloseable, final String message,
final Logger logger) {
+    if (autoCloseable == null) {
+      return;
+    }
+
+    try {
+      autoCloseable.close();
+    } catch(Exception e) {
+      logger.warn(message, e);
+    }
+  }
+
+  // TODO this should be an AutoCloseable, and this should be close()
   void cleanup() {
     // First close any open JDBC Statement objects, to close any open ResultSet
     // objects and release their buffers/vectors.
     openStatementsRegistry.close();
 
-    client.close();
-    allocator.close();
+    // TODO all of these should use DeferredException when it is available from DRILL-2245
+    closeOrWarn(client, "Exception while closing client.", logger);
+    closeOrWarn(allocator, "Exception while closing allocator.", logger);
+
     if (bit != null) {
       bit.close();
     }
 
-    if (serviceSet != null) {
-      try {
-        serviceSet.close();
-      } catch (IOException e) {
-        logger.warn("Exception while closing service set.", e);
-      }
-    }
+    closeOrWarn(serviceSet, "Exception while closing service set.", logger);
   }
 
   /**


Mime
View raw message