accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch 2.0 updated: fix #1231 prevent hadoop shutdown hook from failing FATE tx (#1245)
Date Wed, 03 Jul 2019 13:34:06 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/2.0 by this push:
     new aebaf86  fix #1231 prevent hadoop shutdown hook from failing FATE tx (#1245)
aebaf86 is described below

commit aebaf86abba74235e33e18cc3ff507372bbc3f61
Author: Keith Turner <kturner@apache.org>
AuthorDate: Wed Jul 3 09:34:01 2019 -0400

    fix #1231 prevent hadoop shutdown hook from failing FATE tx (#1245)
---
 .../apache/accumulo/core/util/ShutdownUtil.java    | 36 ++++++++++++++++++++
 .../main/java/org/apache/accumulo/fate/Fate.java   | 38 ++++++++++++++++++++++
 .../apache/accumulo/tserver/tablet/Scanner.java    |  3 +-
 .../org/apache/accumulo/tserver/tablet/Tablet.java | 17 ++--------
 4 files changed, 78 insertions(+), 16 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
new file mode 100644
index 0000000..f9b7a7b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.core.util;
+
+public class ShutdownUtil {
+
+  /**
+   * Determine if a JVM shutdown is in progress.
+   *
+   */
+  public static boolean isShutdownInProgress() {
+    try {
+      Runtime.getRuntime().removeShutdownHook(new Thread(() -> {}));
+    } catch (IllegalStateException ise) {
+      return true;
+    }
+
+    return false;
+  }
+
+}
diff --git a/core/src/main/java/org/apache/accumulo/fate/Fate.java b/core/src/main/java/org/apache/accumulo/fate/Fate.java
index ec2df1b..a2a4f72 100644
--- a/core/src/main/java/org/apache/accumulo/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/fate/Fate.java
@@ -16,14 +16,18 @@
  */
 package org.apache.accumulo.fate;
 
+import java.io.IOException;
 import java.util.EnumSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.accumulo.core.util.ShutdownUtil;
 import org.apache.accumulo.fate.ReadOnlyTStore.TStatus;
 import org.apache.accumulo.fate.util.LoggingRunnable;
+import org.apache.accumulo.fate.util.UtilWaitThread;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -74,6 +78,7 @@ public class Fate<T> {
                 continue;
 
             } catch (Exception e) {
+              blockIfHadoopShutdown(tid, e);
               transitionToFailed(tid, e);
               continue;
             }
@@ -108,6 +113,39 @@ public class Fate<T> {
       }
     }
 
+    private boolean isIOException(Throwable e) {
+      if (e == null)
+        return false;
+
+      if (e instanceof IOException)
+        return true;
+
+      for (Throwable suppressed : e.getSuppressed())
+        if (isIOException(suppressed))
+          return true;
+
+      return isIOException(e.getCause());
+    }
+
+    /**
+     * The Hadoop Filesystem registers a java shutdown hook that closes the file system.
This can
+     * cause threads to get spurious IOException. If this happens, instead of failing a FATE
+     * transaction just wait for process to die. When the master start elsewhere the FATE
+     * transaction can resume.
+     */
+    private void blockIfHadoopShutdown(long tid, Exception e) {
+      if (isIOException(e) && ShutdownUtil.isShutdownInProgress()) {
+        String tidStr = String.format("%016x", tid);
+        log.info("Ignoring exception that was likely caused by Hadoop Shutdown hook. tid
: {} ",
+            tidStr, e);
+
+        while (true) {
+          // Nothing is going to work well at this point, so why even try. Just wait for
the end.
+          UtilWaitThread.sleepUninterruptibly(1, TimeUnit.MINUTES);
+        }
+      }
+    }
+
     private void transitionToFailed(long tid, Exception e) {
       String tidStr = String.format("%016x", tid);
       final String msg = "Failed to execute Repo, tid=" + tidStr;
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
index 3e5e118..cbd27f2 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
@@ -27,6 +27,7 @@ import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IterationInterruptedException;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.system.SourceSwitchingIterator;
+import org.apache.accumulo.core.util.ShutdownUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -118,7 +119,7 @@ public class Scanner {
       else
         throw iie;
     } catch (IOException ioe) {
-      if (tablet.shutdownInProgress()) {
+      if (ShutdownUtil.isShutdownInProgress()) {
         log.debug("IOException while shutdown in progress ", ioe);
         throw new TabletClosedException(ioe); // assume IOException was caused by execution
of HDFS
                                               // shutdown hook
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 84f25c1..e16fc59 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -90,6 +90,7 @@ import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
 import org.apache.accumulo.core.trace.TraceUtil;
 import org.apache.accumulo.core.util.LocalityGroupUtil;
 import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.ShutdownUtil;
 import org.apache.accumulo.core.util.ratelimit.RateLimiter;
 import org.apache.accumulo.server.ServerConstants;
 import org.apache.accumulo.server.ServerContext;
@@ -665,7 +666,7 @@ public class Tablet {
             entriesAdded);
         tabletClosed = true;
       } catch (IOException ioe) {
-        if (shutdownInProgress()) {
+        if (ShutdownUtil.isShutdownInProgress()) {
           // assume HDFS shutdown hook caused this exception
           log.debug("IOException while shutdown in progress", ioe);
           handleTabletClosedDuringScan(results, lookupResult, exceededMemoryUsage, range,
@@ -872,20 +873,6 @@ public class Tablet {
     return new Batch(skipContinueKey, results, continueKey, resultBytes);
   }
 
-  /**
-   * Determine if a JVM shutdown is in progress.
-   *
-   */
-  boolean shutdownInProgress() {
-    try {
-      Runtime.getRuntime().removeShutdownHook(new Thread(() -> {}));
-    } catch (IllegalStateException ise) {
-      return true;
-    }
-
-    return false;
-  }
-
   public Scanner createScanner(Range range, int num, Set<Column> columns,
       Authorizations authorizations, List<IterInfo> ssiList, Map<String,Map<String,String>>
ssio,
       boolean isolated, AtomicBoolean interruptFlag, SamplerConfiguration samplerConfig,


Mime
View raw message