asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject asterixdb git commit: [NO ISSUE][OTH] Log open file count on unexpected HttpServer close
Date Thu, 15 Mar 2018 00:34:32 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master d6d42c861 -> 57df8db1b


[NO ISSUE][OTH] Log open file count on unexpected HttpServer close

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Recently, there has been some cases where the Http server
  repeatedly drops its listener. One of the possible causes
  for this is a leak in open file descriptor. To determine if
  this is the case, we log the open files count when that happens.

Change-Id: Ie43f3392268b3994839d1f98f4de9fe669c7be62
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2483
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/57df8db1
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/57df8db1
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/57df8db1

Branch: refs/heads/master
Commit: 57df8db1b4188940d2f3c8632dcdf1a4c10d0229
Parents: d6d42c8
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Wed Mar 14 15:25:29 2018 -0700
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Wed Mar 14 17:26:57 2018 -0700

----------------------------------------------------------------------
 .../control/common/heartbeat/HeartbeatData.java |   2 +-
 .../hyracks/control/common/utils/MXHelper.java  |  48 ---------
 .../control/nc/NodeControllerService.java       |   8 +-
 .../nc/heartbeat/HeartbeatComputeTask.java      |   8 +-
 .../apache/hyracks/http/server/HttpServer.java  |   2 +
 .../java/org/apache/hyracks/util/MXHelper.java  | 103 +++++++++++++++++++
 6 files changed, 114 insertions(+), 57 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
index d09d0b4..76b68bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -18,7 +18,7 @@
  */
 package org.apache.hyracks.control.common.heartbeat;
 
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
 
 import java.io.DataInput;
 import java.io.DataOutput;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
deleted file mode 100644
index cfe8c21..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.common.utils;
-
-import java.lang.management.GarbageCollectorMXBean;
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import java.lang.management.OperatingSystemMXBean;
-import java.lang.management.RuntimeMXBean;
-import java.lang.management.ThreadMXBean;
-import java.util.Collections;
-import java.util.List;
-
-public class MXHelper {
-    public static final MemoryMXBean memoryMXBean;
-    public static final List<GarbageCollectorMXBean> gcMXBeans;
-    public static final ThreadMXBean threadMXBean;
-    public static final RuntimeMXBean runtimeMXBean;
-    public static final OperatingSystemMXBean osMXBean;
-
-    static {
-        memoryMXBean = ManagementFactory.getMemoryMXBean();
-        gcMXBeans = Collections.unmodifiableList(ManagementFactory.getGarbageCollectorMXBeans());
-        threadMXBean = ManagementFactory.getThreadMXBean();
-        runtimeMXBean = ManagementFactory.getRuntimeMXBean();
-        osMXBean = ManagementFactory.getOperatingSystemMXBean();
-    }
-
-    private MXHelper() {
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 6c3a641..0756210 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -18,10 +18,10 @@
  */
 package org.apache.hyracks.control.nc;
 
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
-import static org.apache.hyracks.control.common.utils.MXHelper.memoryMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.osMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.runtimeMXBean;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.memoryMXBean;
+import static org.apache.hyracks.util.MXHelper.osMXBean;
+import static org.apache.hyracks.util.MXHelper.runtimeMXBean;
 
 import java.io.File;
 import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
index 4965b85..dfa5ff3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
@@ -18,10 +18,10 @@
  */
 package org.apache.hyracks.control.nc.heartbeat;
 
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
-import static org.apache.hyracks.control.common.utils.MXHelper.memoryMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.osMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.threadMXBean;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.memoryMXBean;
+import static org.apache.hyracks.util.MXHelper.osMXBean;
+import static org.apache.hyracks.util.MXHelper.threadMXBean;
 
 import java.lang.management.GarbageCollectorMXBean;
 import java.lang.management.MemoryUsage;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index d971a7c..41e0088 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hyracks.http.api.IChannelClosedHandler;
 import org.apache.hyracks.http.api.IServlet;
+import org.apache.hyracks.util.MXHelper;
 import org.apache.hyracks.util.ThreadDumpUtil;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -251,6 +252,7 @@ public class HttpServer {
                     return;
                 }
                 LOGGER.log(Level.WARN, "{} has stopped unexpectedly. Starting server recovery",
this);
+                MXHelper.logFileDescriptors();
                 triggerRecovery();
             }
         });

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
new file mode 100644
index 0000000..2b65106
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.lang.management.ThreadMXBean;
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class MXHelper {
+    private static final Logger LOGGER = LogManager.getLogger();
+    public static final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+    public static final List<GarbageCollectorMXBean> gcMXBeans =
+            Collections.unmodifiableList(ManagementFactory.getGarbageCollectorMXBeans());
+    public static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+    public static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+    public static final OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
+    private static Method getOpenFileDescriptorCount;
+    private static Method getMaxFileDescriptorCount;
+
+    static {
+        try {
+            getOpenFileDescriptorCount = osMXBean.getClass().getMethod("getOpenFileDescriptorCount");
+            getMaxFileDescriptorCount = osMXBean.getClass().getDeclaredMethod("getMaxFileDescriptorCount");
+            getOpenFileDescriptorCount.setAccessible(true);
+            getMaxFileDescriptorCount.setAccessible(true);
+        } catch (Throwable th) { // NOSONAR: diagnostic code shouldn't cause server failure
+            getOpenFileDescriptorCount = null;
+            getMaxFileDescriptorCount = null;
+            LOGGER.log(Level.WARN, "Failed setting up the methods to get the number of file
descriptors through {}",
+                    osMXBean.getClass().getName(), th);
+        }
+    }
+
+    private MXHelper() {
+    }
+
+    public static boolean supportOpenFileCount() {
+        return getOpenFileDescriptorCount != null;
+    }
+
+    public static Integer getCurrentOpenFileCount() {
+        if (getOpenFileDescriptorCount == null) {
+            return -1;
+        }
+        try {
+            return (Integer) getOpenFileDescriptorCount.invoke(osMXBean);
+        } catch (Throwable e) { // NOSONAR
+            LOGGER.log(Level.WARN, "Failure invoking getOpenFileDescriptorCount", e);
+            return -1;
+        }
+    }
+
+    public static Integer getMaxOpenFileCount() {
+        if (getMaxFileDescriptorCount == null) {
+            return -1;
+        }
+        try {
+            return (Integer) getMaxFileDescriptorCount.invoke(osMXBean);
+        } catch (Throwable e) { // NOSONAR
+            LOGGER.log(Level.WARN, "Failure invoking getMaxFileDescriptorCount", e);
+            return -1;
+        }
+    }
+
+    public static void logFileDescriptors() {
+        try {
+            if (supportOpenFileCount()) {
+                LOGGER.log(Level.WARN,
+                        "Number of open files by this process is {}. Max number of allowed
open files is {} ",
+                        MXHelper::getCurrentOpenFileCount, MXHelper::getMaxOpenFileCount);
+            }
+        } catch (Throwable th) { // NOSONAR: diagnostic code shouldn't cause server failure
+            LOGGER.log(Level.WARN, "Failed getting the count of open files", th);
+        }
+    }
+
+}


Mime
View raw message