abdullah alamoudi has submitted this change and it was merged. Change subject: [NO ISSUE][OTH] Log open file count on unexpected HttpServer close ......................................................................
[NO ISSUE][OTH] Log open file count on unexpected HttpServer close - user model changes: no - storage format changes: no - interface changes: no Details: - Recently, there has been some cases where the Http server repeatedly drops its listener. One of the possible causes for this is a leak in open file descriptor. To determine if this is the case, we log the open files count when that happens. Change-Id: Ie43f3392268b3994839d1f98f4de9fe669c7be62 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2483 Sonar-Qube: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java D hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java A hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java 6 files changed, 114 insertions(+), 57 deletions(-) Approvals: Anon. E. Moose #1000171: Jenkins: Verified; No violations found; ; Verified Michael Blow: Looks good to me, approved diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java index d09d0b4..76b68bb 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java @@ -18,7 +18,7 @@ */ package org.apache.hyracks.control.common.heartbeat; -import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans; +import static org.apache.hyracks.util.MXHelper.gcMXBeans; import java.io.DataInput; import java.io.DataOutput; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java deleted file mode 100644 index cfe8c21..0000000 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.control.common.utils; - -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.OperatingSystemMXBean; -import java.lang.management.RuntimeMXBean; -import java.lang.management.ThreadMXBean; -import java.util.Collections; -import java.util.List; - -public class MXHelper { - public static final MemoryMXBean memoryMXBean; - public static final List<GarbageCollectorMXBean> gcMXBeans; - public static final ThreadMXBean threadMXBean; - public static final RuntimeMXBean runtimeMXBean; - public static final OperatingSystemMXBean osMXBean; - - static { - memoryMXBean = ManagementFactory.getMemoryMXBean(); - gcMXBeans = Collections.unmodifiableList(ManagementFactory.getGarbageCollectorMXBeans()); - threadMXBean = ManagementFactory.getThreadMXBean(); - runtimeMXBean = ManagementFactory.getRuntimeMXBean(); - osMXBean = ManagementFactory.getOperatingSystemMXBean(); - } - - private MXHelper() { - } - -} diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java index 6c3a641..0756210 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java @@ -18,10 +18,10 @@ */ package org.apache.hyracks.control.nc; -import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans; -import static org.apache.hyracks.control.common.utils.MXHelper.memoryMXBean; -import static org.apache.hyracks.control.common.utils.MXHelper.osMXBean; -import static org.apache.hyracks.control.common.utils.MXHelper.runtimeMXBean; +import static org.apache.hyracks.util.MXHelper.gcMXBeans; +import static org.apache.hyracks.util.MXHelper.memoryMXBean; +import static org.apache.hyracks.util.MXHelper.osMXBean; +import static org.apache.hyracks.util.MXHelper.runtimeMXBean; import java.io.File; import java.io.IOException; diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java index 4965b85..dfa5ff3 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java @@ -18,10 +18,10 @@ */ package org.apache.hyracks.control.nc.heartbeat; -import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans; -import static org.apache.hyracks.control.common.utils.MXHelper.memoryMXBean; -import static org.apache.hyracks.control.common.utils.MXHelper.osMXBean; -import static org.apache.hyracks.control.common.utils.MXHelper.threadMXBean; +import static org.apache.hyracks.util.MXHelper.gcMXBeans; +import static org.apache.hyracks.util.MXHelper.memoryMXBean; +import static org.apache.hyracks.util.MXHelper.osMXBean; +import static org.apache.hyracks.util.MXHelper.threadMXBean; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.MemoryUsage; diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java index d971a7c..41e0088 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java @@ -30,6 +30,7 @@ import org.apache.hyracks.http.api.IChannelClosedHandler; import org.apache.hyracks.http.api.IServlet; +import org.apache.hyracks.util.MXHelper; import org.apache.hyracks.util.ThreadDumpUtil; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -251,6 +252,7 @@ return; } LOGGER.log(Level.WARN, "{} has stopped unexpectedly. Starting server recovery", this); + MXHelper.logFileDescriptors(); triggerRecovery(); } }); diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java new file mode 100644 index 0000000..2b65106 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.util; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; +import java.lang.management.ThreadMXBean; +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class MXHelper { + private static final Logger LOGGER = LogManager.getLogger(); + public static final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + public static final List<GarbageCollectorMXBean> gcMXBeans = + Collections.unmodifiableList(ManagementFactory.getGarbageCollectorMXBeans()); + public static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + public static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean(); + public static final OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean(); + private static Method getOpenFileDescriptorCount; + private static Method getMaxFileDescriptorCount; + + static { + try { + getOpenFileDescriptorCount = osMXBean.getClass().getMethod("getOpenFileDescriptorCount"); + getMaxFileDescriptorCount = osMXBean.getClass().getDeclaredMethod("getMaxFileDescriptorCount"); + getOpenFileDescriptorCount.setAccessible(true); + getMaxFileDescriptorCount.setAccessible(true); + } catch (Throwable th) { // NOSONAR: diagnostic code shouldn't cause server failure + getOpenFileDescriptorCount = null; + getMaxFileDescriptorCount = null; + LOGGER.log(Level.WARN, "Failed setting up the methods to get the number of file descriptors through {}", + osMXBean.getClass().getName(), th); + } + } + + private MXHelper() { + } + + public static boolean supportOpenFileCount() { + return getOpenFileDescriptorCount != null; + } + + public static Integer getCurrentOpenFileCount() { + if (getOpenFileDescriptorCount == null) { + return -1; + } + try { + return (Integer) getOpenFileDescriptorCount.invoke(osMXBean); + } catch (Throwable e) { // NOSONAR + LOGGER.log(Level.WARN, "Failure invoking getOpenFileDescriptorCount", e); + return -1; + } + } + + public static Integer getMaxOpenFileCount() { + if (getMaxFileDescriptorCount == null) { + return -1; + } + try { + return (Integer) getMaxFileDescriptorCount.invoke(osMXBean); + } catch (Throwable e) { // NOSONAR + LOGGER.log(Level.WARN, "Failure invoking getMaxFileDescriptorCount", e); + return -1; + } + } + + public static void logFileDescriptors() { + try { + if (supportOpenFileCount()) { + LOGGER.log(Level.WARN, + "Number of open files by this process is {}. Max number of allowed open files is {} ", + MXHelper::getCurrentOpenFileCount, MXHelper::getMaxOpenFileCount); + } + } catch (Throwable th) { // NOSONAR: diagnostic code shouldn't cause server failure + LOGGER.log(Level.WARN, "Failed getting the count of open files", th); + } + } + +} -- To view, visit https://asterix-gerrit.ics.uci.edu/2483 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ie43f3392268b3994839d1f98f4de9fe669c7be62 Gerrit-PatchSet: 6 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: abdullah alamoudi <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
