- Revision
- 285227
- Author
- repst...@apple.com
- Date
- 2021-11-03 13:26:04 -0700 (Wed, 03 Nov 2021)
Log Message
Cherry-pick r285177. rdar://problem/84987165
Terminate unresponsive network process by crashing it
https://bugs.webkit.org/show_bug.cgi?id=232603
Reviewed by Chris Dumez.
UI process currently kills network process when it does not respond message in some time (network process being
unresponsive). We've found one common case where network process becomes unresponsive is that it is blocked by
some slow operation on the main thread (like file operation in rdar://84511633). To understand what the
operations are and make a fix, we now ask network process to crash itself on IPC thread. In this way, we can get
crash report that includes the call stack of the main thread. To avoid generating too many crash reports, we
only send the crash message to network process when it becomes unresponsive multiple times in a short time
period.
* Platform/IPC/Connection.cpp:
(IPC::terminateDueToIPCTerminateMessage):
(IPC::Connection::processIncomingMessage):
* Scripts/webkit/model.py:
* Scripts/webkit/tests/MessageNames.cpp:
(IPC::description):
(IPC::receiverName):
(IPC::isValidMessageName):
* Scripts/webkit/tests/MessageNames.h:
* UIProcess/Network/NetworkProcessProxy.cpp:
(WebKit::shouldTerminateNetworkProcessBySendingMessage):
(WebKit::NetworkProcessProxy::didBecomeUnresponsive):
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@285177 268f45cc-cd09-0410-ab3c-d52691b4dbfc
Modified Paths
Diff
Modified: branches/safari-612.3.3.1-branch/Source/WebKit/ChangeLog (285226 => 285227)
--- branches/safari-612.3.3.1-branch/Source/WebKit/ChangeLog 2021-11-03 20:26:00 UTC (rev 285226)
+++ branches/safari-612.3.3.1-branch/Source/WebKit/ChangeLog 2021-11-03 20:26:04 UTC (rev 285227)
@@ -1,5 +1,66 @@
2021-11-03 Russell Epstein <repst...@apple.com>
+ Cherry-pick r285177. rdar://problem/84987165
+
+ Terminate unresponsive network process by crashing it
+ https://bugs.webkit.org/show_bug.cgi?id=232603
+
+ Reviewed by Chris Dumez.
+
+ UI process currently kills network process when it does not respond message in some time (network process being
+ unresponsive). We've found one common case where network process becomes unresponsive is that it is blocked by
+ some slow operation on the main thread (like file operation in rdar://84511633). To understand what the
+ operations are and make a fix, we now ask network process to crash itself on IPC thread. In this way, we can get
+ crash report that includes the call stack of the main thread. To avoid generating too many crash reports, we
+ only send the crash message to network process when it becomes unresponsive multiple times in a short time
+ period.
+
+ * Platform/IPC/Connection.cpp:
+ (IPC::terminateDueToIPCTerminateMessage):
+ (IPC::Connection::processIncomingMessage):
+ * Scripts/webkit/model.py:
+ * Scripts/webkit/tests/MessageNames.cpp:
+ (IPC::description):
+ (IPC::receiverName):
+ (IPC::isValidMessageName):
+ * Scripts/webkit/tests/MessageNames.h:
+ * UIProcess/Network/NetworkProcessProxy.cpp:
+ (WebKit::shouldTerminateNetworkProcessBySendingMessage):
+ (WebKit::NetworkProcessProxy::didBecomeUnresponsive):
+
+
+ git-svn-id: https://svn.webkit.org/repository/webkit/trunk@285177 268f45cc-cd09-0410-ab3c-d52691b4dbfc
+
+ 2021-11-02 Sihui Liu <sihui_...@apple.com>
+
+ Terminate unresponsive network process by crashing it
+ https://bugs.webkit.org/show_bug.cgi?id=232603
+
+ Reviewed by Chris Dumez.
+
+ UI process currently kills network process when it does not respond message in some time (network process being
+ unresponsive). We've found one common case where network process becomes unresponsive is that it is blocked by
+ some slow operation on the main thread (like file operation in rdar://84511633). To understand what the
+ operations are and make a fix, we now ask network process to crash itself on IPC thread. In this way, we can get
+ crash report that includes the call stack of the main thread. To avoid generating too many crash reports, we
+ only send the crash message to network process when it becomes unresponsive multiple times in a short time
+ period.
+
+ * Platform/IPC/Connection.cpp:
+ (IPC::terminateDueToIPCTerminateMessage):
+ (IPC::Connection::processIncomingMessage):
+ * Scripts/webkit/model.py:
+ * Scripts/webkit/tests/MessageNames.cpp:
+ (IPC::description):
+ (IPC::receiverName):
+ (IPC::isValidMessageName):
+ * Scripts/webkit/tests/MessageNames.h:
+ * UIProcess/Network/NetworkProcessProxy.cpp:
+ (WebKit::shouldTerminateNetworkProcessBySendingMessage):
+ (WebKit::NetworkProcessProxy::didBecomeUnresponsive):
+
+2021-11-03 Russell Epstein <repst...@apple.com>
+
Cherry-pick r285115. rdar://problem/84984094
Increase responsiveness timeout for network process
Modified: branches/safari-612.3.3.1-branch/Source/WebKit/Platform/IPC/Connection.cpp (285226 => 285227)
--- branches/safari-612.3.3.1-branch/Source/WebKit/Platform/IPC/Connection.cpp 2021-11-03 20:26:00 UTC (rev 285226)
+++ branches/safari-612.3.3.1-branch/Source/WebKit/Platform/IPC/Connection.cpp 2021-11-03 20:26:04 UTC (rev 285227)
@@ -42,6 +42,7 @@
#if PLATFORM(COCOA)
#include "MachMessage.h"
+#include "WKCrashReporter.h"
#endif
#if USE(UNIX_DOMAIN_SOCKETS)
@@ -744,6 +745,16 @@
// This can happen if the send timed out, so it's fine to ignore.
}
+static NEVER_INLINE NO_RETURN_DUE_TO_CRASH void terminateDueToIPCTerminateMessage()
+{
+#if PLATFORM(COCOA)
+ WebKit::logAndSetCrashLogMessage("Receives Terminate message");
+#else
+ WTFLogAlways("Receives Terminate message");
+#endif
+ CRASH();
+}
+
void Connection::processIncomingMessage(std::unique_ptr<Decoder> message)
{
ASSERT(message->messageReceiverName() != ReceiverName::Invalid);
@@ -753,6 +764,9 @@
return;
}
+ if (message->messageName() == MessageName::Terminate)
+ return terminateDueToIPCTerminateMessage();
+
if (!MessageReceiveQueueMap::isValidMessage(*message)) {
RunLoop::main().dispatch([protectedThis = makeRef(*this), messageName = message->messageName()]() mutable {
protectedThis->dispatchDidReceiveInvalidMessage(messageName);
Modified: branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/model.py (285226 => 285227)
--- branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/model.py 2021-11-03 20:26:00 UTC (rev 285226)
+++ branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/model.py 2021-11-03 20:26:04 UTC (rev 285227)
@@ -77,7 +77,8 @@
Message('InitializeConnection', [], [], attributes=[BUILTIN_ATTRIBUTE], condition="PLATFORM(COCOA)"),
Message('LegacySessionState', [], [], attributes=[BUILTIN_ATTRIBUTE], condition=None),
Message('SetStreamDestinationID', [], [], attributes=[BUILTIN_ATTRIBUTE], condition=None),
- Message('ProcessOutOfStreamMessage', [], [], attributes=[BUILTIN_ATTRIBUTE], condition=None)
+ Message('ProcessOutOfStreamMessage', [], [], attributes=[BUILTIN_ATTRIBUTE], condition=None),
+ Message('Terminate', [], [], attributes=[BUILTIN_ATTRIBUTE], condition=None),
], condition=None)
Modified: branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/tests/MessageNames.cpp (285226 => 285227)
--- branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/tests/MessageNames.cpp 2021-11-03 20:26:00 UTC (rev 285226)
+++ branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/tests/MessageNames.cpp 2021-11-03 20:26:04 UTC (rev 285227)
@@ -156,6 +156,8 @@
return "SetStreamDestinationID";
case MessageName::SyncMessageReply:
return "SyncMessageReply";
+ case MessageName::Terminate:
+ return "Terminate";
case MessageName::TestWithSuperclass_TestAsyncMessageReply:
return "TestWithSuperclass_TestAsyncMessageReply";
case MessageName::TestWithSuperclass_TestAsyncMessageWithConnectionReply:
@@ -258,6 +260,7 @@
case MessageName::ProcessOutOfStreamMessage:
case MessageName::SetStreamDestinationID:
case MessageName::SyncMessageReply:
+ case MessageName::Terminate:
return ReceiverName::IPC;
case MessageName::TestWithSuperclass_TestAsyncMessageReply:
case MessageName::TestWithSuperclass_TestAsyncMessageWithConnectionReply:
@@ -466,6 +469,8 @@
return true;
if (messageName == IPC::MessageName::SyncMessageReply)
return true;
+ if (messageName == IPC::MessageName::Terminate)
+ return true;
#if ENABLE(TEST_FEATURE)
if (messageName == IPC::MessageName::TestWithSuperclass_TestAsyncMessageReply)
return true;
Modified: branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/tests/MessageNames.h (285226 => 285227)
--- branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/tests/MessageNames.h 2021-11-03 20:26:00 UTC (rev 285226)
+++ branches/safari-612.3.3.1-branch/Source/WebKit/Scripts/webkit/tests/MessageNames.h 2021-11-03 20:26:04 UTC (rev 285227)
@@ -107,6 +107,7 @@
, ProcessOutOfStreamMessage
, SetStreamDestinationID
, SyncMessageReply
+ , Terminate
, TestWithSuperclass_TestAsyncMessageReply
, TestWithSuperclass_TestAsyncMessageWithConnectionReply
, TestWithSuperclass_TestAsyncMessageWithMultipleArgumentsReply
Modified: branches/safari-612.3.3.1-branch/Source/WebKit/UIProcess/Network/NetworkProcessProxy.cpp (285226 => 285227)
--- branches/safari-612.3.3.1-branch/Source/WebKit/UIProcess/Network/NetworkProcessProxy.cpp 2021-11-03 20:26:00 UTC (rev 285226)
+++ branches/safari-612.3.3.1-branch/Source/WebKit/UIProcess/Network/NetworkProcessProxy.cpp 2021-11-03 20:26:04 UTC (rev 285227)
@@ -86,6 +86,8 @@
using namespace WebCore;
static constexpr Seconds networkProcessResponsivenessTimeout = 6_s;
+static constexpr int unresponsivenessCountLimit = 3;
+static constexpr Seconds unresponsivenessCheckPeriod = 15_s;
static HashSet<NetworkProcessProxy*>& networkProcessesSet()
{
@@ -94,6 +96,25 @@
return set;
}
+static bool shouldTerminateNetworkProcessBySendingMessage()
+{
+ static WallTime unresponsivenessPeriodStartTime = WallTime::now();
+ static int unresponsivenessCountDuringThisPeriod = 0;
+ auto now = WallTime::now();
+
+ if (now - unresponsivenessPeriodStartTime > unresponsivenessCheckPeriod) {
+ unresponsivenessCountDuringThisPeriod = 1;
+ unresponsivenessPeriodStartTime = now;
+ return false;
+ }
+
+ ++unresponsivenessCountDuringThisPeriod;
+ if (unresponsivenessCountDuringThisPeriod >= unresponsivenessCountLimit)
+ return true;
+
+ return false;
+}
+
Vector<Ref<NetworkProcessProxy>> NetworkProcessProxy::allNetworkProcesses()
{
Vector<Ref<NetworkProcessProxy>> processes;
@@ -127,6 +148,19 @@
void NetworkProcessProxy::didBecomeUnresponsive()
{
RELEASE_LOG_ERROR(Process, "NetworkProcessProxy::didBecomeUnresponsive: NetworkProcess with PID %d became unresponsive, terminating it", processIdentifier());
+
+ // Let network process terminates itself and generate crash report for investigation of hangs.
+ // We currently only do this when network process becomes unresponsive multiple times in a short
+ // time period to avoid generating too many crash reports with same back trace on user's device.
+ if (shouldTerminateNetworkProcessBySendingMessage()) {
+ sendMessage(makeUniqueRef<IPC::Encoder>(IPC::MessageName::Terminate, 0), { });
+ RunLoop::main().dispatchAfter(1_s, [weakThis = WeakPtr { *this }] () mutable {
+ if (weakThis)
+ weakThis->terminate();
+ });
+ return;
+ }
+
terminate();
}
