Diff
Modified: trunk/JSTests/ChangeLog (287378 => 287379)
--- trunk/JSTests/ChangeLog 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/JSTests/ChangeLog 2021-12-23 01:12:14 UTC (rev 287379)
@@ -1,3 +1,12 @@
+2021-12-22 Saam Barati <sbar...@apple.com>
+
+ LLInt should loop OSR into BBQ and BBQ should loop OSR into OMG
+ https://bugs.webkit.org/show_bug.cgi?id=234542
+
+ Reviewed by Yusuke Suzuki.
+
+ * wasm/wast-tests/harness.js:
+
2021-12-19 Ross Kirsling <ross.kirsl...@sony.com>
[JSC] OpPow should have a "small int exponent" fast path at lower tiers
Modified: trunk/JSTests/stress/callerIsOMGCompiled-should-be-robust.js (287378 => 287379)
--- trunk/JSTests/stress/callerIsOMGCompiled-should-be-robust.js 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/JSTests/stress/callerIsOMGCompiled-should-be-robust.js 2021-12-23 01:12:14 UTC (rev 287379)
@@ -1,2 +1,2 @@
-setUnhandledRejectionCallback(callerIsOMGCompiled);
+setUnhandledRejectionCallback(callerIsBBQOrOMGCompiled);
Promise.reject();
Modified: trunk/JSTests/wasm/wast-tests/harness.js (287378 => 287379)
--- trunk/JSTests/wasm/wast-tests/harness.js 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/JSTests/wasm/wast-tests/harness.js 2021-12-23 01:12:14 UTC (rev 287379)
@@ -4,6 +4,7 @@
};
globalThis.__linear_memory = new WebAssembly.Memory({ initial: 1 });
+globalThis.callerIsOMGCompiled = callerIsBBQOrOMGCompiled; // Can't change the .wast easily...
async function runWasmFile(filePath) {
let blob = readFile(filePath, "binary");
Modified: trunk/Source/_javascript_Core/ChangeLog (287378 => 287379)
--- trunk/Source/_javascript_Core/ChangeLog 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/ChangeLog 2021-12-23 01:12:14 UTC (rev 287379)
@@ -1,3 +1,52 @@
+2021-12-22 Saam Barati <sbar...@apple.com>
+
+ LLInt should loop OSR into BBQ and BBQ should loop OSR into OMG
+ https://bugs.webkit.org/show_bug.cgi?id=234542
+
+ Reviewed by Yusuke Suzuki.
+
+ It's a startup perf improvement on some Wasm benchmarks I'm running to have
+ Wasm LLInt do loop OSR entry into BBQ instead of OMG. This improves this
+ benchmark by 5%. There is probably more perf to be had here. Currently,
+ we're just OSR entering into B3 BBQ O1. However, in the future, we should
+ just compile a single Air BBQ Callee that allows for OSR entry at loop
+ boundaries. Maybe we can model this using EntrySwitch without any real
+ harm to throughput.
+
+ * _javascript_Core.xcodeproj/project.pbxproj:
+ * Sources.txt:
+ * assembler/MacroAssemblerCodeRef.cpp:
+ (JSC::shouldDumpDisassemblyFor):
+ * jsc.cpp:
+ (JSC_DEFINE_HOST_FUNCTION):
+ * wasm/WasmB3IRGenerator.cpp:
+ (JSC::Wasm::B3IRGenerator::B3IRGenerator):
+ (JSC::Wasm::parseAndCompile):
+ * wasm/WasmCallee.h:
+ (JSC::Wasm::Callee::setOSREntryCallee): Deleted.
+ * wasm/WasmCalleeGroup.h:
+ * wasm/WasmCompilationMode.cpp:
+ (JSC::Wasm::makeString):
+ * wasm/WasmCompilationMode.h:
+ (JSC::Wasm::isOSREntry):
+ (JSC::Wasm::isAnyBBQ):
+ (JSC::Wasm::isAnyOMG):
+ * wasm/WasmOMGForOSREntryPlan.cpp: Removed.
+ * wasm/WasmOMGForOSREntryPlan.h: Removed.
+ * wasm/WasmOSREntryPlan.cpp: Copied from Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.cpp.
+ (JSC::Wasm::OSREntryPlan::OSREntryPlan):
+ (JSC::Wasm::OSREntryPlan::work):
+ (JSC::Wasm::OMGForOSREntryPlan::OMGForOSREntryPlan): Deleted.
+ (JSC::Wasm::OMGForOSREntryPlan::work): Deleted.
+ * wasm/WasmOSREntryPlan.h: Copied from Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.h.
+ * wasm/WasmOperations.cpp:
+ (JSC::Wasm::doOSREntry):
+ (JSC::Wasm::JSC_DEFINE_JIT_OPERATION):
+ * wasm/WasmPlan.cpp:
+ (JSC::Wasm::Plan::updateCallSitesToCallUs):
+ * wasm/WasmSlowPaths.cpp:
+ (JSC::LLInt::WASM_SLOW_PATH_DECL):
+
2021-12-22 Alex Christensen <achristen...@webkit.org>
Fix compiling with pickier compiler
Modified: trunk/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj (287378 => 287379)
--- trunk/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/_javascript_Core.xcodeproj/project.pbxproj 2021-12-23 01:12:14 UTC (rev 287379)
@@ -895,6 +895,7 @@
5267CF82249316B10022BF6D /* FastJITPermissions.h in Headers */ = {isa = PBXBuildFile; fileRef = 5267CF81249316AD0022BF6D /* FastJITPermissions.h */; settings = {ATTRIBUTES = (Private, ); }; };
526AC4B71E977C5D003500E1 /* WasmCalleeGroup.h in Headers */ = {isa = PBXBuildFile; fileRef = 526AC4B51E977C5D003500E1 /* WasmCalleeGroup.h */; settings = {ATTRIBUTES = (Private, ); }; };
527CE35422555FE500C6F382 /* JSToWasmICCallee.h in Headers */ = {isa = PBXBuildFile; fileRef = 527CE35322555FDD00C6F382 /* JSToWasmICCallee.h */; };
+ 527E6CEC2772B9CB005E0782 /* WasmOSREntryPlan.h in Headers */ = {isa = PBXBuildFile; fileRef = 527E6CEB2772B9C5005E0782 /* WasmOSREntryPlan.h */; };
52847ADC21FFB8690061A9DB /* WasmAirIRGenerator.h in Headers */ = {isa = PBXBuildFile; fileRef = 52847ADA21FFB8630061A9DB /* WasmAirIRGenerator.h */; };
52B310FB1974AE610080857C /* FunctionHasExecutedCache.h in Headers */ = {isa = PBXBuildFile; fileRef = 52B310FA1974AE610080857C /* FunctionHasExecutedCache.h */; settings = {ATTRIBUTES = (Private, ); }; };
52B311011975B4670080857C /* TypeLocationCache.h in Headers */ = {isa = PBXBuildFile; fileRef = 52B311001975B4670080857C /* TypeLocationCache.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -1913,7 +1914,6 @@
E365F33A24AA623900C991B2 /* IntlDisplayNamesConstructor.lut.h in Headers */ = {isa = PBXBuildFile; fileRef = E365F33924AA621200C991B2 /* IntlDisplayNamesConstructor.lut.h */; };
E365F33B24AA623B00C991B2 /* IntlDisplayNamesPrototype.lut.h in Headers */ = {isa = PBXBuildFile; fileRef = E365F33824AA621100C991B2 /* IntlDisplayNamesPrototype.lut.h */; };
E366441E254409B30001876F /* IntlListFormat.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E318CA69254406B5004DC129 /* IntlListFormat.cpp */; };
- E36B767022F8D61900D09818 /* WasmOMGForOSREntryPlan.h in Headers */ = {isa = PBXBuildFile; fileRef = E36B766F22F8D61100D09818 /* WasmOMGForOSREntryPlan.h */; };
E36CC9472086314F0051FFD6 /* WasmCreationMode.h in Headers */ = {isa = PBXBuildFile; fileRef = E36CC9462086314F0051FFD6 /* WasmCreationMode.h */; settings = {ATTRIBUTES = (Private, ); }; };
E36EDCE524F0975700E60DA2 /* Concurrency.h in Headers */ = {isa = PBXBuildFile; fileRef = E36EDCE424F0975700E60DA2 /* Concurrency.h */; settings = {ATTRIBUTES = (Private, ); }; };
E3711992253FA87F00BA69A0 /* Gate.h in Headers */ = {isa = PBXBuildFile; fileRef = E3711991253FA87E00BA69A0 /* Gate.h */; settings = {ATTRIBUTES = (Private, ); }; };
@@ -3700,6 +3700,8 @@
527773DD1AAF83AC00BDE7E8 /* RuntimeType.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = RuntimeType.cpp; sourceTree = "<group>"; };
527CE35222555FDD00C6F382 /* JSToWasmICCallee.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = JSToWasmICCallee.cpp; path = js/JSToWasmICCallee.cpp; sourceTree = "<group>"; };
527CE35322555FDD00C6F382 /* JSToWasmICCallee.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = JSToWasmICCallee.h; path = js/JSToWasmICCallee.h; sourceTree = "<group>"; };
+ 527E6CEA2772B9C5005E0782 /* WasmOSREntryPlan.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = WasmOSREntryPlan.cpp; sourceTree = "<group>"; };
+ 527E6CEB2772B9C5005E0782 /* WasmOSREntryPlan.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WasmOSREntryPlan.h; sourceTree = "<group>"; };
52847AD921FFB8630061A9DB /* WasmAirIRGenerator.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = WasmAirIRGenerator.cpp; sourceTree = "<group>"; };
52847ADA21FFB8630061A9DB /* WasmAirIRGenerator.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WasmAirIRGenerator.h; sourceTree = "<group>"; };
52B310FA1974AE610080857C /* FunctionHasExecutedCache.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FunctionHasExecutedCache.h; sourceTree = "<group>"; };
@@ -5240,8 +5242,6 @@
E365F33824AA621100C991B2 /* IntlDisplayNamesPrototype.lut.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = IntlDisplayNamesPrototype.lut.h; sourceTree = "<group>"; };
E365F33924AA621200C991B2 /* IntlDisplayNamesConstructor.lut.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = IntlDisplayNamesConstructor.lut.h; sourceTree = "<group>"; };
E36B480123E9573800E4A66E /* UnlinkedCodeBlockGenerator.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = UnlinkedCodeBlockGenerator.cpp; sourceTree = "<group>"; };
- E36B766E22F8D61100D09818 /* WasmOMGForOSREntryPlan.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = WasmOMGForOSREntryPlan.cpp; sourceTree = "<group>"; };
- E36B766F22F8D61100D09818 /* WasmOMGForOSREntryPlan.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = WasmOMGForOSREntryPlan.h; sourceTree = "<group>"; };
E36CC9462086314F0051FFD6 /* WasmCreationMode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = WasmCreationMode.h; sourceTree = "<group>"; };
E36EDCE424F0975700E60DA2 /* Concurrency.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Concurrency.h; sourceTree = "<group>"; };
E3711991253FA87E00BA69A0 /* Gate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Gate.h; sourceTree = "<group>"; };
@@ -6609,8 +6609,6 @@
E3BF1BAD238AAED1003A1C2B /* IsoHeapCellType.h */,
E3C8ED4223A1DBC500131958 /* IsoInlinedHeapCellType.h */,
FE65B1502748601F00DB5B16 /* IsoInlinedHeapCellTypeInlines.h */,
- 537FEECD2742BDE000C9EFEE /* IsoMemoryAllocatorBase.h */,
- 537FEECA2742BDE000C9EFEE /* IsoMemoryAllocatorBaseInlines.h */,
0FDCE12C1FAFB4DE006F3901 /* IsoSubspace.cpp */,
0FDCE12B1FAFB4DE006F3901 /* IsoSubspace.h */,
0FD2FD9220B52BDC00F09441 /* IsoSubspaceInlines.h */,
@@ -7255,10 +7253,10 @@
AD7B4B2D1FA3E28600C9DF79 /* WasmNameSection.h */,
ADD8FA441EB3077100DF542F /* WasmNameSectionParser.cpp */,
ADD8FA431EB3077100DF542F /* WasmNameSectionParser.h */,
- E36B766E22F8D61100D09818 /* WasmOMGForOSREntryPlan.cpp */,
- E36B766F22F8D61100D09818 /* WasmOMGForOSREntryPlan.h */,
5311BD481EA581E500525281 /* WasmOMGPlan.cpp */,
5311BD491EA581E500525281 /* WasmOMGPlan.h */,
+ 527E6CEA2772B9C5005E0782 /* WasmOSREntryPlan.cpp */,
+ 527E6CEB2772B9C5005E0782 /* WasmOSREntryPlan.h */,
53C6FEF01E8AFE0C00B18425 /* WasmOpcodeOrigin.cpp */,
53C6FEEE1E8ADFA900B18425 /* WasmOpcodeOrigin.h */,
E39D8B2C23021E1E00265852 /* WasmOperations.cpp */,
@@ -9717,6 +9715,7 @@
A7E5A3A81797432D00E893C0 /* CompilationResult.h in Headers */,
0F4F11E8209BCDAB00709654 /* CompilerTimingScope.h in Headers */,
0FDCE12A1FAFA85F006F3901 /* CompleteSubspace.h in Headers */,
+ 527E6CEC2772B9CB005E0782 /* WasmOSREntryPlan.h in Headers */,
0FD2FD9420B52BDE00F09441 /* CompleteSubspaceInlines.h in Headers */,
BC18C3F40E16F5CD00B34460 /* Completion.h in Headers */,
0F6FC751196110A800E1D02D /* ComplexGetStatus.h in Headers */,
@@ -10975,7 +10974,6 @@
AD5B416F1EBAFB77008EFA43 /* WasmName.h in Headers */,
AD7B4B2E1FA3E29800C9DF79 /* WasmNameSection.h in Headers */,
ADD8FA461EB3079700DF542F /* WasmNameSectionParser.h in Headers */,
- E36B767022F8D61900D09818 /* WasmOMGForOSREntryPlan.h in Headers */,
5311BD4B1EA581E500525281 /* WasmOMGPlan.h in Headers */,
53C6FEEF1E8ADFA900B18425 /* WasmOpcodeOrigin.h in Headers */,
E39D8B2E23021E2600265852 /* WasmOperations.h in Headers */,
Modified: trunk/Source/_javascript_Core/Sources.txt (287378 => 287379)
--- trunk/Source/_javascript_Core/Sources.txt 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/Sources.txt 2021-12-23 01:12:14 UTC (rev 287379)
@@ -1098,8 +1098,8 @@
wasm/WasmModule.cpp
wasm/WasmModuleInformation.cpp
wasm/WasmNameSectionParser.cpp
-wasm/WasmOMGForOSREntryPlan.cpp
wasm/WasmOMGPlan.cpp
+wasm/WasmOSREntryPlan.cpp
wasm/WasmOpcodeOrigin.cpp
wasm/WasmOperations.cpp
wasm/WasmPageCount.cpp
Modified: trunk/Source/_javascript_Core/assembler/MacroAssemblerCodeRef.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/assembler/MacroAssemblerCodeRef.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/assembler/MacroAssemblerCodeRef.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -77,15 +77,10 @@
{
if (Options::asyncDisassembly() || Options::dumpDisassembly() || Options::dumpWasmDisassembly())
return true;
- switch (mode) {
- case Wasm::CompilationMode::BBQMode:
+ if (Wasm::isAnyBBQ(mode))
return Options::dumpBBQDisassembly();
- case Wasm::CompilationMode::OMGMode:
- case Wasm::CompilationMode::OMGForOSREntryMode:
+ if (Wasm::isAnyOMG(mode))
return Options::dumpOMGDisassembly();
- default:
- break;
- }
return false;
}
Modified: trunk/Source/_javascript_Core/jsc.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/jsc.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/jsc.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -309,7 +309,7 @@
static JSC_DECLARE_HOST_FUNCTION(functionNoOSRExitFuzzing);
static JSC_DECLARE_HOST_FUNCTION(functionOptimizeNextInvocation);
static JSC_DECLARE_HOST_FUNCTION(functionNumberOfDFGCompiles);
-static JSC_DECLARE_HOST_FUNCTION(functionCallerIsOMGCompiled);
+static JSC_DECLARE_HOST_FUNCTION(functionCallerIsBBQOrOMGCompiled);
static JSC_DECLARE_HOST_FUNCTION(functionJSCOptions);
static JSC_DECLARE_HOST_FUNCTION(functionReoptimizationRetryCount);
static JSC_DECLARE_HOST_FUNCTION(functionTransferArrayBuffer);
@@ -559,7 +559,7 @@
addFunction(vm, "noFTL", functionNoFTL, 1);
addFunction(vm, "noOSRExitFuzzing", functionNoOSRExitFuzzing, 1);
addFunction(vm, "numberOfDFGCompiles", functionNumberOfDFGCompiles, 1);
- addFunction(vm, "callerIsOMGCompiled", functionCallerIsOMGCompiled, 0);
+ addFunction(vm, "callerIsBBQOrOMGCompiled", functionCallerIsBBQOrOMGCompiled, 0);
addFunction(vm, "jscOptions", functionJSCOptions, 0);
addFunction(vm, "optimizeNextInvocation", functionOptimizeNextInvocation, 1);
addFunction(vm, "reoptimizationRetryCount", functionReoptimizationRetryCount, 1);
@@ -1886,7 +1886,7 @@
return JSValue::encode(numberOfDFGCompiles(globalObject, callFrame));
}
-JSC_DEFINE_HOST_FUNCTION(functionCallerIsOMGCompiled, (JSGlobalObject* globalObject, CallFrame* callFrame))
+JSC_DEFINE_HOST_FUNCTION(functionCallerIsBBQOrOMGCompiled, (JSGlobalObject* globalObject, CallFrame* callFrame))
{
VM& vm = globalObject->vm();
auto scope = DECLARE_THROW_SCOPE(vm);
@@ -1906,7 +1906,7 @@
ASSERT(wasmFrame.callerFrame()->callee().isWasm());
#if ENABLE(WEBASSEMBLY)
auto mode = wasmFrame.callerFrame()->callee().asWasmCallee()->compilationMode();
- return JSValue::encode(jsBoolean(mode == Wasm::CompilationMode::OMGMode || mode == Wasm::CompilationMode::OMGForOSREntryMode));
+ return JSValue::encode(jsBoolean(isAnyBBQ(mode) || isAnyOMG(mode)));
#endif
RELEASE_ASSERT_NOT_REACHED();
}
Modified: trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmB3IRGenerator.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -535,7 +535,7 @@
FunctionParser<B3IRGenerator>* m_parser { nullptr };
const ModuleInformation& m_info;
const MemoryMode m_mode { MemoryMode::BoundsChecking };
- const CompilationMode m_compilationMode { CompilationMode::BBQMode };
+ const CompilationMode m_compilationMode;
const unsigned m_functionIndex { UINT_MAX };
const unsigned m_loopIndexForOSREntry { UINT_MAX };
TierUpCount* m_tierUp { nullptr };
@@ -782,7 +782,7 @@
emitEntryTierUpCheck();
- if (m_compilationMode == CompilationMode::OMGForOSREntryMode)
+ if (isOSREntry(m_compilationMode))
m_currentBlock = m_proc.addBlock();
}
@@ -3280,7 +3280,7 @@
// optLevel=1.
procedure.setNeedsUsedRegisters(false);
- procedure.setOptLevel(compilationMode == CompilationMode::BBQMode
+ procedure.setOptLevel(isAnyBBQ(compilationMode)
? Options::webAssemblyBBQB3OptimizationLevel()
: Options::webAssemblyOMGOptimizationLevel());
Modified: trunk/Source/_javascript_Core/wasm/WasmCallee.h (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmCallee.h 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmCallee.h 2021-12-23 01:12:14 UTC (rev 287379)
@@ -46,10 +46,6 @@
namespace Wasm {
-#if ENABLE(WEBASSEMBLY_B3JIT)
-class OMGForOSREntryCallee;
-#endif
-
class Callee : public ThreadSafeRefCounted<Callee> {
WTF_MAKE_FAST_ALLOCATED;
@@ -67,13 +63,6 @@
bool hasExceptionHandlers() const { return !m_exceptionHandlers.isEmpty(); }
-#if ENABLE(WEBASSEMBLY_B3JIT)
- virtual void setOSREntryCallee(Ref<OMGForOSREntryCallee>&&, MemoryMode)
- {
- RELEASE_ASSERT_NOT_REACHED();
- }
-#endif
-
void dump(PrintStream&) const;
virtual FunctionCodeBlock* llintFunctionCodeBlock() const { return nullptr; }
@@ -161,11 +150,11 @@
}
};
-class OMGForOSREntryCallee final : public OptimizingJITCallee {
+class OSREntryCallee final : public OptimizingJITCallee {
public:
- static Ref<OMGForOSREntryCallee> create(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations)
+ static Ref<OSREntryCallee> create(CompilationMode compilationMode, Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations)
{
- return adoptRef(*new OMGForOSREntryCallee(WTFMove(entrypoint), index, WTFMove(name), osrEntryScratchBufferSize, loopIndex, WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations)));
+ return adoptRef(*new OSREntryCallee(compilationMode, WTFMove(entrypoint), index, WTFMove(name), osrEntryScratchBufferSize, loopIndex, WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations)));
}
unsigned osrEntryScratchBufferSize() const { return m_osrEntryScratchBufferSize; }
@@ -172,8 +161,8 @@
uint32_t loopIndex() const { return m_loopIndex; }
private:
- OMGForOSREntryCallee(Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations)
- : OptimizingJITCallee(Wasm::CompilationMode::OMGForOSREntryMode, WTFMove(entrypoint), index, WTFMove(name), WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations))
+ OSREntryCallee(CompilationMode compilationMode, Wasm::Entrypoint&& entrypoint, size_t index, std::pair<const Name*, RefPtr<NameSection>>&& name, unsigned osrEntryScratchBufferSize, uint32_t loopIndex, Vector<UnlinkedWasmToWasmCall>&& unlinkedCalls, StackMaps&& stackmaps, Vector<UnlinkedHandlerInfo>&& exceptionHandlers, Vector<CodeLocationLabel<ExceptionHandlerPtrTag>>&& exceptionHandlerLocations)
+ : OptimizingJITCallee(compilationMode, WTFMove(entrypoint), index, WTFMove(name), WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations))
, m_osrEntryScratchBufferSize(osrEntryScratchBufferSize)
, m_loopIndex(loopIndex)
{
@@ -190,8 +179,8 @@
return adoptRef(*new BBQCallee(WTFMove(entrypoint), index, WTFMove(name), WTFMove(tierUpCount), WTFMove(unlinkedCalls), WTFMove(stackmaps), WTFMove(exceptionHandlers), WTFMove(exceptionHandlerLocations)));
}
- OMGForOSREntryCallee* osrEntryCallee() { return m_osrEntryCallee.get(); }
- void setOSREntryCallee(Ref<OMGForOSREntryCallee>&& osrEntryCallee, MemoryMode) final
+ OSREntryCallee* osrEntryCallee() { return m_osrEntryCallee.get(); }
+ void setOSREntryCallee(Ref<OSREntryCallee>&& osrEntryCallee, MemoryMode)
{
m_osrEntryCallee = WTFMove(osrEntryCallee);
}
@@ -214,7 +203,7 @@
{
}
- RefPtr<OMGForOSREntryCallee> m_osrEntryCallee;
+ RefPtr<OSREntryCallee> m_osrEntryCallee;
RefPtr<OMGCallee> m_replacement;
std::unique_ptr<TierUpCount> m_tierUpCount;
bool m_didStartCompilingOSREntryCallee { false };
@@ -242,8 +231,8 @@
m_replacements[static_cast<uint8_t>(mode)] = WTFMove(replacement);
}
- OMGForOSREntryCallee* osrEntryCallee(MemoryMode mode) { return m_osrEntryCallees[static_cast<uint8_t>(mode)].get(); }
- void setOSREntryCallee(Ref<OMGForOSREntryCallee>&& osrEntryCallee, MemoryMode mode) final
+ OSREntryCallee* osrEntryCallee(MemoryMode mode) { return m_osrEntryCallees[static_cast<uint8_t>(mode)].get(); }
+ void setOSREntryCallee(Ref<OSREntryCallee>&& osrEntryCallee, MemoryMode mode)
{
m_osrEntryCallees[static_cast<uint8_t>(mode)] = WTFMove(osrEntryCallee);
}
@@ -259,7 +248,7 @@
#if ENABLE(WEBASSEMBLY_B3JIT)
RefPtr<JITCallee> m_replacements[Wasm::NumberOfMemoryModes];
- RefPtr<OMGForOSREntryCallee> m_osrEntryCallees[Wasm::NumberOfMemoryModes];
+ RefPtr<OSREntryCallee> m_osrEntryCallees[Wasm::NumberOfMemoryModes];
#endif
std::unique_ptr<FunctionCodeBlock> m_codeBlock;
MacroAssemblerCodePtr<WasmEntryPtrTag> m_entrypoint;
Modified: trunk/Source/_javascript_Core/wasm/WasmCalleeGroup.h (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmCalleeGroup.h 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmCalleeGroup.h 2021-12-23 01:12:14 UTC (rev 287379)
@@ -163,7 +163,7 @@
#if ENABLE(WEBASSEMBLY_B3JIT)
friend class BBQPlan;
friend class OMGPlan;
- friend class OMGForOSREntryPlan;
+ friend class OSREntryPlan;
#endif
CalleeGroup(Context*, MemoryMode, ModuleInformation&, RefPtr<LLIntCallees>);
Modified: trunk/Source/_javascript_Core/wasm/WasmCompilationMode.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmCompilationMode.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmCompilationMode.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -38,6 +38,8 @@
return "LLInt";
case CompilationMode::BBQMode:
return "BBQ";
+ case CompilationMode::BBQForOSREntryMode:
+ return "BBQForOSREntry";
case CompilationMode::OMGMode:
return "OMG";
case CompilationMode::OMGForOSREntryMode:
Modified: trunk/Source/_javascript_Core/wasm/WasmCompilationMode.h (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmCompilationMode.h 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmCompilationMode.h 2021-12-23 01:12:14 UTC (rev 287379)
@@ -30,6 +30,7 @@
enum class CompilationMode : uint8_t {
LLIntMode,
BBQMode,
+ BBQForOSREntryMode,
OMGMode,
OMGForOSREntryMode,
EmbedderEntrypointMode,
@@ -37,4 +38,46 @@
const char* makeString(CompilationMode);
+constexpr inline bool isOSREntry(CompilationMode compilationMode)
+{
+ switch (compilationMode) {
+ case CompilationMode::LLIntMode:
+ case CompilationMode::BBQMode:
+ case CompilationMode::OMGMode:
+ case CompilationMode::EmbedderEntrypointMode:
+ return false;
+ case CompilationMode::BBQForOSREntryMode:
+ case CompilationMode::OMGForOSREntryMode:
+ return true;
+ }
+}
+
+constexpr inline bool isAnyBBQ(CompilationMode compilationMode)
+{
+ switch (compilationMode) {
+ case CompilationMode::BBQMode:
+ case CompilationMode::BBQForOSREntryMode:
+ return true;
+ case CompilationMode::OMGForOSREntryMode:
+ case CompilationMode::LLIntMode:
+ case CompilationMode::OMGMode:
+ case CompilationMode::EmbedderEntrypointMode:
+ return false;
+ }
+}
+
+constexpr inline bool isAnyOMG(CompilationMode compilationMode)
+{
+ switch (compilationMode) {
+ case CompilationMode::OMGMode:
+ case CompilationMode::OMGForOSREntryMode:
+ return true;
+ case CompilationMode::BBQMode:
+ case CompilationMode::BBQForOSREntryMode:
+ case CompilationMode::LLIntMode:
+ case CompilationMode::EmbedderEntrypointMode:
+ return false;
+ }
+}
+
} } // namespace JSC::Wasm
Deleted: trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -1,152 +0,0 @@
-/*
- * Copyright (C) 2019-2021 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "config.h"
-#include "WasmOMGForOSREntryPlan.h"
-
-#if ENABLE(WEBASSEMBLY_B3JIT)
-
-#include "JITCompilation.h"
-#include "LinkBuffer.h"
-#include "WasmB3IRGenerator.h"
-#include "WasmCallee.h"
-#include "WasmMachineThreads.h"
-#include "WasmNameSection.h"
-#include "WasmSignatureInlines.h"
-#include <wtf/DataLog.h>
-#include <wtf/Locker.h>
-#include <wtf/StdLibExtras.h>
-
-namespace JSC { namespace Wasm {
-
-namespace WasmOMGForOSREntryPlanInternal {
-static constexpr bool verbose = false;
-}
-
-OMGForOSREntryPlan::OMGForOSREntryPlan(Context* context, Ref<Module>&& module, Ref<Callee>&& callee, uint32_t functionIndex, uint32_t loopIndex, MemoryMode mode, CompletionTask&& task)
- : Base(context, const_cast<ModuleInformation&>(module->moduleInformation()), WTFMove(task))
- , m_module(WTFMove(module))
- , m_calleeGroup(*m_module->calleeGroupFor(mode))
- , m_callee(WTFMove(callee))
- , m_functionIndex(functionIndex)
- , m_loopIndex(loopIndex)
-{
- ASSERT(Options::useOMGJIT());
- setMode(mode);
- ASSERT(m_calleeGroup->runnable());
- ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(m_mode));
- dataLogLnIf(WasmOMGForOSREntryPlanInternal::verbose, "Starting OMGForOSREntry plan for ", functionIndex, " of module: ", RawPointer(&m_module.get()));
-}
-
-void OMGForOSREntryPlan::work(CompilationEffort)
-{
- ASSERT(m_calleeGroup->runnable());
- ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(mode()));
- const FunctionData& function = m_moduleInformation->functions[m_functionIndex];
-
- const uint32_t functionIndexSpace = m_functionIndex + m_module->moduleInformation().importFunctionCount();
- ASSERT(functionIndexSpace < m_module->moduleInformation().functionIndexSpaceSize());
-
- SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[m_functionIndex];
- const Signature& signature = SignatureInformation::get(signatureIndex);
-
- Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
- CompilationContext context;
- unsigned osrEntryScratchBufferSize = 0;
- auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, CompilationMode::OMGForOSREntryMode, m_functionIndex, m_loopIndex);
-
- if (UNLIKELY(!parseAndCompileResult)) {
- Locker locker { m_lock };
- fail(makeString(parseAndCompileResult.error(), "when trying to tier up ", String::number(m_functionIndex)));
- return;
- }
-
- Entrypoint omgEntrypoint;
- LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, LinkBuffer::Profile::Wasm, JITCompilationCanFail);
- if (UNLIKELY(linkBuffer.didFailToAllocate())) {
- Locker locker { m_lock };
- Base::fail(makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
- return;
- }
-
- InternalFunction* internalFunction = parseAndCompileResult->get();
- Vector<CodeLocationLabel<ExceptionHandlerPtrTag>> exceptionHandlerLocations;
- computeExceptionHandlerLocations(exceptionHandlerLocations, internalFunction, context, linkBuffer);
-
- omgEntrypoint.compilation = makeUnique<Compilation>(
- FINALIZE_CODE_IF(context.procedure->shouldDumpIR() || shouldDumpDisassemblyFor(CompilationMode::OMGForOSREntryMode), linkBuffer, JITCompilationPtrTag, "WebAssembly OMGForOSREntry function[%i] %s name %s", m_functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
- WTFMove(context.wasmEntrypointByproducts));
-
- omgEntrypoint.calleeSaveRegisters = WTFMove(internalFunction->entrypoint.calleeSaveRegisters);
-
- ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(mode()));
- Ref<OMGForOSREntryCallee> callee = OMGForOSREntryCallee::create(WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), osrEntryScratchBufferSize, m_loopIndex, WTFMove(unlinkedCalls), WTFMove(internalFunction->stackmaps), WTFMove(internalFunction->exceptionHandlers), WTFMove(exceptionHandlerLocations));
- {
- MacroAssembler::repatchPointer(internalFunction->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
-
- Locker locker { m_calleeGroup->m_lock };
- for (auto& call : callee->wasmToWasmCallsites()) {
- MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
- if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
- entrypoint = m_calleeGroup->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
- else
- entrypoint = m_calleeGroup->wasmEntrypointCalleeFromFunctionIndexSpace(locker, call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
-
- MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
- }
-
- resetInstructionCacheOnAllThreads();
- WTF::storeStoreFence();
-
- {
- switch (m_callee->compilationMode()) {
- case CompilationMode::LLIntMode: {
- LLIntCallee* llintCallee = static_cast<LLIntCallee*>(m_callee.ptr());
- Locker locker { llintCallee->tierUpCounter().m_lock };
- llintCallee->setOSREntryCallee(callee.copyRef(), mode());
- llintCallee->tierUpCounter().m_loopCompilationStatus = LLIntTierUpCounter::CompilationStatus::Compiled;
- break;
- }
- case CompilationMode::BBQMode: {
- BBQCallee* bbqCallee = static_cast<BBQCallee*>(m_callee.ptr());
- Locker locker { bbqCallee->tierUpCount()->getLock() };
- bbqCallee->setOSREntryCallee(callee.copyRef(), mode());
- bbqCallee->tierUpCount()->osrEntryTriggers()[m_loopIndex] = TierUpCount::TriggerReason::CompilationDone;
- bbqCallee->tierUpCount()->m_compilationStatusForOMGForOSREntry = TierUpCount::CompilationStatus::Compiled;
- break;
- }
- default:
- RELEASE_ASSERT_NOT_REACHED();
- }
- }
- }
- dataLogLnIf(WasmOMGForOSREntryPlanInternal::verbose, "Finished OMGForOSREntry ", m_functionIndex);
- Locker locker { m_lock };
- complete();
-}
-
-} } // namespace JSC::Wasm
-
-#endif // ENABLE(WEBASSEMBLY_B3JIT)
Deleted: trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.h (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.h 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.h 2021-12-23 01:12:14 UTC (rev 287379)
@@ -1,74 +0,0 @@
-/*
- * Copyright (C) 2019 Apple Inc. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#pragma once
-
-#if ENABLE(WEBASSEMBLY_B3JIT)
-
-#include "WasmCallee.h"
-#include "WasmContext.h"
-#include "WasmModule.h"
-#include "WasmOperations.h"
-#include "WasmPlan.h"
-
-namespace JSC {
-
-class CallLinkInfo;
-
-namespace Wasm {
-
-class OMGForOSREntryPlan final : public Plan {
-public:
- using Base = Plan;
-
- bool hasWork() const final { return !m_completed; }
- void work(CompilationEffort) final;
- bool multiThreaded() const final { return false; }
-
- // Note: CompletionTask should not hold a reference to the Plan otherwise there will be a reference cycle.
- OMGForOSREntryPlan(Context*, Ref<Module>&&, Ref<Callee>&&, uint32_t functionIndex, uint32_t loopIndex, MemoryMode, CompletionTask&&);
-
-private:
- // For some reason friendship doesn't extend to parent classes...
- using Base::m_lock;
-
- bool isComplete() const final { return m_completed; }
- void complete() WTF_REQUIRES_LOCK(m_lock) final
- {
- m_completed = true;
- runCompletionTasks();
- }
-
- Ref<Module> m_module;
- Ref<CalleeGroup> m_calleeGroup;
- Ref<Callee> m_callee;
- bool m_completed { false };
- uint32_t m_functionIndex;
- uint32_t m_loopIndex;
-};
-
-} } // namespace JSC::Wasm
-
-#endif // ENABLE(WEBASSEMBLY_B3JIT)
Copied: trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.cpp (from rev 287378, trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.cpp) (0 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.cpp (rev 0)
+++ trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2019-2021 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+#include "WasmOSREntryPlan.h"
+
+#if ENABLE(WEBASSEMBLY_B3JIT)
+
+#include "JITCompilation.h"
+#include "LinkBuffer.h"
+#include "WasmB3IRGenerator.h"
+#include "WasmCallee.h"
+#include "WasmMachineThreads.h"
+#include "WasmNameSection.h"
+#include "WasmSignatureInlines.h"
+#include <wtf/DataLog.h>
+#include <wtf/Locker.h>
+#include <wtf/StdLibExtras.h>
+
+namespace JSC { namespace Wasm {
+
+namespace WasmOSREntryPlanInternal {
+static constexpr bool verbose = false;
+}
+
+OSREntryPlan::OSREntryPlan(Context* context, Ref<Module>&& module, Ref<Callee>&& callee, uint32_t functionIndex, uint32_t loopIndex, MemoryMode mode, CompletionTask&& task)
+ : Base(context, const_cast<ModuleInformation&>(module->moduleInformation()), WTFMove(task))
+ , m_module(WTFMove(module))
+ , m_calleeGroup(*m_module->calleeGroupFor(mode))
+ , m_callee(WTFMove(callee))
+ , m_functionIndex(functionIndex)
+ , m_loopIndex(loopIndex)
+{
+ ASSERT(Options::useOMGJIT());
+ setMode(mode);
+ ASSERT(m_calleeGroup->runnable());
+ ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(m_mode));
+ dataLogLnIf(WasmOSREntryPlanInternal::verbose, "Starting OMGForOSREntry plan for ", functionIndex, " of module: ", RawPointer(&m_module.get()));
+}
+
+void OSREntryPlan::work(CompilationEffort)
+{
+ ASSERT(m_calleeGroup->runnable());
+ ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(mode()));
+ const FunctionData& function = m_moduleInformation->functions[m_functionIndex];
+
+ const uint32_t functionIndexSpace = m_functionIndex + m_module->moduleInformation().importFunctionCount();
+ ASSERT(functionIndexSpace < m_module->moduleInformation().functionIndexSpaceSize());
+
+ SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[m_functionIndex];
+ const Signature& signature = SignatureInformation::get(signatureIndex);
+
+ CompilationMode targetCompilationMode = m_callee->compilationMode() == CompilationMode::LLIntMode ? CompilationMode::BBQForOSREntryMode : CompilationMode::OMGForOSREntryMode;
+
+ Vector<UnlinkedWasmToWasmCall> unlinkedCalls;
+ CompilationContext context;
+ unsigned osrEntryScratchBufferSize = 0;
+ auto parseAndCompileResult = parseAndCompile(context, function, signature, unlinkedCalls, osrEntryScratchBufferSize, m_moduleInformation.get(), m_mode, targetCompilationMode, m_functionIndex, m_loopIndex);
+
+ if (UNLIKELY(!parseAndCompileResult)) {
+ Locker locker { m_lock };
+ fail(makeString(parseAndCompileResult.error(), "when trying to tier up ", String::number(m_functionIndex)));
+ return;
+ }
+
+ Entrypoint omgEntrypoint;
+ LinkBuffer linkBuffer(*context.wasmEntrypointJIT, nullptr, LinkBuffer::Profile::Wasm, JITCompilationCanFail);
+ if (UNLIKELY(linkBuffer.didFailToAllocate())) {
+ Locker locker { m_lock };
+ Base::fail(makeString("Out of executable memory while tiering up function at index ", String::number(m_functionIndex)));
+ return;
+ }
+
+ InternalFunction* internalFunction = parseAndCompileResult->get();
+ Vector<CodeLocationLabel<ExceptionHandlerPtrTag>> exceptionHandlerLocations;
+ computeExceptionHandlerLocations(exceptionHandlerLocations, internalFunction, context, linkBuffer);
+
+ omgEntrypoint.compilation = makeUnique<Compilation>(
+ FINALIZE_CODE_IF(context.procedure->shouldDumpIR() || shouldDumpDisassemblyFor(targetCompilationMode), linkBuffer, JITCompilationPtrTag, "WebAssembly OMGForOSREntry function[%i] %s name %s", m_functionIndex, signature.toString().ascii().data(), makeString(IndexOrName(functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace))).ascii().data()),
+ WTFMove(context.wasmEntrypointByproducts));
+
+ omgEntrypoint.calleeSaveRegisters = WTFMove(internalFunction->entrypoint.calleeSaveRegisters);
+
+ ASSERT(m_calleeGroup.ptr() == m_module->calleeGroupFor(mode()));
+ Ref<OSREntryCallee> callee = OSREntryCallee::create(targetCompilationMode, WTFMove(omgEntrypoint), functionIndexSpace, m_moduleInformation->nameSection->get(functionIndexSpace), osrEntryScratchBufferSize, m_loopIndex, WTFMove(unlinkedCalls), WTFMove(internalFunction->stackmaps), WTFMove(internalFunction->exceptionHandlers), WTFMove(exceptionHandlerLocations));
+ {
+ MacroAssembler::repatchPointer(internalFunction->calleeMoveLocation, CalleeBits::boxWasm(callee.ptr()));
+
+ Locker locker { m_calleeGroup->m_lock };
+ for (auto& call : callee->wasmToWasmCallsites()) {
+ MacroAssemblerCodePtr<WasmEntryPtrTag> entrypoint;
+ if (call.functionIndexSpace < m_module->moduleInformation().importFunctionCount())
+ entrypoint = m_calleeGroup->m_wasmToWasmExitStubs[call.functionIndexSpace].code();
+ else
+ entrypoint = m_calleeGroup->wasmEntrypointCalleeFromFunctionIndexSpace(locker, call.functionIndexSpace).entrypoint().retagged<WasmEntryPtrTag>();
+
+ MacroAssembler::repatchNearCall(call.callLocation, CodeLocationLabel<WasmEntryPtrTag>(entrypoint));
+ }
+
+ resetInstructionCacheOnAllThreads();
+ WTF::storeStoreFence();
+
+ {
+ switch (m_callee->compilationMode()) {
+ case CompilationMode::LLIntMode: {
+ LLIntCallee* llintCallee = static_cast<LLIntCallee*>(m_callee.ptr());
+ Locker locker { llintCallee->tierUpCounter().m_lock };
+ llintCallee->setOSREntryCallee(callee.copyRef(), mode());
+ llintCallee->tierUpCounter().m_loopCompilationStatus = LLIntTierUpCounter::CompilationStatus::Compiled;
+ break;
+ }
+ case CompilationMode::BBQMode: {
+ BBQCallee* bbqCallee = static_cast<BBQCallee*>(m_callee.ptr());
+ Locker locker { bbqCallee->tierUpCount()->getLock() };
+ bbqCallee->setOSREntryCallee(callee.copyRef(), mode());
+ bbqCallee->tierUpCount()->osrEntryTriggers()[m_loopIndex] = TierUpCount::TriggerReason::CompilationDone;
+ bbqCallee->tierUpCount()->m_compilationStatusForOMGForOSREntry = TierUpCount::CompilationStatus::Compiled;
+ break;
+ }
+ default:
+ RELEASE_ASSERT_NOT_REACHED();
+ }
+ }
+ }
+ dataLogLnIf(WasmOSREntryPlanInternal::verbose, "Finished OMGForOSREntry ", m_functionIndex);
+ Locker locker { m_lock };
+ complete();
+}
+
+} } // namespace JSC::Wasm
+
+#endif // ENABLE(WEBASSEMBLY_B3JIT)
Copied: trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.h (from rev 287378, trunk/Source/_javascript_Core/wasm/WasmOMGForOSREntryPlan.h) (0 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.h (rev 0)
+++ trunk/Source/_javascript_Core/wasm/WasmOSREntryPlan.h 2021-12-23 01:12:14 UTC (rev 287379)
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2019 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#if ENABLE(WEBASSEMBLY_B3JIT)
+
+#include "WasmCallee.h"
+#include "WasmContext.h"
+#include "WasmModule.h"
+#include "WasmOperations.h"
+#include "WasmPlan.h"
+
+namespace JSC {
+
+class CallLinkInfo;
+
+namespace Wasm {
+
+class OSREntryPlan final : public Plan {
+public:
+ using Base = Plan;
+
+ bool hasWork() const final { return !m_completed; }
+ void work(CompilationEffort) final;
+ bool multiThreaded() const final { return false; }
+
+ // Note: CompletionTask should not hold a reference to the Plan otherwise there will be a reference cycle.
+ OSREntryPlan(Context*, Ref<Module>&&, Ref<Callee>&&, uint32_t functionIndex, uint32_t loopIndex, MemoryMode, CompletionTask&&);
+
+private:
+ // For some reason friendship doesn't extend to parent classes...
+ using Base::m_lock;
+
+ bool isComplete() const final { return m_completed; }
+ void complete() WTF_REQUIRES_LOCK(m_lock) final
+ {
+ m_completed = true;
+ runCompletionTasks();
+ }
+
+ Ref<Module> m_module;
+ Ref<CalleeGroup> m_calleeGroup;
+ Ref<Callee> m_callee;
+ bool m_completed { false };
+ uint32_t m_functionIndex;
+ uint32_t m_loopIndex;
+};
+
+} } // namespace JSC::Wasm
+
+#endif // ENABLE(WEBASSEMBLY_B3JIT)
Modified: trunk/Source/_javascript_Core/wasm/WasmOperations.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmOperations.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmOperations.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -47,9 +47,9 @@
#include "WasmInstance.h"
#include "WasmMemory.h"
#include "WasmModuleInformation.h"
-#include "WasmOMGForOSREntryPlan.h"
#include "WasmOMGPlan.h"
#include "WasmOSREntryData.h"
+#include "WasmOSREntryPlan.h"
#include "WasmWorklist.h"
#include <wtf/CheckedArithmetic.h>
#include <wtf/DataLog.h>
@@ -157,7 +157,7 @@
}
SUPPRESS_ASAN
-static void doOSREntry(Instance* instance, Probe::Context& context, BBQCallee& callee, OMGForOSREntryCallee& osrEntryCallee, OSREntryData& osrEntryData)
+static void doOSREntry(Instance* instance, Probe::Context& context, BBQCallee& callee, OSREntryCallee& osrEntryCallee, OSREntryData& osrEntryData)
{
auto returnWithoutOSREntry = [&] {
context.gpr(GPRInfo::argumentGPR0) = 0;
@@ -247,7 +247,7 @@
return returnWithoutOSREntry();
}
- dataLogLnIf(Options::verboseOSR(), "Consider OMGForOSREntryPlan for [", functionIndex, "] loopIndex#", loopIndex, " with executeCounter = ", tierUp, " ", RawPointer(callee.replacement()));
+ dataLogLnIf(Options::verboseOSR(), "Consider OSREntryPlan for [", functionIndex, "] loopIndex#", loopIndex, " with executeCounter = ", tierUp, " ", RawPointer(callee.replacement()));
if (!Options::useWebAssemblyOSR()) {
if (shouldTriggerOMGCompile(tierUp, callee.replacement(), functionIndex))
@@ -307,7 +307,7 @@
return returnWithoutOSREntry();
}
- if (OMGForOSREntryCallee* osrEntryCallee = callee.osrEntryCallee()) {
+ if (OSREntryCallee* osrEntryCallee = callee.osrEntryCallee()) {
if (osrEntryCallee->loopIndex() == loopIndex)
return doOSREntry(instance, context, callee, *osrEntryCallee, osrEntryData);
}
@@ -322,7 +322,7 @@
return returnWithoutOSREntry();
}
- if (OMGForOSREntryCallee* osrEntryCallee = callee.osrEntryCallee()) {
+ if (OSREntryCallee* osrEntryCallee = callee.osrEntryCallee()) {
if (osrEntryCallee->loopIndex() == loopIndex)
return doOSREntry(instance, context, callee, *osrEntryCallee, osrEntryData);
tierUp.dontOptimizeAnytimeSoon(functionIndex);
@@ -347,7 +347,7 @@
uint32_t currentLoopIndex = tierUp.outerLoops()[loopIndex];
Locker locker { tierUp.getLock() };
- // We already started OMGForOSREntryPlan.
+ // We already started OSREntryPlan.
if (callee.didStartCompilingOSREntryCallee())
return false;
@@ -390,7 +390,7 @@
if (startOSREntryCompilation) {
dataLogLnIf(Options::verboseOSR(), "triggerOMGOSR for ", functionIndex);
- Ref<Plan> plan = adoptRef(*new OMGForOSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::BBQCallee>(callee), functionIndex, loopIndex, calleeGroup.mode(), Plan::dontFinalize()));
+ Ref<Plan> plan = adoptRef(*new OSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::BBQCallee>(callee), functionIndex, loopIndex, calleeGroup.mode(), Plan::dontFinalize()));
ensureWorklist().enqueue(plan.copyRef());
if (UNLIKELY(!Options::useConcurrentJIT()))
plan->waitForCompletion();
@@ -398,7 +398,7 @@
tierUp.setOptimizationThresholdBasedOnCompilationResult(functionIndex, CompilationDeferred);
}
- OMGForOSREntryCallee* osrEntryCallee = callee.osrEntryCallee();
+ OSREntryCallee* osrEntryCallee = callee.osrEntryCallee();
if (!osrEntryCallee) {
tierUp.setOptimizationThresholdBasedOnCompilationResult(functionIndex, CompilationDeferred);
return returnWithoutOSREntry();
Modified: trunk/Source/_javascript_Core/wasm/WasmPlan.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmPlan.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmPlan.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -141,13 +141,13 @@
LLIntCallee& llintCallee = calleeGroup.m_llintCallees->at(i).get();
if (JITCallee* replacementCallee = llintCallee.replacement(calleeGroup.mode()))
stageRepatch(replacementCallee->wasmToWasmCallsites());
- if (OMGForOSREntryCallee* osrEntryCallee = llintCallee.osrEntryCallee(calleeGroup.mode()))
+ if (OSREntryCallee* osrEntryCallee = llintCallee.osrEntryCallee(calleeGroup.mode()))
stageRepatch(osrEntryCallee->wasmToWasmCallsites());
}
if (BBQCallee* bbqCallee = calleeGroup.bbqCallee(calleeGroupLocker, i)) {
if (OMGCallee* replacementCallee = bbqCallee->replacement())
stageRepatch(replacementCallee->wasmToWasmCallsites());
- if (OMGForOSREntryCallee* osrEntryCallee = bbqCallee->osrEntryCallee())
+ if (OSREntryCallee* osrEntryCallee = bbqCallee->osrEntryCallee())
stageRepatch(osrEntryCallee->wasmToWasmCallsites());
}
}
@@ -176,13 +176,13 @@
LLIntCallee& llintCallee = calleeGroup.m_llintCallees->at(i).get();
if (JITCallee* replacementCallee = llintCallee.replacement(calleeGroup.mode()))
repatchCalls(replacementCallee->wasmToWasmCallsites());
- if (OMGForOSREntryCallee* osrEntryCallee = llintCallee.osrEntryCallee(calleeGroup.mode()))
+ if (OSREntryCallee* osrEntryCallee = llintCallee.osrEntryCallee(calleeGroup.mode()))
repatchCalls(osrEntryCallee->wasmToWasmCallsites());
}
if (BBQCallee* bbqCallee = calleeGroup.bbqCallee(calleeGroupLocker, i)) {
if (OMGCallee* replacementCallee = bbqCallee->replacement())
repatchCalls(replacementCallee->wasmToWasmCallsites());
- if (OMGForOSREntryCallee* osrEntryCallee = bbqCallee->osrEntryCallee())
+ if (OSREntryCallee* osrEntryCallee = bbqCallee->osrEntryCallee())
repatchCalls(osrEntryCallee->wasmToWasmCallsites());
}
}
Modified: trunk/Source/_javascript_Core/wasm/WasmSlowPaths.cpp (287378 => 287379)
--- trunk/Source/_javascript_Core/wasm/WasmSlowPaths.cpp 2021-12-23 00:49:44 UTC (rev 287378)
+++ trunk/Source/_javascript_Core/wasm/WasmSlowPaths.cpp 2021-12-23 01:12:14 UTC (rev 287379)
@@ -38,8 +38,8 @@
#include "WasmFunctionCodeBlock.h"
#include "WasmInstance.h"
#include "WasmModuleInformation.h"
-#include "WasmOMGForOSREntryPlan.h"
#include "WasmOMGPlan.h"
+#include "WasmOSREntryPlan.h"
#include "WasmOperations.h"
#include "WasmSignatureInlines.h"
#include "WasmWorklist.h"
@@ -190,7 +190,7 @@
WASM_RETURN_TWO(nullptr, nullptr);
}
- const auto doOSREntry = [&](Wasm::OMGForOSREntryCallee* osrEntryCallee) {
+ const auto doOSREntry = [&](Wasm::OSREntryCallee* osrEntryCallee) {
if (osrEntryCallee->loopIndex() != osrEntryData.loopIndex)
WASM_RETURN_TWO(nullptr, nullptr);
@@ -227,7 +227,7 @@
}
if (compile) {
- Ref<Wasm::Plan> plan = adoptRef(*static_cast<Wasm::Plan*>(new Wasm::OMGForOSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::Callee>(*callee), codeBlock->functionIndex(), osrEntryData.loopIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize())));
+ Ref<Wasm::Plan> plan = adoptRef(*static_cast<Wasm::Plan*>(new Wasm::OSREntryPlan(instance->context(), Ref<Wasm::Module>(instance->module()), Ref<Wasm::Callee>(*callee), codeBlock->functionIndex(), osrEntryData.loopIndex, instance->memory()->mode(), Wasm::Plan::dontFinalize())));
Wasm::ensureWorklist().enqueue(plan.copyRef());
if (UNLIKELY(!Options::useConcurrentJIT()))
plan->waitForCompletion();
