Commit 4092e56d authored by fpizlo@apple.com's avatar fpizlo@apple.com

DFG should be able to cache closure calls (part 2/2)

https://bugs.webkit.org/show_bug.cgi?id=102662

Reviewed by Gavin Barraclough.

Added caching of calls where the JSFunction* varies, but the Structure* and ExecutableBase*
stay the same. This is accomplished by replacing the branch that compares against a constant
JSFunction* with a jump to a closure call stub. The closure call stub contains a fast path,
and jumps slow directly to the virtual call thunk.

Looks like a 1% win on V8v7.

* CMakeLists.txt:
* GNUmakefile.list.am:
* JavaScriptCore.vcproj/JavaScriptCore/JavaScriptCore.vcproj:
* JavaScriptCore.xcodeproj/project.pbxproj:
* Target.pri:
* bytecode/CallLinkInfo.cpp:
(JSC::CallLinkInfo::unlink):
* bytecode/CallLinkInfo.h:
(CallLinkInfo):
(JSC::CallLinkInfo::isLinked):
(JSC::getCallLinkInfoBytecodeIndex):
* bytecode/CodeBlock.cpp:
(JSC::CodeBlock::finalizeUnconditionally):
(JSC):
(JSC::CodeBlock::findClosureCallForReturnPC):
(JSC::CodeBlock::bytecodeOffset):
(JSC::CodeBlock::codeOriginForReturn):
* bytecode/CodeBlock.h:
(JSC::CodeBlock::getCallLinkInfo):
(CodeBlock):
(JSC::CodeBlock::isIncomingCallAlreadyLinked):
* dfg/DFGJITCompiler.cpp:
(JSC::DFG::JITCompiler::link):
* dfg/DFGJITCompiler.h:
(JSC::DFG::JITCompiler::addJSCall):
(JSC::DFG::JITCompiler::JSCallRecord::JSCallRecord):
(JSCallRecord):
* dfg/DFGOperations.cpp:
* dfg/DFGOperations.h:
* dfg/DFGRepatch.cpp:
(JSC::DFG::linkSlowFor):
(DFG):
(JSC::DFG::dfgLinkFor):
(JSC::DFG::dfgLinkSlowFor):
(JSC::DFG::dfgLinkClosureCall):
* dfg/DFGRepatch.h:
(DFG):
* dfg/DFGSpeculativeJIT32_64.cpp:
(JSC::DFG::SpeculativeJIT::emitCall):
* dfg/DFGSpeculativeJIT64.cpp:
(JSC::DFG::SpeculativeJIT::emitCall):
* dfg/DFGThunks.cpp:
(DFG):
(JSC::DFG::linkClosureCallThunkGenerator):
* dfg/DFGThunks.h:
(DFG):
* heap/Heap.h:
(Heap):
(JSC::Heap::jitStubRoutines):
* heap/JITStubRoutineSet.h:
(JSC::JITStubRoutineSet::size):
(JSC::JITStubRoutineSet::at):
(JITStubRoutineSet):
* jit/ClosureCallStubRoutine.cpp: Added.
(JSC):
(JSC::ClosureCallStubRoutine::ClosureCallStubRoutine):
(JSC::ClosureCallStubRoutine::~ClosureCallStubRoutine):
(JSC::ClosureCallStubRoutine::markRequiredObjectsInternal):
* jit/ClosureCallStubRoutine.h: Added.
(JSC):
(ClosureCallStubRoutine):
(JSC::ClosureCallStubRoutine::structure):
(JSC::ClosureCallStubRoutine::executable):
(JSC::ClosureCallStubRoutine::codeOrigin):
* jit/GCAwareJITStubRoutine.cpp:
(JSC::GCAwareJITStubRoutine::GCAwareJITStubRoutine):
* jit/GCAwareJITStubRoutine.h:
(GCAwareJITStubRoutine):
(JSC::GCAwareJITStubRoutine::isClosureCall):
* jit/JIT.cpp:
(JSC::JIT::privateCompile):



git-svn-id: http://svn.webkit.org/repository/webkit/trunk@135336 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent da1e5cb6
......@@ -139,6 +139,7 @@ SET(JavaScriptCore_SOURCES
interpreter/JSStack.cpp
interpreter/VMInspector.cpp
jit/ClosureCallStubRoutine.cpp
jit/ExecutableAllocator.cpp
jit/HostCallReturnValue.cpp
jit/GCAwareJITStubRoutine.cpp
......
2012-11-20 Filip Pizlo <fpizlo@apple.com>
DFG should be able to cache closure calls (part 2/2)
https://bugs.webkit.org/show_bug.cgi?id=102662
Reviewed by Gavin Barraclough.
Added caching of calls where the JSFunction* varies, but the Structure* and ExecutableBase*
stay the same. This is accomplished by replacing the branch that compares against a constant
JSFunction* with a jump to a closure call stub. The closure call stub contains a fast path,
and jumps slow directly to the virtual call thunk.
Looks like a 1% win on V8v7.
* CMakeLists.txt:
* GNUmakefile.list.am:
* JavaScriptCore.vcproj/JavaScriptCore/JavaScriptCore.vcproj:
* JavaScriptCore.xcodeproj/project.pbxproj:
* Target.pri:
* bytecode/CallLinkInfo.cpp:
(JSC::CallLinkInfo::unlink):
* bytecode/CallLinkInfo.h:
(CallLinkInfo):
(JSC::CallLinkInfo::isLinked):
(JSC::getCallLinkInfoBytecodeIndex):
* bytecode/CodeBlock.cpp:
(JSC::CodeBlock::finalizeUnconditionally):
(JSC):
(JSC::CodeBlock::findClosureCallForReturnPC):
(JSC::CodeBlock::bytecodeOffset):
(JSC::CodeBlock::codeOriginForReturn):
* bytecode/CodeBlock.h:
(JSC::CodeBlock::getCallLinkInfo):
(CodeBlock):
(JSC::CodeBlock::isIncomingCallAlreadyLinked):
* dfg/DFGJITCompiler.cpp:
(JSC::DFG::JITCompiler::link):
* dfg/DFGJITCompiler.h:
(JSC::DFG::JITCompiler::addJSCall):
(JSC::DFG::JITCompiler::JSCallRecord::JSCallRecord):
(JSCallRecord):
* dfg/DFGOperations.cpp:
* dfg/DFGOperations.h:
* dfg/DFGRepatch.cpp:
(JSC::DFG::linkSlowFor):
(DFG):
(JSC::DFG::dfgLinkFor):
(JSC::DFG::dfgLinkSlowFor):
(JSC::DFG::dfgLinkClosureCall):
* dfg/DFGRepatch.h:
(DFG):
* dfg/DFGSpeculativeJIT32_64.cpp:
(JSC::DFG::SpeculativeJIT::emitCall):
* dfg/DFGSpeculativeJIT64.cpp:
(JSC::DFG::SpeculativeJIT::emitCall):
* dfg/DFGThunks.cpp:
(DFG):
(JSC::DFG::linkClosureCallThunkGenerator):
* dfg/DFGThunks.h:
(DFG):
* heap/Heap.h:
(Heap):
(JSC::Heap::jitStubRoutines):
* heap/JITStubRoutineSet.h:
(JSC::JITStubRoutineSet::size):
(JSC::JITStubRoutineSet::at):
(JITStubRoutineSet):
* jit/ClosureCallStubRoutine.cpp: Added.
(JSC):
(JSC::ClosureCallStubRoutine::ClosureCallStubRoutine):
(JSC::ClosureCallStubRoutine::~ClosureCallStubRoutine):
(JSC::ClosureCallStubRoutine::markRequiredObjectsInternal):
* jit/ClosureCallStubRoutine.h: Added.
(JSC):
(ClosureCallStubRoutine):
(JSC::ClosureCallStubRoutine::structure):
(JSC::ClosureCallStubRoutine::executable):
(JSC::ClosureCallStubRoutine::codeOrigin):
* jit/GCAwareJITStubRoutine.cpp:
(JSC::GCAwareJITStubRoutine::GCAwareJITStubRoutine):
* jit/GCAwareJITStubRoutine.h:
(GCAwareJITStubRoutine):
(JSC::GCAwareJITStubRoutine::isClosureCall):
* jit/JIT.cpp:
(JSC::JIT::privateCompile):
2012-11-20 Filip Pizlo <fpizlo@apple.com>
DFG should be able to cache closure calls (part 1/2)
......
......@@ -384,6 +384,8 @@ javascriptcore_sources += \
Source/JavaScriptCore/interpreter/VMInspector.h \
Source/JavaScriptCore/JavaScriptCorePrefix.h \
Source/JavaScriptCore/jit/CompactJITCodeMap.h \
Source/JavaScriptCore/jit/ClosureCallStubRoutine.cpp \
Source/JavaScriptCore/jit/ClosureCallStubRoutine.h \
Source/JavaScriptCore/jit/ExecutableAllocator.cpp \
Source/JavaScriptCore/jit/ExecutableAllocator.h \
Source/JavaScriptCore/jit/GCAwareJITStubRoutine.cpp \
......
......@@ -1929,6 +1929,14 @@
<Filter
Name="jit"
>
<File
RelativePath="..\..\jit\ClosureCallStubRoutine.cpp"
>
</File>
<File
RelativePath="..\..\jit\ClosureCallStubRoutine.h"
>
</File>
<File
RelativePath="..\..\jit\ExecutableAllocator.cpp"
>
......
......@@ -147,6 +147,8 @@
0F63948515E4811B006A597C /* DFGArrayMode.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F63948215E48114006A597C /* DFGArrayMode.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F66E16B14DF3F1600B7B2E4 /* DFGAdjacencyList.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F66E16814DF3F1300B7B2E4 /* DFGAdjacencyList.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F66E16C14DF3F1600B7B2E4 /* DFGEdge.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F66E16914DF3F1300B7B2E4 /* DFGEdge.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F73D7AE165A142D00ACAB71 /* ClosureCallStubRoutine.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F73D7AB165A142A00ACAB71 /* ClosureCallStubRoutine.cpp */; };
0F73D7AF165A143000ACAB71 /* ClosureCallStubRoutine.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F73D7AC165A142A00ACAB71 /* ClosureCallStubRoutine.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F766D2815A8CC1E008F363E /* JITStubRoutine.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F766D2615A8CC1B008F363E /* JITStubRoutine.cpp */; };
0F766D2B15A8CC38008F363E /* JITStubRoutineSet.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F766D2915A8CC34008F363E /* JITStubRoutineSet.cpp */; };
0F766D2C15A8CC3A008F363E /* JITStubRoutineSet.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F766D2A15A8CC34008F363E /* JITStubRoutineSet.h */; settings = {ATTRIBUTES = (Private, ); }; };
......@@ -165,9 +167,9 @@
0F7B294C14C3CD43007C3DB1 /* DFGByteCodeCache.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5F08CC146BE602000472A9 /* DFGByteCodeCache.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F7B294D14C3CD4C007C3DB1 /* DFGCommon.h in Headers */ = {isa = PBXBuildFile; fileRef = 0FC0977E1469EBC400CF2442 /* DFGCommon.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F8023EA1613832B00A0BA45 /* ByValInfo.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F8023E91613832300A0BA45 /* ByValInfo.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F8364B7164B0C110053329A /* DFGBranchDirection.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F8364B5164B0C0E0053329A /* DFGBranchDirection.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F8335B71639C1E6001443B5 /* ArrayAllocationProfile.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F8335B41639C1E3001443B5 /* ArrayAllocationProfile.cpp */; };
0F8335B81639C1EA001443B5 /* ArrayAllocationProfile.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F8335B51639C1E3001443B5 /* ArrayAllocationProfile.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F8364B7164B0C110053329A /* DFGBranchDirection.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F8364B5164B0C0E0053329A /* DFGBranchDirection.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F919D0C157EE09F004A4E7D /* JSSymbolTableObject.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F919D09157EE09D004A4E7D /* JSSymbolTableObject.cpp */; };
0F919D0D157EE0A2004A4E7D /* JSSymbolTableObject.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F919D0A157EE09D004A4E7D /* JSSymbolTableObject.h */; settings = {ATTRIBUTES = (Private, ); }; };
0F919D10157F3329004A4E7D /* JSSegmentedVariableObject.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F919D0E157F3327004A4E7D /* JSSegmentedVariableObject.cpp */; };
......@@ -938,6 +940,8 @@
0F63948215E48114006A597C /* DFGArrayMode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGArrayMode.h; path = dfg/DFGArrayMode.h; sourceTree = "<group>"; };
0F66E16814DF3F1300B7B2E4 /* DFGAdjacencyList.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGAdjacencyList.h; path = dfg/DFGAdjacencyList.h; sourceTree = "<group>"; };
0F66E16914DF3F1300B7B2E4 /* DFGEdge.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGEdge.h; path = dfg/DFGEdge.h; sourceTree = "<group>"; };
0F73D7AB165A142A00ACAB71 /* ClosureCallStubRoutine.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ClosureCallStubRoutine.cpp; sourceTree = "<group>"; };
0F73D7AC165A142A00ACAB71 /* ClosureCallStubRoutine.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ClosureCallStubRoutine.h; sourceTree = "<group>"; };
0F766D1C15A5028D008F363E /* JITStubRoutine.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JITStubRoutine.h; sourceTree = "<group>"; };
0F766D2615A8CC1B008F363E /* JITStubRoutine.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JITStubRoutine.cpp; sourceTree = "<group>"; };
0F766D2915A8CC34008F363E /* JITStubRoutineSet.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JITStubRoutineSet.cpp; sourceTree = "<group>"; };
......@@ -954,9 +958,9 @@
0F7700911402FF280078EB39 /* SamplingCounter.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = SamplingCounter.cpp; sourceTree = "<group>"; };
0F7B294814C3CD23007C3DB1 /* DFGCCallHelpers.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGCCallHelpers.h; path = dfg/DFGCCallHelpers.h; sourceTree = "<group>"; };
0F8023E91613832300A0BA45 /* ByValInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ByValInfo.h; sourceTree = "<group>"; };
0F8364B5164B0C0E0053329A /* DFGBranchDirection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGBranchDirection.h; path = dfg/DFGBranchDirection.h; sourceTree = "<group>"; };
0F8335B41639C1E3001443B5 /* ArrayAllocationProfile.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ArrayAllocationProfile.cpp; sourceTree = "<group>"; };
0F8335B51639C1E3001443B5 /* ArrayAllocationProfile.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ArrayAllocationProfile.h; sourceTree = "<group>"; };
0F8364B5164B0C0E0053329A /* DFGBranchDirection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = DFGBranchDirection.h; path = dfg/DFGBranchDirection.h; sourceTree = "<group>"; };
0F919D09157EE09D004A4E7D /* JSSymbolTableObject.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSSymbolTableObject.cpp; sourceTree = "<group>"; };
0F919D0A157EE09D004A4E7D /* JSSymbolTableObject.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JSSymbolTableObject.h; sourceTree = "<group>"; };
0F919D0E157F3327004A4E7D /* JSSegmentedVariableObject.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = JSSegmentedVariableObject.cpp; sourceTree = "<group>"; };
......@@ -1803,6 +1807,8 @@
1429D92C0ED22D7000B89619 /* jit */ = {
isa = PBXGroup;
children = (
0F73D7AB165A142A00ACAB71 /* ClosureCallStubRoutine.cpp */,
0F73D7AC165A142A00ACAB71 /* ClosureCallStubRoutine.h */,
0FD82E37141AB14200179C94 /* CompactJITCodeMap.h */,
A7B48DB60EE74CFC00DCBDB6 /* ExecutableAllocator.cpp */,
A7B48DB50EE74CFC00DCBDB6 /* ExecutableAllocator.h */,
......@@ -3022,6 +3028,7 @@
A7B601821639FD2A00372BA3 /* UnlinkedCodeBlock.h in Headers */,
A77F1822164088B200640A47 /* CodeCache.h in Headers */,
A77F1825164192C700640A47 /* ParserModes.h in Headers */,
0F73D7AF165A143000ACAB71 /* ClosureCallStubRoutine.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
......@@ -3601,6 +3608,7 @@
0F8335B71639C1E6001443B5 /* ArrayAllocationProfile.cpp in Sources */,
A76F279415F13C9600517D67 /* UnlinkedCodeBlock.cpp in Sources */,
A77F1821164088B200640A47 /* CodeCache.cpp in Sources */,
0F73D7AE165A142D00ACAB71 /* ClosureCallStubRoutine.cpp in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
......
......@@ -144,6 +144,7 @@ SOURCES += \
interpreter/CallFrame.cpp \
interpreter/Interpreter.cpp \
interpreter/JSStack.cpp \
jit/ClosureCallStubRoutine.cpp \
jit/ExecutableAllocatorFixedVMPool.cpp \
jit/ExecutableAllocator.cpp \
jit/HostCallReturnValue.cpp \
......
......@@ -37,6 +37,7 @@ void CallLinkInfo::unlink(JSGlobalData& globalData, RepatchBuffer& repatchBuffer
{
ASSERT(isLinked());
repatchBuffer.revertJumpReplacementToBranchPtrWithPatch(RepatchBuffer::startOfBranchPtrWithPatchOnRegister(hotPathBegin), static_cast<MacroAssembler::RegisterID>(calleeGPR), 0);
if (isDFG) {
#if ENABLE(DFG_JIT)
repatchBuffer.relink(callReturnLocation, (callType == Construct ? globalData.getCTIStub(DFG::linkConstructThunkGenerator) : globalData.getCTIStub(DFG::linkCallThunkGenerator)).code());
......@@ -47,6 +48,7 @@ void CallLinkInfo::unlink(JSGlobalData& globalData, RepatchBuffer& repatchBuffer
repatchBuffer.relink(callReturnLocation, callType == Construct ? globalData.jitStubs->ctiVirtualConstructLink() : globalData.jitStubs->ctiVirtualCallLink());
hasSeenShouldRepatch = false;
callee.clear();
stub.clear();
// It will be on a list if the callee has a code block.
if (isOnList())
......
......@@ -26,6 +26,7 @@
#ifndef CallLinkInfo_h
#define CallLinkInfo_h
#include "ClosureCallStubRoutine.h"
#include "CodeLocation.h"
#include "JITWriteBarrier.h"
#include "JSFunction.h"
......@@ -70,12 +71,14 @@ struct CallLinkInfo : public BasicRawSentinelNode<CallLinkInfo> {
CodeLocationNearCall hotPathOther;
JITWriteBarrier<JSFunction> callee;
WriteBarrier<JSFunction> lastSeenCallee;
RefPtr<ClosureCallStubRoutine> stub;
bool hasSeenShouldRepatch : 1;
bool isDFG : 1;
CallType callType : 6;
unsigned bytecodeIndex;
unsigned calleeGPR : 8;
CodeOrigin codeOrigin;
bool isLinked() { return callee; }
bool isLinked() { return stub || callee; }
void unlink(JSGlobalData&, RepatchBuffer&);
bool seenOnce()
......@@ -96,7 +99,7 @@ inline void* getCallLinkInfoReturnLocation(CallLinkInfo* callLinkInfo)
inline unsigned getCallLinkInfoBytecodeIndex(CallLinkInfo* callLinkInfo)
{
return callLinkInfo->bytecodeIndex;
return callLinkInfo->codeOrigin.bytecodeIndex;
}
#endif // ENABLE(JIT)
......
......@@ -2188,7 +2188,7 @@ void CodeBlock::finalizeUnconditionally()
// Check if we're not live. If we are, then jettison.
if (!(shouldImmediatelyAssumeLivenessDuringScan() || m_dfgData->livenessHasBeenProved)) {
if (verboseUnlinking)
dataLog("Code block %p has dead weak references, jettisoning during GC.\n", this);
dataLog("Code block %p (executable %p) has dead weak references, jettisoning during GC.\n", this, ownerExecutable());
// Make sure that the baseline JIT knows that it should re-warm-up before
// optimizing.
......@@ -2247,10 +2247,19 @@ void CodeBlock::finalizeUnconditionally()
if (!!getJITCode()) {
RepatchBuffer repatchBuffer(this);
for (unsigned i = 0; i < numberOfCallLinkInfos(); ++i) {
if (callLinkInfo(i).isLinked() && !Heap::isMarked(callLinkInfo(i).callee.get())) {
if (verboseUnlinking)
dataLog("Clearing call from %p to %p.\n", this, callLinkInfo(i).callee.get());
callLinkInfo(i).unlink(*m_globalData, repatchBuffer);
if (callLinkInfo(i).isLinked()) {
if (ClosureCallStubRoutine* stub = callLinkInfo(i).stub.get()) {
if (!Heap::isMarked(stub->structure())
|| !Heap::isMarked(stub->executable())) {
if (verboseUnlinking)
dataLog("Clearing closure call from %p to %p, stub routine %p.\n", this, stub->executable(), stub);
callLinkInfo(i).unlink(*m_globalData, repatchBuffer);
}
} else if (!Heap::isMarked(callLinkInfo(i).callee.get())) {
if (verboseUnlinking)
dataLog("Clearing call from %p to %p.\n", this, callLinkInfo(i).callee.get());
callLinkInfo(i).unlink(*m_globalData, repatchBuffer);
}
}
if (!!callLinkInfo(i).lastSeenCallee
&& !Heap::isMarked(callLinkInfo(i).lastSeenCallee.get()))
......@@ -2605,6 +2614,35 @@ Instruction* CodeBlock::adjustPCIfAtCallSite(Instruction* potentialReturnPC)
}
#endif // ENABLE(LLINT)
#if ENABLE(JIT)
ClosureCallStubRoutine* CodeBlock::findClosureCallForReturnPC(ReturnAddressPtr returnAddress)
{
for (unsigned i = m_callLinkInfos.size(); i--;) {
CallLinkInfo& info = m_callLinkInfos[i];
if (!info.stub)
continue;
if (!info.stub->code().executableMemory()->contains(returnAddress.value()))
continue;
return info.stub.get();
}
// The stub routine may have been jettisoned. This is rare, but we have to handle it.
const JITStubRoutineSet& set = m_globalData->heap.jitStubRoutines();
for (unsigned i = set.size(); i--;) {
GCAwareJITStubRoutine* genericStub = set.at(i);
if (!genericStub->isClosureCall())
continue;
ClosureCallStubRoutine* stub = static_cast<ClosureCallStubRoutine*>(genericStub);
if (!stub->code().executableMemory()->contains(returnAddress.value()))
continue;
return stub;
}
return 0;
}
#endif
unsigned CodeBlock::bytecodeOffset(ExecState* exec, ReturnAddressPtr returnAddress)
{
UNUSED_PARAM(exec);
......@@ -2642,7 +2680,16 @@ unsigned CodeBlock::bytecodeOffset(ExecState* exec, ReturnAddressPtr returnAddre
Vector<CallReturnOffsetToBytecodeOffset>& callIndices = m_rareData->m_callReturnIndexVector;
if (!callIndices.size())
return 1;
return binarySearch<CallReturnOffsetToBytecodeOffset, unsigned, getCallReturnOffset>(callIndices.begin(), callIndices.size(), getJITCode().offsetOf(returnAddress.value()))->bytecodeOffset;
if (getJITCode().getExecutableMemory()->contains(returnAddress.value())) {
unsigned callReturnOffset = getJITCode().offsetOf(returnAddress.value());
CallReturnOffsetToBytecodeOffset* result =
binarySearch<CallReturnOffsetToBytecodeOffset, unsigned, getCallReturnOffset>(callIndices.begin(), callIndices.size(), callReturnOffset);
ASSERT(result->callReturnOffset == callReturnOffset);
return result->bytecodeOffset;
}
return findClosureCallForReturnPC(returnAddress)->codeOrigin().bytecodeIndex;
#endif // ENABLE(JIT)
#if !ENABLE(LLINT) && !ENABLE(JIT)
......@@ -2650,6 +2697,26 @@ unsigned CodeBlock::bytecodeOffset(ExecState* exec, ReturnAddressPtr returnAddre
#endif
}
#if ENABLE(DFG_JIT)
bool CodeBlock::codeOriginForReturn(ReturnAddressPtr returnAddress, CodeOrigin& codeOrigin)
{
if (!hasCodeOrigins())
return false;
if (!getJITCode().getExecutableMemory()->contains(returnAddress.value())) {
codeOrigin = findClosureCallForReturnPC(returnAddress)->codeOrigin();
return true;
}
unsigned offset = getJITCode().offsetOf(returnAddress.value());
CodeOriginAtCallReturnOffset* entry = binarySearch<CodeOriginAtCallReturnOffset, unsigned, getCallReturnOffsetForCodeOrigin>(codeOrigins().begin(), codeOrigins().size(), offset, WTF::KeyMustNotBePresentInArray);
if (entry->callReturnOffset != offset)
return false;
codeOrigin = entry->codeOrigin;
return true;
}
#endif // ENABLE(DFG_JIT)
void CodeBlock::clearEvalCache()
{
if (!!m_alternative)
......
......@@ -245,6 +245,7 @@ namespace JSC {
CallLinkInfo& getCallLinkInfo(unsigned bytecodeIndex)
{
ASSERT(JITCode::isBaselineCode(getJITType()));
return *(binarySearch<CallLinkInfo, unsigned, getCallLinkInfoBytecodeIndex>(m_callLinkInfos.begin(), m_callLinkInfos.size(), bytecodeIndex));
}
#endif // ENABLE(JIT)
......@@ -274,6 +275,11 @@ namespace JSC {
{
m_incomingCalls.push(incoming);
}
bool isIncomingCallAlreadyLinked(CallLinkInfo* incoming)
{
return m_incomingCalls.isOnList(incoming);
}
#endif // ENABLE(JIT)
#if ENABLE(LLINT)
......@@ -813,17 +819,7 @@ namespace JSC {
return m_rareData && !!m_rareData->m_codeOrigins.size();
}
bool codeOriginForReturn(ReturnAddressPtr returnAddress, CodeOrigin& codeOrigin)
{
if (!hasCodeOrigins())
return false;
unsigned offset = getJITCode().offsetOf(returnAddress.value());
CodeOriginAtCallReturnOffset* entry = binarySearch<CodeOriginAtCallReturnOffset, unsigned, getCallReturnOffsetForCodeOrigin>(codeOrigins().begin(), codeOrigins().size(), offset, WTF::KeyMustNotBePresentInArray);
if (entry->callReturnOffset != offset)
return false;
codeOrigin = entry->codeOrigin;
return true;
}
bool codeOriginForReturn(ReturnAddressPtr, CodeOrigin&);
CodeOrigin codeOrigin(unsigned index)
{
......@@ -1187,6 +1183,10 @@ namespace JSC {
private:
friend class DFGCodeBlocks;
#if ENABLE(JIT)
ClosureCallStubRoutine* findClosureCallForReturnPC(ReturnAddressPtr);
#endif
#if ENABLE(DFG_JIT)
void tallyFrequentExitSites();
......
......@@ -188,11 +188,12 @@ void JITCompiler::link(LinkBuffer& linkBuffer)
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.callType = m_jsCalls[i].m_callType;
info.isDFG = true;
info.bytecodeIndex = m_jsCalls[i].m_codeOrigin.bytecodeIndex;
info.codeOrigin = m_jsCalls[i].m_codeOrigin;
linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_globalData->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress()));
info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall);
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee);
}
MacroAssemblerCodeRef osrExitThunk = globalData()->getCTIStub(osrExitGenerationThunkGenerator);
......
......@@ -357,9 +357,9 @@ public:
m_propertyAccesses.append(record);
}
void addJSCall(Call fastCall, Call slowCall, DataLabelPtr targetToCheck, CallLinkInfo::CallType callType, CodeOrigin codeOrigin)
void addJSCall(Call fastCall, Call slowCall, DataLabelPtr targetToCheck, CallLinkInfo::CallType callType, GPRReg callee, CodeOrigin codeOrigin)
{
m_jsCalls.append(JSCallRecord(fastCall, slowCall, targetToCheck, callType, codeOrigin));
m_jsCalls.append(JSCallRecord(fastCall, slowCall, targetToCheck, callType, callee, codeOrigin));
}
void addWeakReference(JSCell* target)
......@@ -440,11 +440,12 @@ private:
Vector<CallExceptionRecord> m_exceptionChecks;
struct JSCallRecord {
JSCallRecord(Call fastCall, Call slowCall, DataLabelPtr targetToCheck, CallLinkInfo::CallType callType, CodeOrigin codeOrigin)
JSCallRecord(Call fastCall, Call slowCall, DataLabelPtr targetToCheck, CallLinkInfo::CallType callType, GPRReg callee, CodeOrigin codeOrigin)
: m_fastCall(fastCall)
, m_slowCall(slowCall)
, m_targetToCheck(targetToCheck)
, m_callType(callType)
, m_callee(callee)
, m_codeOrigin(codeOrigin)
{
}
......@@ -453,6 +454,7 @@ private:
Call m_slowCall;
DataLabelPtr m_targetToCheck;
CallLinkInfo::CallType m_callType;
GPRReg m_callee;
CodeOrigin m_codeOrigin;
};
......
......@@ -1062,14 +1062,14 @@ char* DFG_OPERATION operationLinkConstruct(ExecState* execCallee)
return linkFor(execCallee, CodeForConstruct);
}
inline char* virtualFor(ExecState* execCallee, CodeSpecializationKind kind)
inline char* virtualForWithFunction(ExecState* execCallee, CodeSpecializationKind kind, JSCell*& calleeAsFunctionCell)
{
ExecState* exec = execCallee->callerFrame();
JSGlobalData* globalData = &exec->globalData();
NativeCallFrameTracer tracer(globalData, exec);
JSValue calleeAsValue = execCallee->calleeAsValue();
JSCell* calleeAsFunctionCell = getJSFunction(calleeAsValue);
calleeAsFunctionCell = getJSFunction(calleeAsValue);
if (UNLIKELY(!calleeAsFunctionCell))
return reinterpret_cast<char*>(handleHostCall(execCallee, calleeAsValue, kind));
......@@ -1087,6 +1087,56 @@ inline char* virtualFor(ExecState* execCallee, CodeSpecializationKind kind)
return reinterpret_cast<char*>(executable->generatedJITCodeWithArityCheckFor(kind).executableAddress());
}
inline char* virtualFor(ExecState* execCallee, CodeSpecializationKind kind)
{
JSCell* calleeAsFunctionCellIgnored;
return virtualForWithFunction(execCallee, kind, calleeAsFunctionCellIgnored);
}
static bool attemptToOptimizeClosureCall(ExecState* execCallee, JSCell* calleeAsFunctionCell, CallLinkInfo& callLinkInfo)
{
if (!calleeAsFunctionCell)
return false;
JSFunction* callee = jsCast<JSFunction*>(calleeAsFunctionCell);
JSFunction* oldCallee = callLinkInfo.callee.get();
if (!oldCallee
|| oldCallee->structure() != callee->structure()
|| oldCallee->executable() != callee->executable())
return false;
ASSERT(callee->executable()->hasJITCodeForCall());
MacroAssemblerCodePtr codePtr = callee->executable()->generatedJITCodeForCall().addressForCall();
CodeBlock* codeBlock;
if (callee->executable()->isHostFunction())
codeBlock = 0;
else {
codeBlock = &jsCast<FunctionExecutable*>(callee->executable())->generatedBytecodeForCall();
if (execCallee->argumentCountIncludingThis() < static_cast<size_t>(codeBlock->numParameters()))
return false;
}
dfgLinkClosureCall(
execCallee, callLinkInfo, codeBlock,
callee->structure(), callee->executable(), codePtr);
return true;
}
char* DFG_OPERATION operationLinkClosureCall(ExecState* execCallee)
{
JSCell* calleeAsFunctionCell;
char* result = virtualForWithFunction(execCallee, CodeForCall, calleeAsFunctionCell);
CallLinkInfo& callLinkInfo = execCallee->callerFrame()->codeBlock()->getCallLinkInfo(execCallee->returnPC());
if (!attemptToOptimizeClosureCall(execCallee, calleeAsFunctionCell, callLinkInfo))
dfgLinkSlowFor(execCallee, callLinkInfo, CodeForCall);
return result;
}
char* DFG_OPERATION operationVirtualCall(ExecState* execCallee)
{
return virtualFor(execCallee, CodeForCall);
......
......@@ -181,6 +181,7 @@ size_t DFG_OPERATION operationCompareStrictEqCell(ExecState*, EncodedJSValue enc
size_t DFG_OPERATION operationCompareStrictEq(ExecState*, EncodedJSValue encodedOp1, EncodedJSValue encodedOp2) WTF_INTERNAL;
char* DFG_OPERATION operationVirtualCall(ExecState*) WTF_INTERNAL;
char* DFG_OPERATION operationLinkCall(ExecState*) WTF_INTERNAL;
char* DFG_OPERATION operationLinkClosureCall(ExecState*) WTF_INTERNAL;
char* DFG_OPERATION operationVirtualConstruct(ExecState*) WTF_INTERNAL;
char* DFG_OPERATION operationLinkConstruct(ExecState*) WTF_INTERNAL;
JSCell* DFG_OPERATION operationCreateActivation(ExecState*) WTF_INTERNAL;
......
......@@ -1113,8 +1113,20 @@ void dfgBuildPutByIdList(ExecState* exec, JSValue baseValue, const Identifier& p
dfgRepatchCall(exec->codeBlock(), stubInfo.callReturnLocation, appropriateGenericPutByIdFunction(slot, putKind));
}
static void linkSlowFor(RepatchBuffer& repatchBuffer, JSGlobalData* globalData, CallLinkInfo& callLinkInfo, CodeSpecializationKind kind)
{
if (kind == CodeForCall) {
repatchBuffer.relink(callLinkInfo.callReturnLocation, globalData->getCTIStub(virtualCallThunkGenerator).code());
return;
}
ASSERT(kind == CodeForConstruct);
repatchBuffer.relink(callLinkInfo.callReturnLocation, globalData->getCTIStub(virtualConstructThunkGenerator).code());
}
void dfgLinkFor(ExecState* exec, CallLinkInfo& callLinkInfo, CodeBlock* calleeCodeBlock, JSFunction* callee, MacroAssemblerCodePtr codePtr, CodeSpecializationKind kind)
{
ASSERT(!callLinkInfo.stub);
CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
JSGlobalData* globalData = callerCodeBlock->globalData();
......@@ -1129,11 +1141,110 @@ void dfgLinkFor(ExecState* exec, CallLinkInfo& callLinkInfo, CodeBlock* calleeCo
calleeCodeBlock->linkIncomingCall(&callLinkInfo);
if (kind == CodeForCall) {
repatchBuffer.relink(callLinkInfo.callReturnLocation, globalData->getCTIStub(virtualCallThunkGenerator).code());
repatchBuffer.relink(callLinkInfo.callReturnLocation, globalData->getCTIStub(linkClosureCallThunkGenerator).code());
return;
}
ASSERT(kind == CodeForConstruct);
repatchBuffer.relink(callLinkInfo.callReturnLocation, globalData->getCTIStub(virtualConstructThunkGenerator).code());
linkSlowFor(repatchBuffer, globalData, callLinkInfo, CodeForConstruct);
}
void dfgLinkSlowFor(ExecState* exec, CallLinkInfo& callLinkInfo, CodeSpecializationKind kind)
{
CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
JSGlobalData* globalData = callerCodeBlock->globalData();
RepatchBuffer repatchBuffer(callerCodeBlock);
linkSlowFor(repatchBuffer, globalData, callLinkInfo, kind);
}
void dfgLinkClosureCall(ExecState* exec, CallLinkInfo& callLinkInfo, CodeBlock* calleeCodeBlock, Structure* structure, ExecutableBase* executable, MacroAssemblerCodePtr codePtr)
{
ASSERT(!callLinkInfo.stub);
CodeBlock* callerCodeBlock = exec->callerFrame()->codeBlock();
JSGlobalData* globalData = callerCodeBlock->globalData();