FTLLowerDFGToLLVM.cpp 98.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * Copyright (C) 2013 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

#include "config.h"
#include "FTLLowerDFGToLLVM.h"

#if ENABLE(FTL_JIT)

#include "CodeBlockWithJITType.h"
32 33
#include "DFGAbstractInterpreterInlines.h"
#include "DFGInPlaceAbstractState.h"
34 35 36
#include "FTLAbstractHeapRepository.h"
#include "FTLExitThunkGenerator.h"
#include "FTLFormattedValue.h"
37
#include "FTLLoweredNodeValue.h"
38 39 40 41
#include "FTLOutput.h"
#include "FTLThunks.h"
#include "FTLValueSource.h"
#include "LinkBuffer.h"
42
#include "OperandsInlines.h"
43
#include "Operations.h"
44
#include <wtf/ProcessID.h>
45 46 47 48 49

namespace JSC { namespace FTL {

using namespace DFG;

50 51
static int compileCounter;

52 53 54 55 56 57
// Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
// significantly less dead code.
#define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
        FormattedValue _ftc_lowValue = (lowValue);                      \
        Edge _ftc_highValue = (highValue);                              \
        SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
58
        if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
59 60 61 62 63 64 65 66 67
            break;                                                      \
        typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
    } while (false)

class LowerDFGToLLVM {
public:
    LowerDFGToLLVM(State& state)
        : m_graph(state.graph)
        , m_ftlState(state)
68 69
        , m_heaps(state.context)
        , m_out(state.context)
70
        , m_valueSources(OperandsLike, state.graph.block(0)->variablesAtHead)
71 72 73
        , m_lastSetOperand(std::numeric_limits<int>::max())
        , m_exitThunkGenerator(state)
        , m_state(state.graph)
74
        , m_interpreter(state.graph, m_state)
75 76 77 78 79
    {
    }
    
    void lower()
    {
80 81 82 83 84 85 86 87 88 89
        CString name;
        if (verboseCompilationEnabled()) {
            name = toCString(
                "jsBody_", atomicIncrement(&compileCounter), "_", codeBlock()->inferredName(),
                "_", codeBlock()->hash());
        } else
            name = "jsBody";
        
        m_graph.m_dominators.computeIfNecessary(m_graph);
        
90
        m_ftlState.module =
91
            LLVMModuleCreateWithNameInContext(name.data(), m_ftlState.context);
92 93
        
        m_ftlState.function = addFunction(
94
            m_ftlState.module, name.data(), functionType(m_out.int64, m_out.intPtr));
95 96 97 98
        setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
        
        m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
        
99
        m_prologue = appendBasicBlock(m_ftlState.context, m_ftlState.function);
100
        m_out.appendTo(m_prologue);
101
        createPhiVariables();
102
        
103
        m_initialization = appendBasicBlock(m_ftlState.context, m_ftlState.function);
104 105 106 107 108

        m_callFrame = m_out.param(0);
        m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
        m_tagMask = m_out.constInt64(TagMask);
        
109 110
        for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
            m_highBlock = m_graph.block(blockIndex);
111 112
            if (!m_highBlock)
                continue;
113
            m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
114 115
        }
        
116 117 118 119
        Vector<BasicBlock*> depthFirst;
        m_graph.getBlocksInDepthFirstOrder(depthFirst);
        for (unsigned i = 0; i < depthFirst.size(); ++i)
            compileBlock(depthFirst[i]);
120 121 122 123
        
        // And now complete the initialization block.
        linkOSRExitsAndCompleteInitializationBlocks();

124 125 126
        if (Options::dumpLLVMIR())
            dumpModule(m_ftlState.module);
        
127 128 129 130 131 132 133 134
        if (verboseCompilationEnabled())
            m_ftlState.dumpState("after lowering");
        if (validationEnabled())
            verifyModule(m_ftlState.module);
    }

private:
    
135
    void createPhiVariables()
136
    {
137 138 139
        for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
140
                continue;
141 142 143
            for (unsigned nodeIndex = block->size(); nodeIndex--;) {
                Node* node = block->at(nodeIndex);
                if (node->op() != Phi)
144
                    continue;
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
                LType type;
                switch (node->flags() & NodeResultMask) {
                case NodeResultNumber:
                    type = m_out.doubleType;
                    break;
                case NodeResultInt32:
                    type = m_out.int32;
                    break;
                case NodeResultBoolean:
                    type = m_out.boolean;
                    break;
                case NodeResultJS:
                    type = m_out.int64;
                    break;
                default:
                    RELEASE_ASSERT_NOT_REACHED();
                    break;
162
                }
163
                m_phis.add(node, buildAlloca(m_out.m_builder, type));
164 165 166 167
            }
        }
    }
    
168
    void compileBlock(BasicBlock* block)
169
    {
170
        if (!block)
171 172
            return;
        
173 174 175 176 177
        if (verboseCompilationEnabled())
            dataLog("Compiling block ", *block, "\n");
        
        m_highBlock = block;
        
178 179 180
        LBasicBlock lowBlock = m_blocks.get(m_highBlock);
        
        m_nextHighBlock = 0;
181
        for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
182
            m_nextHighBlock = m_graph.block(nextBlockIndex);
183 184 185 186 187 188 189 190 191 192 193
            if (m_nextHighBlock)
                break;
        }
        m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
        
        // All of this effort to find the next block gives us the ability to keep the
        // generated IR in roughly program order. This ought not affect the performance
        // of the generated code (since we expect LLVM to reorder things) but it will
        // make IR dumps easier to read.
        m_out.appendTo(lowBlock, m_nextLowBlock);
        
194 195 196 197 198
        if (!m_highBlock->cfaHasVisited) {
            m_out.crash();
            return;
        }
        
199 200
        initializeOSRExitStateForBlock();
        
201
        m_live = block->ssa->liveAtHead;
202 203 204 205
        
        m_state.reset();
        m_state.beginBasicBlock(m_highBlock);
        
206 207 208 209
        for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
            if (!compileNode(m_nodeIndex))
                break;
        }
210 211
    }
    
212
    bool compileNode(unsigned nodeIndex)
213
    {
214 215 216 217 218
        if (!m_state.isValid()) {
            m_out.unreachable();
            return false;
        }
        
219
        m_node = m_highBlock->at(nodeIndex);
220 221
        m_codeOriginForExitProfile = m_node->codeOrigin;
        m_codeOriginForExitTarget = m_node->codeOriginForExitTarget;
222 223 224 225
        
        if (verboseCompilationEnabled())
            dataLog("Lowering ", m_node, "\n");
        
226
        bool shouldExecuteEffects = m_interpreter.startExecuting(m_node);
227 228 229 230
        
        m_direction = (m_node->flags() & NodeExitsForward) ? ForwardSpeculation : BackwardSpeculation;
        
        switch (m_node->op()) {
231 232 233 234 235 236
        case Upsilon:
            compileUpsilon();
            break;
        case Phi:
            compilePhi();
            break;
237 238 239 240 241 242
        case JSConstant:
            compileJSConstant();
            break;
        case WeakJSConstant:
            compileWeakJSConstant();
            break;
243 244 245
        case GetArgument:
            compileGetArgument();
            break;
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
        case GetLocal:
            compileGetLocal();
            break;
        case SetLocal:
            compileSetLocal();
            break;
        case MovHint:
            compileMovHint();
            break;
        case ZombieHint:
            compileZombieHint();
            break;
        case MovHintAndCheck:
            compileMovHintAndCheck();
            break;
        case Phantom:
            compilePhantom();
            break;
        case Flush:
        case PhantomLocal:
        case SetArgument:
            break;
        case ArithAdd:
        case ValueAdd:
            compileAdd();
            break;
        case ArithSub:
            compileArithSub();
            break;
        case ArithMul:
            compileArithMul();
            break;
278 279 280
        case ArithDiv:
            compileArithDiv();
            break;
281 282 283
        case ArithMod:
            compileArithMod();
            break;
284 285 286 287
        case ArithMin:
        case ArithMax:
            compileArithMinOrMax();
            break;
288 289 290
        case ArithAbs:
            compileArithAbs();
            break;
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
        case ArithNegate:
            compileArithNegate();
            break;
        case BitAnd:
            compileBitAnd();
            break;
        case BitOr:
            compileBitOr();
            break;
        case BitXor:
            compileBitXor();
            break;
        case BitRShift:
            compileBitRShift();
            break;
        case BitLShift:
            compileBitLShift();
            break;
        case BitURShift:
            compileBitURShift();
            break;
        case UInt32ToNumber:
            compileUInt32ToNumber();
            break;
315 316 317
        case Int32ToDouble:
            compileInt32ToDouble();
            break;
318 319 320 321 322 323
        case CheckStructure:
            compileCheckStructure();
            break;
        case StructureTransitionWatchpoint:
            compileStructureTransitionWatchpoint();
            break;
324 325 326
        case ArrayifyToStructure:
            compileArrayifyToStructure();
            break;
327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
        case PutStructure:
            compilePutStructure();
            break;
        case PhantomPutStructure:
            compilePhantomPutStructure();
            break;
        case GetButterfly:
            compileGetButterfly();
            break;
        case GetArrayLength:
            compileGetArrayLength();
            break;
        case GetByVal:
            compileGetByVal();
            break;
342 343 344 345
        case PutByVal:
        case PutByValAlias:
            compilePutByVal();
            break;
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
        case GetByOffset:
            compileGetByOffset();
            break;
        case PutByOffset:
            compilePutByOffset();
            break;
        case GetGlobalVar:
            compileGetGlobalVar();
            break;
        case PutGlobalVar:
            compilePutGlobalVar();
            break;
        case CompareEq:
            compileCompareEq();
            break;
361 362 363
        case CompareEqConstant:
            compileCompareEqConstant();
            break;
364 365 366
        case CompareStrictEq:
            compileCompareStrictEq();
            break;
367 368 369
        case CompareStrictEqConstant:
            compileCompareStrictEqConstant();
            break;
370 371 372
        case CompareLess:
            compileCompareLess();
            break;
373 374 375 376 377 378 379 380 381
        case CompareLessEq:
            compileCompareLessEq();
            break;
        case CompareGreater:
            compileCompareGreater();
            break;
        case CompareGreaterEq:
            compileCompareGreaterEq();
            break;
382 383 384
        case LogicalNot:
            compileLogicalNot();
            break;
385 386 387
        case Jump:
            compileJump();
            break;
388 389 390
        case Branch:
            compileBranch();
            break;
391 392 393
        case Switch:
            compileSwitch();
            break;
394 395 396 397 398 399
        case Return:
            compileReturn();
            break;
        case ForceOSRExit:
            compileForceOSRExit();
            break;
400 401 402 403 404 405 406 407
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
        
        if (m_node->shouldGenerate())
            DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, use);
        
408 409
        if (m_node->adjustedRefCount())
            m_live.add(m_node);
410 411
        
        if (shouldExecuteEffects)
412
            m_interpreter.executeEffects(nodeIndex);
413 414
        
        return true;
415 416
    }
    
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
    void compileUpsilon()
    {
        LValue destination = m_phis.get(m_node->phi());
        
        switch (m_node->child1().useKind()) {
        case NumberUse:
            m_out.set(lowDouble(m_node->child1()), destination);
            break;
        case Int32Use:
            m_out.set(lowInt32(m_node->child1()), destination);
            break;
        case BooleanUse:
            m_out.set(lowBoolean(m_node->child1()), destination);
            break;
        case CellUse:
            m_out.set(lowCell(m_node->child1()), destination);
            break;
        case UntypedUse:
            m_out.set(lowJSValue(m_node->child1()), destination);
            break;
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
    void compilePhi()
    {
        LValue source = m_phis.get(m_node);
        
        switch (m_node->flags() & NodeResultMask) {
        case NodeResultNumber:
            setDouble(m_out.get(source));
            break;
        case NodeResultInt32:
            setInt32(m_out.get(source));
            break;
        case NodeResultBoolean:
            setBoolean(m_out.get(source));
            break;
        case NodeResultJS:
            setJSValue(m_out.get(source));
            break;
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
466 467
    void compileJSConstant()
    {
468 469
        JSValue value = m_graph.valueOfJSConstant(m_node);
        if (value.isDouble())
470
            setDouble(m_out.constDouble(value.asDouble()));
471
        else
472
            setJSValue(m_out.constInt64(JSValue::encode(value)));
473 474 475 476
    }
    
    void compileWeakJSConstant()
    {
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
        setJSValue(weakPointer(m_node->weakConstant()));
    }
    
    void compileGetArgument()
    {
        VariableAccessData* variable = m_node->variableAccessData();
        int operand = variable->operand();

        LValue jsValue = m_out.load64(addressFor(operand));

        switch (useKindFor(variable->flushFormat())) {
        case Int32Use:
            speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotInt32(jsValue));
            setInt32(unboxInt32(jsValue));
            break;
        case CellUse:
            speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotCell(jsValue));
            setJSValue(jsValue);
            break;
        case BooleanUse:
            speculateBackward(BadType, jsValueValue(jsValue), m_node, isNotBoolean(jsValue));
            setBoolean(unboxBoolean(jsValue));
            break;
        case UntypedUse:
            setJSValue(jsValue);
            break;
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
507 508 509 510
    }
    
    void compileGetLocal()
    {
511
        // GetLocals arise only for captured variables.
512 513 514 515
        
        VariableAccessData* variable = m_node->variableAccessData();
        AbstractValue& value = m_state.variables().operand(variable->local());
        
516
        RELEASE_ASSERT(variable->isCaptured());
517
        
518 519 520 521
        if (isInt32Speculation(value.m_value))
            setInt32(m_out.load32(payloadFor(variable->local())));
        else
            setJSValue(m_out.load64(addressFor(variable->local())));
522 523 524 525 526 527 528 529 530 531 532
    }
    
    void compileSetLocal()
    {
        observeMovHint(m_node);
        
        VariableAccessData* variable = m_node->variableAccessData();
        SpeculatedType prediction = variable->argumentAwarePrediction();
        
        if (variable->shouldUnboxIfPossible()) {
            if (variable->shouldUseDoubleFormat()) {
533
                LValue value = lowDouble(m_node->child1());
534 535
                m_out.storeDouble(value, addressFor(variable->local()));
                m_valueSources.operand(variable->local()) = ValueSource(DoubleInJSStack);
536 537 538 539 540
                return;
            }
            
            if (isInt32Speculation(prediction)) {
                LValue value = lowInt32(m_node->child1());
541 542
                m_out.store32(value, payloadFor(variable->local()));
                m_valueSources.operand(variable->local()) = ValueSource(Int32InJSStack);
543 544 545 546
                return;
            }
            if (isCellSpeculation(prediction)) {
                LValue value = lowCell(m_node->child1());
547 548
                m_out.store64(value, addressFor(variable->local()));
                m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
549 550 551
                return;
            }
            if (isBooleanSpeculation(prediction)) {
552 553 554 555 556
                speculateBoolean(m_node->child1());
                m_out.store64(
                    lowJSValue(m_node->child1(), ManualOperandSpeculation),
                    addressFor(variable->local()));
                m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
557 558 559 560 561
                return;
            }
        }
        
        LValue value = lowJSValue(m_node->child1());
562 563
        m_out.store64(value, addressFor(variable->local()));
        m_valueSources.operand(variable->local()) = ValueSource(ValueInJSStack);
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
    }
    
    void compileMovHint()
    {
        observeMovHint(m_node);
    }
    
    void compileZombieHint()
    {
        VariableAccessData* data = m_node->variableAccessData();
        m_lastSetOperand = data->local();
        m_valueSources.operand(data->local()) = ValueSource(SourceIsDead);
    }
    
    void compileMovHintAndCheck()
    {
        observeMovHint(m_node);
        speculate(m_node->child1());
    }
    
    void compilePhantom()
    {
        DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
    }
    
    void compileAdd()
    {
        switch (m_node->binaryUseKind()) {
        case Int32Use: {
            LValue left = lowInt32(m_node->child1());
            LValue right = lowInt32(m_node->child2());
            
            if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
597
                setInt32(m_out.add(left, right));
598 599 600 601 602
                break;
            }
            
            LValue result = m_out.addWithOverflow32(left, right);
            speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
603
            setInt32(m_out.extractValue(result, 0));
604 605 606
            break;
        }
            
607
        case NumberUse: {
608
            setDouble(
609 610 611 612
                m_out.doubleAdd(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
            break;
        }
            
613 614 615 616 617 618 619 620 621 622 623 624 625 626
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
    void compileArithSub()
    {
        switch (m_node->binaryUseKind()) {
        case Int32Use: {
            LValue left = lowInt32(m_node->child1());
            LValue right = lowInt32(m_node->child2());
            
            if (nodeCanTruncateInteger(m_node->arithNodeFlags())) {
627
                setInt32(m_out.sub(left, right));
628 629 630 631 632
                break;
            }
            
            LValue result = m_out.subWithOverflow32(left, right);
            speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
633
            setInt32(m_out.extractValue(result, 0));
634 635 636
            break;
        }
            
637
        case NumberUse: {
638
            setDouble(
639 640 641 642
                m_out.doubleSub(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
            break;
        }
            
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
    void compileArithMul()
    {
        switch (m_node->binaryUseKind()) {
        case Int32Use: {
            LValue left = lowInt32(m_node->child1());
            LValue right = lowInt32(m_node->child2());
            
            LValue result;
            if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
                result = m_out.mul(left, right);
            else {
                LValue overflowResult = m_out.mulWithOverflow32(left, right);
                speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
                result = m_out.extractValue(overflowResult, 0);
            }
            
            if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
                LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
                LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
                
669 670
                m_out.branch(m_out.notZero32(result), continuation, slowCase);
                
671 672 673 674 675 676 677
                LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
                speculate(NegativeZero, noValue(), 0, m_out.lessThan(left, m_out.int32Zero));
                speculate(NegativeZero, noValue(), 0, m_out.lessThan(right, m_out.int32Zero));
                m_out.jump(continuation);
                m_out.appendTo(continuation, lastNext);
            }
            
678
            setInt32(result);
679 680 681
            break;
        }
            
682
        case NumberUse: {
683
            setDouble(
684 685 686 687
                m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
            break;
        }
            
688 689 690 691 692 693
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
    void compileArithDiv()
    {
        switch (m_node->binaryUseKind()) {
        case Int32Use: {
            LValue numerator = lowInt32(m_node->child1());
            LValue denominator = lowInt32(m_node->child2());
            
            LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
            LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
            LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
            
            Vector<ValueFromBlock, 3> results;
            
            LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
            
            m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
            
            LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
            
            LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
            
            if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
                speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
                speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
                m_out.jump(continuation);
            } else {
                // This is the case where we convert the result to an int after we're done. So,
                // if the denominator is zero, then the result should be result should be zero.
                // If the denominator is not zero (i.e. it's -1 because we're guarded by the
                // check above) and the numerator is -2^31 then the result should be -2^31.
                
                LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
                LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
                LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
                
                m_out.branch(m_out.isZero32(denominator), divByZero, notDivByZero);
                
                m_out.appendTo(divByZero, notDivByZero);
                results.append(m_out.anchor(m_out.int32Zero));
                m_out.jump(done);
                
                m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
                m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
                
                m_out.appendTo(neg2ToThe31ByNeg1, continuation);
                results.append(m_out.anchor(neg2ToThe31));
                m_out.jump(done);
            }
            
            m_out.appendTo(continuation, done);
            
            if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
                LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
                LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
                
                m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
                
                LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
                
                speculate(
                    NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
                
                m_out.jump(numeratorContinuation);
                
                m_out.appendTo(numeratorContinuation, innerLastNext);
            }
            
            LValue divisionResult = m_out.div(numerator, denominator);
            
            if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
                speculate(
                    Overflow, noValue(), 0,
                    m_out.notEqual(m_out.mul(divisionResult, denominator), numerator));
            }
            
            results.append(m_out.anchor(divisionResult));
            m_out.jump(done);
            
            m_out.appendTo(done, lastNext);
            
774
            setInt32(m_out.phi(m_out.int32, results));
775 776 777 778
            break;
        }
            
        case NumberUse: {
779
            setDouble(
780 781 782 783 784 785 786 787 788 789
                m_out.doubleDiv(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
            break;
        }
            
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863
    void compileArithMod()
    {
        switch (m_node->binaryUseKind()) {
        case Int32Use: {
            LValue numerator = lowInt32(m_node->child1());
            LValue denominator = lowInt32(m_node->child2());
            
            LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
            LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
            LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
            
            Vector<ValueFromBlock, 3> results;
            
            LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
            
            m_out.branch(m_out.above(adjustedDenominator, m_out.int32One), continuation, unsafeDenominator);
            
            LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
            
            LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
            
            // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
            // separate case for that. But it probably doesn't matter so much.
            if (nodeUsedAsNumber(m_node->arithNodeFlags())) {
                speculate(Overflow, noValue(), 0, m_out.isZero32(denominator));
                speculate(Overflow, noValue(), 0, m_out.equal(numerator, neg2ToThe31));
                m_out.jump(continuation);
            } else {
                // This is the case where we convert the result to an int after we're done. So,
                // if the denominator is zero, then the result should be result should be zero.
                // If the denominator is not zero (i.e. it's -1 because we're guarded by the
                // check above) and the numerator is -2^31 then the result should be -2^31.
                
                LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
                LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
                LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
                
                m_out.branch(m_out.isZero32(denominator), modByZero, notModByZero);
                
                m_out.appendTo(modByZero, notModByZero);
                results.append(m_out.anchor(m_out.int32Zero));
                m_out.jump(done);
                
                m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
                m_out.branch(m_out.equal(numerator, neg2ToThe31), neg2ToThe31ByNeg1, continuation);
                
                m_out.appendTo(neg2ToThe31ByNeg1, continuation);
                results.append(m_out.anchor(m_out.int32Zero));
                m_out.jump(done);
            }
            
            m_out.appendTo(continuation, done);
            
            if (!nodeCanIgnoreNegativeZero(m_node->arithNodeFlags())) {
                LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod zero numerator"));
                LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
                
                m_out.branch(m_out.isZero32(numerator), zeroNumerator, numeratorContinuation);
                
                LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
                
                speculate(
                    NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
                
                m_out.jump(numeratorContinuation);
                
                m_out.appendTo(numeratorContinuation, innerLastNext);
            }
            
            results.append(m_out.anchor(m_out.rem(numerator, denominator)));
            m_out.jump(done);
            
            m_out.appendTo(done, lastNext);
            
864
            setInt32(m_out.phi(m_out.int32, results));
865 866 867 868
            break;
        }
            
        case NumberUse: {
869
            setDouble(
870 871 872 873 874 875 876 877 878
                m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
            break;
        }
            
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
879 880 881 882 883 884 885 886

    void compileArithMinOrMax()
    {
        switch (m_node->binaryUseKind()) {
        case Int32Use: {
            LValue left = lowInt32(m_node->child1());
            LValue right = lowInt32(m_node->child2());
            
887
            setInt32(
888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
                m_out.select(
                    m_node->op() == ArithMin
                        ? m_out.lessThan(left, right)
                        : m_out.lessThan(right, left),
                    left, right));
            break;
        }
            
        case NumberUse: {
            LValue left = lowDouble(m_node->child1());
            LValue right = lowDouble(m_node->child2());
            
            LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
            LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
            
            Vector<ValueFromBlock, 2> results;
            
            results.append(m_out.anchor(left));
            m_out.branch(
                m_node->op() == ArithMin
                    ? m_out.doubleLessThan(left, right)
                    : m_out.doubleGreaterThan(left, right),
                continuation, notLessThan);
            
            LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
            results.append(m_out.anchor(m_out.select(
                m_node->op() == ArithMin
                    ? m_out.doubleGreaterThanOrEqual(left, right)
                    : m_out.doubleLessThanOrEqual(left, right),
                right, m_out.constDouble(0.0 / 0.0))));
            m_out.jump(continuation);
            
            m_out.appendTo(continuation, lastNext);
921
            setDouble(m_out.phi(m_out.doubleType, results));
922 923 924 925 926 927 928 929
            break;
        }
            
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
930
    
931 932 933 934 935 936 937 938 939 940 941
    void compileArithAbs()
    {
        switch (m_node->child1().useKind()) {
        case Int32Use: {
            LValue value = lowInt32(m_node->child1());
            
            LValue mask = m_out.aShr(value, m_out.constInt32(31));
            LValue result = m_out.bitXor(mask, m_out.add(mask, value));
            
            speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
            
942
            setInt32(result);
943 944 945 946
            break;
        }
            
        case NumberUse: {
947
            setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
948 949 950 951 952 953 954 955 956
            break;
        }
            
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
    void compileArithNegate()
    {
        switch (m_node->child1().useKind()) {
        case Int32Use: {
            LValue value = lowInt32(m_node->child1());
            
            LValue result;
            if (nodeCanTruncateInteger(m_node->arithNodeFlags()))
                result = m_out.neg(value);
            else {
                // We don't have a negate-with-overflow intrinsic. Hopefully this
                // does the trick, though.
                LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
                speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
                result = m_out.extractValue(overflowResult, 0);
            }
            
974
            setInt32(result);
975 976 977
            break;
        }
            
978
        case NumberUse: {
979
            setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
980 981 982
            break;
        }
            
983 984 985 986 987 988 989 990
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
    void compileBitAnd()
    {
991
        setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
992 993 994 995
    }
    
    void compileBitOr()
    {
996
        setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
997 998 999 1000
    }
    
    void compileBitXor()
    {
1001
        setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1002 1003 1004 1005
    }
    
    void compileBitRShift()
    {
1006 1007 1008
        setInt32(m_out.aShr(
            lowInt32(m_node->child1()),
            m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1009 1010 1011 1012
    }
    
    void compileBitLShift()
    {
1013 1014 1015
        setInt32(m_out.shl(
            lowInt32(m_node->child1()),
            m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1016 1017 1018 1019
    }
    
    void compileBitURShift()
    {
1020 1021 1022
        setInt32(m_out.lShr(
            lowInt32(m_node->child1()),
            m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1023 1024 1025 1026
    }
    
    void compileUInt32ToNumber()
    {
1027 1028
        LValue value = lowInt32(m_node->child1());

1029
        if (!nodeCanSpeculateInteger(m_node->arithNodeFlags())) {
1030
            setDouble(m_out.unsignedToDouble(value));
1031 1032 1033 1034 1035 1036
            return;
        }
        
        speculateForward(
            Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero),
            FormattedValue(ValueFormatUInt32, value));
1037
        setInt32(value);
1038 1039
    }
    
1040 1041 1042 1043 1044 1045 1046 1047 1048
    void compileInt32ToDouble()
    {
        // This node is tricky to compile in the DFG backend because it tries to
        // avoid converting child1 to a double in-place, as that would make subsequent
        // int uses of of child1 fail. But the FTL needs no such special magic, since
        // unlike the DFG backend, the FTL allows each node to have multiple
        // contemporaneous low-level representations. So, this gives child1 a double
        // representation and then forwards that representation to m_node.
        
1049
        setDouble(lowDouble(m_node->child1()));
1050 1051
    }
    
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
    void compileCheckStructure()
    {
        LValue cell = lowCell(m_node->child1());
        
        ExitKind exitKind;
        if (m_node->child1()->op() == WeakJSConstant)
            exitKind = BadWeakConstantCache;
        else
            exitKind = BadCache;
        
        LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
        
        if (m_node->structureSet().size() == 1) {
            speculate(
                exitKind, jsValueValue(cell), 0,
                m_out.notEqual(structure, weakPointer(m_node->structureSet()[0])));
            return;
        }
        
        LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CheckStructure continuation"));
        
        LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
        for (unsigned i = 0; i < m_node->structureSet().size() - 1; ++i) {
            LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("CheckStructure nextStructure"));
            m_out.branch(
                m_out.equal(structure, weakPointer(m_node->structureSet()[i])),
                continuation, nextStructure);
            m_out.appendTo(nextStructure);
        }
        
        speculate(
            exitKind, jsValueValue(cell), 0,
            m_out.notEqual(structure, weakPointer(m_node->structureSet().last())));
        
        m_out.jump(continuation);
        m_out.appendTo(continuation, lastNext);
    }
    
    void compileStructureTransitionWatchpoint()
    {
        addWeakReference(m_node->structure());
        
        // FIXME: Implement structure transition watchpoints.
        // https://bugs.webkit.org/show_bug.cgi?id=113647
        
        speculateCell(m_node->child1());
    }
    
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
    void compileArrayifyToStructure()
    {
        LValue cell = lowCell(m_node->child1());
        LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
        
        LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
        LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
        
        LValue structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
        
        m_out.branch(
            m_out.notEqual(structure, weakPointer(m_node->structure())),
            unexpectedStructure, continuation);
        
        LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
        
        if (property) {
            switch (m_node->arrayMode().type()) {
            case Array::Int32:
            case Array::Double:
            case Array::Contiguous:
                speculate(
                    Uncountable, noValue(), 0,
                    m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
                break;
            default:
                break;
            }
        }
        
        switch (m_node->arrayMode().type()) {
        case Array::Int32:
            vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
            break;
        case Array::Double:
            vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
            break;
        case Array::Contiguous:
            if (m_node->arrayMode().conversion() == Array::RageConvert)
                vmCall(m_out.operation(operationRageEnsureContiguous), m_callFrame, cell);
            else
                vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
            break;
        case Array::ArrayStorage:
        case Array::SlowPutArrayStorage:
            vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
            break;
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
        
        structure = m_out.loadPtr(cell, m_heaps.JSCell_structure);
        speculate(
            BadIndexingType, jsValueValue(cell), 0,
            m_out.notEqual(structure, weakPointer(m_node->structure())));
        m_out.jump(continuation);
        
        m_out.appendTo(continuation, lastNext);
    }
    
1161 1162
    void compilePutStructure()
    {
1163
        m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1164 1165 1166 1167 1168 1169 1170 1171
        
        m_out.store64(
            m_out.constIntPtr(m_node->structureTransitionData().newStructure),
            lowCell(m_node->child1()), m_heaps.JSCell_structure);
    }
    
    void compilePhantomPutStructure()
    {
1172
        m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
1173 1174 1175 1176
    }
    
    void compileGetButterfly()
    {
1177
        setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
1178 1179 1180 1181 1182 1183 1184 1185
    }
    
    void compileGetArrayLength()
    {
        switch (m_node->arrayMode().type()) {
        case Array::Int32:
        case Array::Double:
        case Array::Contiguous: {
1186
            setInt32(m_out.load32(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
            break;
        }
            
        default:
            RELEASE_ASSERT_NOT_REACHED();
            break;
        }
    }
    
    void compileGetByVal()
    {
        LValue index = lowInt32(m_node->child2());
        LValue storage = lowStorage(m_node->child3());
        
        switch (m_node->arrayMode().type()) {
        case Array::Int32:
        case Array::Contiguous: {
            if (m_node->arrayMode().isInBounds()) {
                speculate(
                    OutOfBounds, noValue(), 0,
                    m_out.aboveOrEqual(
                        index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
                
                LValue result = m_out.load64(m_out.baseIndex(
                    m_node->arrayMode().type() == Array::Int32 ?
                        m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
                    storage, m_out.zeroExt(index, m_out.intPtr),
                    m_state.forNode(m_node->child2()).m_value));
                speculate(LoadFromHole, noValue(), 0, m_out.isZero64(result));
1216
                setJSValue(result);
1217
                return;
1218 1219
            }
            
1220 1221
            // FIXME: Implement hole/OOB loads in the FTL.
            // https://bugs.webkit.org/show_bug.cgi?id=118077
1222
            RELEASE_ASSERT_NOT_REACHED();
1223
            return;
1224 1225
        }
            
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
        case Array::Double: {
            if (m_node->arrayMode().isInBounds()) {
                if (m_node->arrayMode().isSaneChain()) {
                    // FIXME: Implement structure transition watchpoints.
                    // https://bugs.webkit.org/show_bug.cgi?id=113647
                }
            
                speculate(
                    OutOfBounds, noValue(), 0,
                    m_out.aboveOrEqual(
                        index, m_out.load32(storage, m_heaps.Butterfly_publicLength)));
                
                LValue result = m_out.loadDouble(m_out.baseIndex(
                    m_heaps.indexedDoubleProperties,
                    storage, m_out.zeroExt(index, m_out.intPtr),
                    m_state.forNode(m_node->child2()).m_value));
                
                if (!m_node->arrayMode().isSaneChain()) {
                    speculate(
                        LoadFromHole, noValue(), 0,
                        m_out.doubleNotEqualOrUnordered(result, result));
                }
1248
                setDouble(result);
1249 1250 1251
                break;
            }
            
1252 1253
            // FIXME: Implement hole/OOB loads in the FTL.
            // https://bugs.webkit.org/show_bug.cgi?id=118077
1254
            RELEASE_ASSERT_NOT_REACHED();
1255
            return;
1256 1257
        }
            
1258 1259
        default:
            RELEASE_ASSERT_NOT_REACHED();
1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
            return;
        }
    }
    
    void compilePutByVal()
    {
        Edge child1 = m_graph.varArgChild(m_node, 0);
        Edge child2 = m_graph.varArgChild(m_node, 1);
        Edge child3 = m_graph.varArgChild(m_node, 2);
        Edge child4 = m_graph.varArgChild(m_node, 3);

1271
        LValue base = lowCell(child1);
1272 1273 1274
        LValue index = lowInt32(child2);
        LValue storage = lowStorage(child4);
        
1275 1276 1277
        LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
        LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);