3460 lines
95 KiB
C++
3460 lines
95 KiB
C++
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
|
|
* vim: set ts=8 sts=4 et sw=4 tw=99:
|
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
#include "jit/arm/Assembler-arm.h"
|
|
|
|
#include "mozilla/DebugOnly.h"
|
|
#include "mozilla/MathAlgorithms.h"
|
|
|
|
#include "jscompartment.h"
|
|
#ifdef JS_DISASM_ARM
|
|
#include "jsprf.h"
|
|
#endif
|
|
#include "jsutil.h"
|
|
|
|
#include "gc/Marking.h"
|
|
#include "jit/arm/disasm/Disasm-arm.h"
|
|
#include "jit/arm/MacroAssembler-arm.h"
|
|
#include "jit/ExecutableAllocator.h"
|
|
#include "jit/JitCompartment.h"
|
|
#include "jit/MacroAssembler.h"
|
|
|
|
using namespace js;
|
|
using namespace js::jit;
|
|
|
|
using mozilla::CountLeadingZeroes32;
|
|
|
|
void dbg_break() {}
|
|
|
|
// The ABIArgGenerator is used for making system ABI calls and for inter-wasm
|
|
// calls. The system ABI can either be SoftFp or HardFp, and inter-wasm calls
|
|
// are always HardFp calls. The initialization defaults to HardFp, and the ABI
|
|
// choice is made before any system ABI calls with the method "setUseHardFp".
|
|
ABIArgGenerator::ABIArgGenerator()
|
|
: intRegIndex_(0),
|
|
floatRegIndex_(0),
|
|
stackOffset_(0),
|
|
current_(),
|
|
useHardFp_(true)
|
|
{ }
|
|
|
|
// See the "Parameter Passing" section of the "Procedure Call Standard for the
|
|
// ARM Architecture" documentation.
|
|
ABIArg
|
|
ABIArgGenerator::softNext(MIRType type)
|
|
{
|
|
switch (type) {
|
|
case MIRType::Int32:
|
|
case MIRType::Pointer:
|
|
if (intRegIndex_ == NumIntArgRegs) {
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint32_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(Register::FromCode(intRegIndex_));
|
|
intRegIndex_++;
|
|
break;
|
|
case MIRType::Int64:
|
|
// Make sure to use an even register index. Increase to next even number
|
|
// when odd.
|
|
intRegIndex_ = (intRegIndex_ + 1) & ~1;
|
|
if (intRegIndex_ == NumIntArgRegs) {
|
|
// Align the stack on 8 bytes.
|
|
static const uint32_t align = sizeof(uint64_t) - 1;
|
|
stackOffset_ = (stackOffset_ + align) & ~align;
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint64_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(Register::FromCode(intRegIndex_), Register::FromCode(intRegIndex_ + 1));
|
|
intRegIndex_ += 2;
|
|
break;
|
|
case MIRType::Float32:
|
|
if (intRegIndex_ == NumIntArgRegs) {
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint32_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(Register::FromCode(intRegIndex_));
|
|
intRegIndex_++;
|
|
break;
|
|
case MIRType::Double:
|
|
// Make sure to use an even register index. Increase to next even number
|
|
// when odd.
|
|
intRegIndex_ = (intRegIndex_ + 1) & ~1;
|
|
if (intRegIndex_ == NumIntArgRegs) {
|
|
// Align the stack on 8 bytes.
|
|
static const uint32_t align = sizeof(double) - 1;
|
|
stackOffset_ = (stackOffset_ + align) & ~align;
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(double);
|
|
break;
|
|
}
|
|
current_ = ABIArg(Register::FromCode(intRegIndex_), Register::FromCode(intRegIndex_ + 1));
|
|
intRegIndex_ += 2;
|
|
break;
|
|
default:
|
|
MOZ_CRASH("Unexpected argument type");
|
|
}
|
|
|
|
return current_;
|
|
}
|
|
|
|
ABIArg
|
|
ABIArgGenerator::hardNext(MIRType type)
|
|
{
|
|
switch (type) {
|
|
case MIRType::Int32:
|
|
case MIRType::Pointer:
|
|
if (intRegIndex_ == NumIntArgRegs) {
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint32_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(Register::FromCode(intRegIndex_));
|
|
intRegIndex_++;
|
|
break;
|
|
case MIRType::Int64:
|
|
// Make sure to use an even register index. Increase to next even number
|
|
// when odd.
|
|
intRegIndex_ = (intRegIndex_ + 1) & ~1;
|
|
if (intRegIndex_ == NumIntArgRegs) {
|
|
// Align the stack on 8 bytes.
|
|
static const uint32_t align = sizeof(uint64_t) - 1;
|
|
stackOffset_ = (stackOffset_ + align) & ~align;
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint64_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(Register::FromCode(intRegIndex_), Register::FromCode(intRegIndex_ + 1));
|
|
intRegIndex_ += 2;
|
|
break;
|
|
case MIRType::Float32:
|
|
if (floatRegIndex_ == NumFloatArgRegs) {
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint32_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(VFPRegister(floatRegIndex_, VFPRegister::Single));
|
|
floatRegIndex_++;
|
|
break;
|
|
case MIRType::Double:
|
|
// Double register are composed of 2 float registers, thus we have to
|
|
// skip any float register which cannot be used in a pair of float
|
|
// registers in which a double value can be stored.
|
|
floatRegIndex_ = (floatRegIndex_ + 1) & ~1;
|
|
if (floatRegIndex_ == NumFloatArgRegs) {
|
|
static const uint32_t align = sizeof(double) - 1;
|
|
stackOffset_ = (stackOffset_ + align) & ~align;
|
|
current_ = ABIArg(stackOffset_);
|
|
stackOffset_ += sizeof(uint64_t);
|
|
break;
|
|
}
|
|
current_ = ABIArg(VFPRegister(floatRegIndex_ >> 1, VFPRegister::Double));
|
|
floatRegIndex_ += 2;
|
|
break;
|
|
default:
|
|
MOZ_CRASH("Unexpected argument type");
|
|
}
|
|
|
|
return current_;
|
|
}
|
|
|
|
ABIArg
|
|
ABIArgGenerator::next(MIRType type)
|
|
{
|
|
if (useHardFp_)
|
|
return hardNext(type);
|
|
return softNext(type);
|
|
}
|
|
|
|
bool
|
|
js::jit::IsUnaligned(const wasm::MemoryAccessDesc& access)
|
|
{
|
|
if (!access.align())
|
|
return false;
|
|
|
|
if (access.type() == Scalar::Float64 && access.align() >= 4)
|
|
return false;
|
|
|
|
return access.align() < access.byteSize();
|
|
}
|
|
|
|
// Encode a standard register when it is being used as src1, the dest, and an
|
|
// extra register. These should never be called with an InvalidReg.
|
|
uint32_t
|
|
js::jit::RT(Register r)
|
|
{
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 12;
|
|
}
|
|
|
|
uint32_t
|
|
js::jit::RN(Register r)
|
|
{
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 16;
|
|
}
|
|
|
|
uint32_t
|
|
js::jit::RD(Register r)
|
|
{
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 12;
|
|
}
|
|
|
|
uint32_t
|
|
js::jit::RM(Register r)
|
|
{
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 8;
|
|
}
|
|
|
|
// Encode a standard register when it is being used as src1, the dest, and an
|
|
// extra register. For these, an InvalidReg is used to indicate a optional
|
|
// register that has been omitted.
|
|
uint32_t
|
|
js::jit::maybeRT(Register r)
|
|
{
|
|
if (r == InvalidReg)
|
|
return 0;
|
|
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 12;
|
|
}
|
|
|
|
uint32_t
|
|
js::jit::maybeRN(Register r)
|
|
{
|
|
if (r == InvalidReg)
|
|
return 0;
|
|
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 16;
|
|
}
|
|
|
|
uint32_t
|
|
js::jit::maybeRD(Register r)
|
|
{
|
|
if (r == InvalidReg)
|
|
return 0;
|
|
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return r.code() << 12;
|
|
}
|
|
|
|
Register
|
|
js::jit::toRD(Instruction i)
|
|
{
|
|
return Register::FromCode((i.encode() >> 12) & 0xf);
|
|
}
|
|
Register
|
|
js::jit::toR(Instruction i)
|
|
{
|
|
return Register::FromCode(i.encode() & 0xf);
|
|
}
|
|
|
|
Register
|
|
js::jit::toRM(Instruction i)
|
|
{
|
|
return Register::FromCode((i.encode() >> 8) & 0xf);
|
|
}
|
|
|
|
Register
|
|
js::jit::toRN(Instruction i)
|
|
{
|
|
return Register::FromCode((i.encode() >> 16) & 0xf);
|
|
}
|
|
|
|
uint32_t
|
|
js::jit::VD(VFPRegister vr)
|
|
{
|
|
if (vr.isMissing())
|
|
return 0;
|
|
|
|
// Bits 15,14,13,12, 22.
|
|
VFPRegister::VFPRegIndexSplit s = vr.encode();
|
|
return s.bit << 22 | s.block << 12;
|
|
}
|
|
uint32_t
|
|
js::jit::VN(VFPRegister vr)
|
|
{
|
|
if (vr.isMissing())
|
|
return 0;
|
|
|
|
// Bits 19,18,17,16, 7.
|
|
VFPRegister::VFPRegIndexSplit s = vr.encode();
|
|
return s.bit << 7 | s.block << 16;
|
|
}
|
|
uint32_t
|
|
js::jit::VM(VFPRegister vr)
|
|
{
|
|
if (vr.isMissing())
|
|
return 0;
|
|
|
|
// Bits 5, 3,2,1,0.
|
|
VFPRegister::VFPRegIndexSplit s = vr.encode();
|
|
return s.bit << 5 | s.block;
|
|
}
|
|
|
|
VFPRegister::VFPRegIndexSplit
|
|
jit::VFPRegister::encode()
|
|
{
|
|
MOZ_ASSERT(!_isInvalid);
|
|
|
|
switch (kind) {
|
|
case Double:
|
|
return VFPRegIndexSplit(code_ & 0xf, code_ >> 4);
|
|
case Single:
|
|
return VFPRegIndexSplit(code_ >> 1, code_ & 1);
|
|
default:
|
|
// VFP register treated as an integer, NOT a gpr.
|
|
return VFPRegIndexSplit(code_ >> 1, code_ & 1);
|
|
}
|
|
}
|
|
|
|
bool
|
|
InstDTR::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & IsDTRMask) == (uint32_t)IsDTR;
|
|
}
|
|
|
|
InstDTR*
|
|
InstDTR::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstDTR*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstLDR::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & IsDTRMask) == (uint32_t)IsDTR;
|
|
}
|
|
|
|
InstLDR*
|
|
InstLDR::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstLDR*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
InstNOP*
|
|
InstNOP::AsTHIS(Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstNOP*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstNOP::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & 0x0fffffff) == NopInst;
|
|
}
|
|
|
|
bool
|
|
InstBranchReg::IsTHIS(const Instruction& i)
|
|
{
|
|
return InstBXReg::IsTHIS(i) || InstBLXReg::IsTHIS(i);
|
|
}
|
|
|
|
InstBranchReg*
|
|
InstBranchReg::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstBranchReg*)&i;
|
|
return nullptr;
|
|
}
|
|
void
|
|
InstBranchReg::extractDest(Register* dest)
|
|
{
|
|
*dest = toR(*this);
|
|
}
|
|
bool
|
|
InstBranchReg::checkDest(Register dest)
|
|
{
|
|
return dest == toR(*this);
|
|
}
|
|
|
|
bool
|
|
InstBranchImm::IsTHIS(const Instruction& i)
|
|
{
|
|
return InstBImm::IsTHIS(i) || InstBLImm::IsTHIS(i);
|
|
}
|
|
|
|
InstBranchImm*
|
|
InstBranchImm::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstBranchImm*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
void
|
|
InstBranchImm::extractImm(BOffImm* dest)
|
|
{
|
|
*dest = BOffImm(*this);
|
|
}
|
|
|
|
bool
|
|
InstBXReg::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & IsBRegMask) == IsBX;
|
|
}
|
|
|
|
InstBXReg*
|
|
InstBXReg::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstBXReg*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstBLXReg::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & IsBRegMask) == IsBLX;
|
|
|
|
}
|
|
InstBLXReg*
|
|
InstBLXReg::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstBLXReg*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstBImm::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode () & IsBImmMask) == IsB;
|
|
}
|
|
InstBImm*
|
|
InstBImm::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstBImm*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstBLImm::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode () & IsBImmMask) == IsBL;
|
|
|
|
}
|
|
InstBLImm*
|
|
InstBLImm::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstBLImm*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstMovWT::IsTHIS(Instruction& i)
|
|
{
|
|
return InstMovW::IsTHIS(i) || InstMovT::IsTHIS(i);
|
|
}
|
|
InstMovWT*
|
|
InstMovWT::AsTHIS(Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstMovWT*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
void
|
|
InstMovWT::extractImm(Imm16* imm)
|
|
{
|
|
*imm = Imm16(*this);
|
|
}
|
|
bool
|
|
InstMovWT::checkImm(Imm16 imm)
|
|
{
|
|
return imm.decode() == Imm16(*this).decode();
|
|
}
|
|
|
|
void
|
|
InstMovWT::extractDest(Register* dest)
|
|
{
|
|
*dest = toRD(*this);
|
|
}
|
|
bool
|
|
InstMovWT::checkDest(Register dest)
|
|
{
|
|
return dest == toRD(*this);
|
|
}
|
|
|
|
bool
|
|
InstMovW::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & IsWTMask) == IsW;
|
|
}
|
|
|
|
InstMovW*
|
|
InstMovW::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstMovW*)&i;
|
|
return nullptr;
|
|
}
|
|
InstMovT*
|
|
InstMovT::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstMovT*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstMovT::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & IsWTMask) == IsT;
|
|
}
|
|
|
|
InstALU*
|
|
InstALU::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstALU*)&i;
|
|
return nullptr;
|
|
}
|
|
bool
|
|
InstALU::IsTHIS(const Instruction& i)
|
|
{
|
|
return (i.encode() & ALUMask) == 0;
|
|
}
|
|
void
|
|
InstALU::extractOp(ALUOp* ret)
|
|
{
|
|
*ret = ALUOp(encode() & (0xf << 21));
|
|
}
|
|
bool
|
|
InstALU::checkOp(ALUOp op)
|
|
{
|
|
ALUOp mine;
|
|
extractOp(&mine);
|
|
return mine == op;
|
|
}
|
|
void
|
|
InstALU::extractDest(Register* ret)
|
|
{
|
|
*ret = toRD(*this);
|
|
}
|
|
bool
|
|
InstALU::checkDest(Register rd)
|
|
{
|
|
return rd == toRD(*this);
|
|
}
|
|
void
|
|
InstALU::extractOp1(Register* ret)
|
|
{
|
|
*ret = toRN(*this);
|
|
}
|
|
bool
|
|
InstALU::checkOp1(Register rn)
|
|
{
|
|
return rn == toRN(*this);
|
|
}
|
|
Operand2
|
|
InstALU::extractOp2()
|
|
{
|
|
return Operand2(encode());
|
|
}
|
|
|
|
InstCMP*
|
|
InstCMP::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstCMP*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstCMP::IsTHIS(const Instruction& i)
|
|
{
|
|
return InstALU::IsTHIS(i) && InstALU::AsTHIS(i)->checkDest(r0) && InstALU::AsTHIS(i)->checkOp(OpCmp);
|
|
}
|
|
|
|
InstMOV*
|
|
InstMOV::AsTHIS(const Instruction& i)
|
|
{
|
|
if (IsTHIS(i))
|
|
return (InstMOV*)&i;
|
|
return nullptr;
|
|
}
|
|
|
|
bool
|
|
InstMOV::IsTHIS(const Instruction& i)
|
|
{
|
|
return InstALU::IsTHIS(i) && InstALU::AsTHIS(i)->checkOp1(r0) && InstALU::AsTHIS(i)->checkOp(OpMov);
|
|
}
|
|
|
|
Op2Reg
|
|
Operand2::toOp2Reg() const {
|
|
return *(Op2Reg*)this;
|
|
}
|
|
|
|
Imm16::Imm16(Instruction& inst)
|
|
: lower_(inst.encode() & 0xfff),
|
|
upper_(inst.encode() >> 16),
|
|
invalid_(0xfff)
|
|
{ }
|
|
|
|
Imm16::Imm16(uint32_t imm)
|
|
: lower_(imm & 0xfff), pad_(0),
|
|
upper_((imm >> 12) & 0xf),
|
|
invalid_(0)
|
|
{
|
|
MOZ_ASSERT(decode() == imm);
|
|
}
|
|
|
|
Imm16::Imm16()
|
|
: invalid_(0xfff)
|
|
{ }
|
|
|
|
void
|
|
jit::PatchJump(CodeLocationJump& jump_, CodeLocationLabel label, ReprotectCode reprotect)
|
|
{
|
|
// We need to determine if this jump can fit into the standard 24+2 bit
|
|
// address or if we need a larger branch (or just need to use our pool
|
|
// entry).
|
|
Instruction* jump = (Instruction*)jump_.raw();
|
|
// jumpWithPatch() returns the offset of the jump and never a pool or nop.
|
|
Assembler::Condition c = jump->extractCond();
|
|
MOZ_ASSERT(jump->is<InstBranchImm>() || jump->is<InstLDR>());
|
|
|
|
int jumpOffset = label.raw() - jump_.raw();
|
|
if (BOffImm::IsInRange(jumpOffset)) {
|
|
// This instruction started off as a branch, and will remain one.
|
|
MaybeAutoWritableJitCode awjc(jump, sizeof(Instruction), reprotect);
|
|
Assembler::RetargetNearBranch(jump, jumpOffset, c);
|
|
} else {
|
|
// This instruction started off as a branch, but now needs to be demoted
|
|
// to an ldr.
|
|
uint8_t** slot = reinterpret_cast<uint8_t**>(jump_.jumpTableEntry());
|
|
|
|
// Ensure both the branch and the slot are writable.
|
|
MOZ_ASSERT(uintptr_t(slot) > uintptr_t(jump));
|
|
size_t size = uintptr_t(slot) - uintptr_t(jump) + sizeof(void*);
|
|
MaybeAutoWritableJitCode awjc(jump, size, reprotect);
|
|
|
|
Assembler::RetargetFarBranch(jump, slot, label.raw(), c);
|
|
}
|
|
}
|
|
|
|
void
|
|
Assembler::finish()
|
|
{
|
|
flush();
|
|
MOZ_ASSERT(!isFinished);
|
|
isFinished = true;
|
|
}
|
|
|
|
bool
|
|
Assembler::asmMergeWith(Assembler& other)
|
|
{
|
|
flush();
|
|
other.flush();
|
|
if (other.oom())
|
|
return false;
|
|
if (!AssemblerShared::asmMergeWith(size(), other))
|
|
return false;
|
|
return m_buffer.appendBuffer(other.m_buffer);
|
|
}
|
|
|
|
void
|
|
Assembler::executableCopy(uint8_t* buffer)
|
|
{
|
|
MOZ_ASSERT(isFinished);
|
|
m_buffer.executableCopy(buffer);
|
|
AutoFlushICache::setRange(uintptr_t(buffer), m_buffer.size());
|
|
}
|
|
|
|
uint32_t
|
|
Assembler::actualIndex(uint32_t idx_) const
|
|
{
|
|
ARMBuffer::PoolEntry pe(idx_);
|
|
return m_buffer.poolEntryOffset(pe);
|
|
}
|
|
|
|
uint8_t*
|
|
Assembler::PatchableJumpAddress(JitCode* code, uint32_t pe_)
|
|
{
|
|
return code->raw() + pe_;
|
|
}
|
|
|
|
class RelocationIterator
|
|
{
|
|
CompactBufferReader reader_;
|
|
// Offset in bytes.
|
|
uint32_t offset_;
|
|
|
|
public:
|
|
RelocationIterator(CompactBufferReader& reader)
|
|
: reader_(reader)
|
|
{ }
|
|
|
|
bool read() {
|
|
if (!reader_.more())
|
|
return false;
|
|
offset_ = reader_.readUnsigned();
|
|
return true;
|
|
}
|
|
|
|
uint32_t offset() const {
|
|
return offset_;
|
|
}
|
|
};
|
|
|
|
template<class Iter>
|
|
const uint32_t*
|
|
Assembler::GetCF32Target(Iter* iter)
|
|
{
|
|
Instruction* inst1 = iter->cur();
|
|
|
|
if (inst1->is<InstBranchImm>()) {
|
|
// See if we have a simple case, b #offset.
|
|
BOffImm imm;
|
|
InstBranchImm* jumpB = inst1->as<InstBranchImm>();
|
|
jumpB->extractImm(&imm);
|
|
return imm.getDest(inst1)->raw();
|
|
}
|
|
|
|
if (inst1->is<InstMovW>())
|
|
{
|
|
// See if we have the complex case:
|
|
// movw r_temp, #imm1
|
|
// movt r_temp, #imm2
|
|
// bx r_temp
|
|
// OR
|
|
// movw r_temp, #imm1
|
|
// movt r_temp, #imm2
|
|
// str pc, [sp]
|
|
// bx r_temp
|
|
|
|
Imm16 targ_bot;
|
|
Imm16 targ_top;
|
|
Register temp;
|
|
|
|
// Extract both the temp register and the bottom immediate.
|
|
InstMovW* bottom = inst1->as<InstMovW>();
|
|
bottom->extractImm(&targ_bot);
|
|
bottom->extractDest(&temp);
|
|
|
|
// Extract the top part of the immediate.
|
|
Instruction* inst2 = iter->next();
|
|
MOZ_ASSERT(inst2->is<InstMovT>());
|
|
InstMovT* top = inst2->as<InstMovT>();
|
|
top->extractImm(&targ_top);
|
|
|
|
// Make sure they are being loaded into the same register.
|
|
MOZ_ASSERT(top->checkDest(temp));
|
|
|
|
// Make sure we're branching to the same register.
|
|
#ifdef DEBUG
|
|
// A toggled call sometimes has a NOP instead of a branch for the third
|
|
// instruction. No way to assert that it's valid in that situation.
|
|
Instruction* inst3 = iter->next();
|
|
if (!inst3->is<InstNOP>()) {
|
|
InstBranchReg* realBranch = nullptr;
|
|
if (inst3->is<InstBranchReg>()) {
|
|
realBranch = inst3->as<InstBranchReg>();
|
|
} else {
|
|
Instruction* inst4 = iter->next();
|
|
realBranch = inst4->as<InstBranchReg>();
|
|
}
|
|
MOZ_ASSERT(realBranch->checkDest(temp));
|
|
}
|
|
#endif
|
|
|
|
uint32_t* dest = (uint32_t*) (targ_bot.decode() | (targ_top.decode() << 16));
|
|
return dest;
|
|
}
|
|
|
|
if (inst1->is<InstLDR>())
|
|
return *(uint32_t**) inst1->as<InstLDR>()->dest();
|
|
|
|
MOZ_CRASH("unsupported branch relocation");
|
|
}
|
|
|
|
uintptr_t
|
|
Assembler::GetPointer(uint8_t* instPtr)
|
|
{
|
|
InstructionIterator iter((Instruction*)instPtr);
|
|
uintptr_t ret = (uintptr_t)GetPtr32Target(&iter, nullptr, nullptr);
|
|
return ret;
|
|
}
|
|
|
|
template<class Iter>
|
|
const uint32_t*
|
|
Assembler::GetPtr32Target(Iter* start, Register* dest, RelocStyle* style)
|
|
{
|
|
Instruction* load1 = start->cur();
|
|
Instruction* load2 = start->next();
|
|
|
|
if (load1->is<InstMovW>() && load2->is<InstMovT>()) {
|
|
if (style)
|
|
*style = L_MOVWT;
|
|
|
|
// See if we have the complex case:
|
|
// movw r_temp, #imm1
|
|
// movt r_temp, #imm2
|
|
|
|
Imm16 targ_bot;
|
|
Imm16 targ_top;
|
|
Register temp;
|
|
|
|
// Extract both the temp register and the bottom immediate.
|
|
InstMovW* bottom = load1->as<InstMovW>();
|
|
bottom->extractImm(&targ_bot);
|
|
bottom->extractDest(&temp);
|
|
|
|
// Extract the top part of the immediate.
|
|
InstMovT* top = load2->as<InstMovT>();
|
|
top->extractImm(&targ_top);
|
|
|
|
// Make sure they are being loaded into the same register.
|
|
MOZ_ASSERT(top->checkDest(temp));
|
|
|
|
if (dest)
|
|
*dest = temp;
|
|
|
|
uint32_t* value = (uint32_t*) (targ_bot.decode() | (targ_top.decode() << 16));
|
|
return value;
|
|
}
|
|
|
|
if (load1->is<InstLDR>()) {
|
|
if (style)
|
|
*style = L_LDR;
|
|
if (dest)
|
|
*dest = toRD(*load1);
|
|
return *(uint32_t**) load1->as<InstLDR>()->dest();
|
|
}
|
|
|
|
MOZ_CRASH("unsupported relocation");
|
|
}
|
|
|
|
static JitCode*
|
|
CodeFromJump(InstructionIterator* jump)
|
|
{
|
|
uint8_t* target = (uint8_t*)Assembler::GetCF32Target(jump);
|
|
return JitCode::FromExecutable(target);
|
|
}
|
|
|
|
void
|
|
Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader)
|
|
{
|
|
RelocationIterator iter(reader);
|
|
while (iter.read()) {
|
|
InstructionIterator institer((Instruction*) (code->raw() + iter.offset()));
|
|
JitCode* child = CodeFromJump(&institer);
|
|
TraceManuallyBarrieredEdge(trc, &child, "rel32");
|
|
}
|
|
}
|
|
|
|
template <class Iter>
|
|
static void
|
|
TraceOneDataRelocation(JSTracer* trc, Iter* iter)
|
|
{
|
|
Instruction* ins = iter->cur();
|
|
Register dest;
|
|
Assembler::RelocStyle rs;
|
|
const void* prior = Assembler::GetPtr32Target(iter, &dest, &rs);
|
|
void* ptr = const_cast<void*>(prior);
|
|
|
|
// No barrier needed since these are constants.
|
|
TraceManuallyBarrieredGenericPointerEdge(trc, reinterpret_cast<gc::Cell**>(&ptr),
|
|
"ion-masm-ptr");
|
|
|
|
if (ptr != prior) {
|
|
MacroAssemblerARM::ma_mov_patch(Imm32(int32_t(ptr)), dest, Assembler::Always, rs, ins);
|
|
|
|
// L_LDR won't cause any instructions to be updated.
|
|
if (rs != Assembler::L_LDR) {
|
|
AutoFlushICache::flush(uintptr_t(ins), 4);
|
|
AutoFlushICache::flush(uintptr_t(ins->next()), 4);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
TraceDataRelocations(JSTracer* trc, uint8_t* buffer, CompactBufferReader& reader)
|
|
{
|
|
while (reader.more()) {
|
|
size_t offset = reader.readUnsigned();
|
|
InstructionIterator iter((Instruction*)(buffer + offset));
|
|
TraceOneDataRelocation(trc, &iter);
|
|
}
|
|
}
|
|
|
|
static void
|
|
TraceDataRelocations(JSTracer* trc, ARMBuffer* buffer, CompactBufferReader& reader)
|
|
{
|
|
while (reader.more()) {
|
|
BufferOffset offset(reader.readUnsigned());
|
|
ARMBuffer::AssemblerBufferInstIterator iter(offset, buffer);
|
|
TraceOneDataRelocation(trc, &iter);
|
|
}
|
|
}
|
|
|
|
void
|
|
Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code, CompactBufferReader& reader)
|
|
{
|
|
::TraceDataRelocations(trc, code->raw(), reader);
|
|
}
|
|
|
|
void
|
|
Assembler::copyJumpRelocationTable(uint8_t* dest)
|
|
{
|
|
if (jumpRelocations_.length())
|
|
memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
|
|
}
|
|
|
|
void
|
|
Assembler::copyDataRelocationTable(uint8_t* dest)
|
|
{
|
|
if (dataRelocations_.length())
|
|
memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
|
|
}
|
|
|
|
void
|
|
Assembler::copyPreBarrierTable(uint8_t* dest)
|
|
{
|
|
if (preBarriers_.length())
|
|
memcpy(dest, preBarriers_.buffer(), preBarriers_.length());
|
|
}
|
|
|
|
void
|
|
Assembler::trace(JSTracer* trc)
|
|
{
|
|
for (size_t i = 0; i < jumps_.length(); i++) {
|
|
RelativePatch& rp = jumps_[i];
|
|
if (rp.kind() == Relocation::JITCODE) {
|
|
JitCode* code = JitCode::FromExecutable((uint8_t*)rp.target());
|
|
TraceManuallyBarrieredEdge(trc, &code, "masmrel32");
|
|
MOZ_ASSERT(code == JitCode::FromExecutable((uint8_t*)rp.target()));
|
|
}
|
|
}
|
|
|
|
if (dataRelocations_.length()) {
|
|
CompactBufferReader reader(dataRelocations_);
|
|
::TraceDataRelocations(trc, &m_buffer, reader);
|
|
}
|
|
}
|
|
|
|
void
|
|
Assembler::processCodeLabels(uint8_t* rawCode)
|
|
{
|
|
for (size_t i = 0; i < codeLabels_.length(); i++) {
|
|
CodeLabel label = codeLabels_[i];
|
|
Bind(rawCode, label.patchAt(), rawCode + label.target()->offset());
|
|
}
|
|
}
|
|
|
|
void
|
|
Assembler::writeCodePointer(CodeOffset* label) {
|
|
BufferOffset off = writeInst(LabelBase::INVALID_OFFSET);
|
|
label->bind(off.getOffset());
|
|
}
|
|
|
|
void
|
|
Assembler::Bind(uint8_t* rawCode, CodeOffset* label, const void* address)
|
|
{
|
|
*reinterpret_cast<const void**>(rawCode + label->offset()) = address;
|
|
}
|
|
|
|
Assembler::Condition
|
|
Assembler::InvertCondition(Condition cond)
|
|
{
|
|
const uint32_t ConditionInversionBit = 0x10000000;
|
|
return Condition(ConditionInversionBit ^ cond);
|
|
}
|
|
|
|
Assembler::Condition
|
|
Assembler::UnsignedCondition(Condition cond)
|
|
{
|
|
switch (cond) {
|
|
case Zero:
|
|
case NonZero:
|
|
return cond;
|
|
case LessThan:
|
|
case Below:
|
|
return Below;
|
|
case LessThanOrEqual:
|
|
case BelowOrEqual:
|
|
return BelowOrEqual;
|
|
case GreaterThan:
|
|
case Above:
|
|
return Above;
|
|
case AboveOrEqual:
|
|
case GreaterThanOrEqual:
|
|
return AboveOrEqual;
|
|
default:
|
|
MOZ_CRASH("unexpected condition");
|
|
}
|
|
}
|
|
|
|
Assembler::Condition
|
|
Assembler::ConditionWithoutEqual(Condition cond)
|
|
{
|
|
switch (cond) {
|
|
case LessThan:
|
|
case LessThanOrEqual:
|
|
return LessThan;
|
|
case Below:
|
|
case BelowOrEqual:
|
|
return Below;
|
|
case GreaterThan:
|
|
case GreaterThanOrEqual:
|
|
return GreaterThan;
|
|
case Above:
|
|
case AboveOrEqual:
|
|
return Above;
|
|
default:
|
|
MOZ_CRASH("unexpected condition");
|
|
}
|
|
}
|
|
Imm8::TwoImm8mData
|
|
Imm8::EncodeTwoImms(uint32_t imm)
|
|
{
|
|
// In the ideal case, we are looking for a number that (in binary) looks
|
|
// like:
|
|
// 0b((00)*)n_1((00)*)n_2((00)*)
|
|
// left n1 mid n2
|
|
// where both n_1 and n_2 fit into 8 bits.
|
|
// Since this is being done with rotates, we also need to handle the case
|
|
// that one of these numbers is in fact split between the left and right
|
|
// sides, in which case the constant will look like:
|
|
// 0bn_1a((00)*)n_2((00)*)n_1b
|
|
// n1a mid n2 rgh n1b
|
|
// Also remember, values are rotated by multiples of two, and left, mid or
|
|
// right can have length zero.
|
|
uint32_t imm1, imm2;
|
|
int left = CountLeadingZeroes32(imm) & 0x1E;
|
|
uint32_t no_n1 = imm & ~(0xff << (24 - left));
|
|
|
|
// Not technically needed: this case only happens if we can encode as a
|
|
// single imm8m. There is a perfectly reasonable encoding in this case, but
|
|
// we shouldn't encourage people to do things like this.
|
|
if (no_n1 == 0)
|
|
return TwoImm8mData();
|
|
|
|
int mid = CountLeadingZeroes32(no_n1) & 0x1E;
|
|
uint32_t no_n2 = no_n1 & ~((0xff << ((24 - mid) & 0x1f)) | 0xff >> ((8 + mid) & 0x1f));
|
|
|
|
if (no_n2 == 0) {
|
|
// We hit the easy case, no wraparound.
|
|
// Note: a single constant *may* look like this.
|
|
int imm1shift = left + 8;
|
|
int imm2shift = mid + 8;
|
|
imm1 = (imm >> (32 - imm1shift)) & 0xff;
|
|
if (imm2shift >= 32) {
|
|
imm2shift = 0;
|
|
// This assert does not always hold, in fact, this would lead to
|
|
// some incredibly subtle bugs.
|
|
// assert((imm & 0xff) == no_n1);
|
|
imm2 = no_n1;
|
|
} else {
|
|
imm2 = ((imm >> (32 - imm2shift)) | (imm << imm2shift)) & 0xff;
|
|
MOZ_ASSERT( ((no_n1 >> (32 - imm2shift)) | (no_n1 << imm2shift)) ==
|
|
imm2);
|
|
}
|
|
MOZ_ASSERT((imm1shift & 0x1) == 0);
|
|
MOZ_ASSERT((imm2shift & 0x1) == 0);
|
|
return TwoImm8mData(datastore::Imm8mData(imm1, imm1shift >> 1),
|
|
datastore::Imm8mData(imm2, imm2shift >> 1));
|
|
}
|
|
|
|
// Either it wraps, or it does not fit. If we initially chopped off more
|
|
// than 8 bits, then it won't fit.
|
|
if (left >= 8)
|
|
return TwoImm8mData();
|
|
|
|
int right = 32 - (CountLeadingZeroes32(no_n2) & 30);
|
|
// All remaining set bits *must* fit into the lower 8 bits.
|
|
// The right == 8 case should be handled by the previous case.
|
|
if (right > 8)
|
|
return TwoImm8mData();
|
|
|
|
// Make sure the initial bits that we removed for no_n1 fit into the
|
|
// 8-(32-right) leftmost bits.
|
|
if (((imm & (0xff << (24 - left))) << (8 - right)) != 0) {
|
|
// BUT we may have removed more bits than we needed to for no_n1
|
|
// 0x04104001 e.g. we can encode 0x104 with a single op, then 0x04000001
|
|
// with a second, but we try to encode 0x0410000 and find that we need a
|
|
// second op for 0x4000, and 0x1 cannot be included in the encoding of
|
|
// 0x04100000.
|
|
no_n1 = imm & ~((0xff >> (8 - right)) | (0xff << (24 + right)));
|
|
mid = CountLeadingZeroes32(no_n1) & 30;
|
|
no_n2 = no_n1 & ~((0xff << ((24 - mid)&31)) | 0xff >> ((8 + mid)&31));
|
|
if (no_n2 != 0)
|
|
return TwoImm8mData();
|
|
}
|
|
|
|
// Now assemble all of this information into a two coherent constants it is
|
|
// a rotate right from the lower 8 bits.
|
|
int imm1shift = 8 - right;
|
|
imm1 = 0xff & ((imm << imm1shift) | (imm >> (32 - imm1shift)));
|
|
MOZ_ASSERT((imm1shift & ~0x1e) == 0);
|
|
// left + 8 + mid is the position of the leftmost bit of n_2.
|
|
// We needed to rotate 0x000000ab right by 8 in order to get 0xab000000,
|
|
// then shift again by the leftmost bit in order to get the constant that we
|
|
// care about.
|
|
int imm2shift = mid + 8;
|
|
imm2 = ((imm >> (32 - imm2shift)) | (imm << imm2shift)) & 0xff;
|
|
MOZ_ASSERT((imm1shift & 0x1) == 0);
|
|
MOZ_ASSERT((imm2shift & 0x1) == 0);
|
|
return TwoImm8mData(datastore::Imm8mData(imm1, imm1shift >> 1),
|
|
datastore::Imm8mData(imm2, imm2shift >> 1));
|
|
}
|
|
|
|
ALUOp
|
|
jit::ALUNeg(ALUOp op, Register dest, Register scratch, Imm32* imm, Register* negDest)
|
|
{
|
|
// Find an alternate ALUOp to get the job done, and use a different imm.
|
|
*negDest = dest;
|
|
switch (op) {
|
|
case OpMov:
|
|
*imm = Imm32(~imm->value);
|
|
return OpMvn;
|
|
case OpMvn:
|
|
*imm = Imm32(~imm->value);
|
|
return OpMov;
|
|
case OpAnd:
|
|
*imm = Imm32(~imm->value);
|
|
return OpBic;
|
|
case OpBic:
|
|
*imm = Imm32(~imm->value);
|
|
return OpAnd;
|
|
case OpAdd:
|
|
*imm = Imm32(-imm->value);
|
|
return OpSub;
|
|
case OpSub:
|
|
*imm = Imm32(-imm->value);
|
|
return OpAdd;
|
|
case OpCmp:
|
|
*imm = Imm32(-imm->value);
|
|
return OpCmn;
|
|
case OpCmn:
|
|
*imm = Imm32(-imm->value);
|
|
return OpCmp;
|
|
case OpTst:
|
|
MOZ_ASSERT(dest == InvalidReg);
|
|
*imm = Imm32(~imm->value);
|
|
*negDest = scratch;
|
|
return OpBic;
|
|
// orr has orn on thumb2 only.
|
|
default:
|
|
return OpInvalid;
|
|
}
|
|
}
|
|
|
|
bool
|
|
jit::can_dbl(ALUOp op)
|
|
{
|
|
// Some instructions can't be processed as two separate instructions such as
|
|
// and, and possibly add (when we're setting ccodes). There is also some
|
|
// hilarity with *reading* condition codes. For example, adc dest, src1,
|
|
// 0xfff; (add with carry) can be split up into adc dest, src1, 0xf00; add
|
|
// dest, dest, 0xff, since "reading" the condition code increments the
|
|
// result by one conditionally, that only needs to be done on one of the two
|
|
// instructions.
|
|
switch (op) {
|
|
case OpBic:
|
|
case OpAdd:
|
|
case OpSub:
|
|
case OpEor:
|
|
case OpOrr:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool
|
|
jit::condsAreSafe(ALUOp op) {
|
|
// Even when we are setting condition codes, sometimes we can get away with
|
|
// splitting an operation into two. For example, if our immediate is
|
|
// 0x00ff00ff, and the operation is eors we can split this in half, since x
|
|
// ^ 0x00ff0000 ^ 0x000000ff should set all of its condition codes exactly
|
|
// the same as x ^ 0x00ff00ff. However, if the operation were adds, we
|
|
// cannot split this in half. If the source on the add is 0xfff00ff0, the
|
|
// result sholud be 0xef10ef, but do we set the overflow bit or not?
|
|
// Depending on which half is performed first (0x00ff0000 or 0x000000ff) the
|
|
// V bit will be set differently, and *not* updating the V bit would be
|
|
// wrong. Theoretically, the following should work:
|
|
// adds r0, r1, 0x00ff0000;
|
|
// addsvs r0, r1, 0x000000ff;
|
|
// addvc r0, r1, 0x000000ff;
|
|
// But this is 3 instructions, and at that point, we might as well use
|
|
// something else.
|
|
switch(op) {
|
|
case OpBic:
|
|
case OpOrr:
|
|
case OpEor:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
ALUOp
|
|
jit::getDestVariant(ALUOp op)
|
|
{
|
|
// All of the compare operations are dest-less variants of a standard
|
|
// operation. Given the dest-less variant, return the dest-ful variant.
|
|
switch (op) {
|
|
case OpCmp:
|
|
return OpSub;
|
|
case OpCmn:
|
|
return OpAdd;
|
|
case OpTst:
|
|
return OpAnd;
|
|
case OpTeq:
|
|
return OpEor;
|
|
default:
|
|
return op;
|
|
}
|
|
}
|
|
|
|
O2RegImmShift
|
|
jit::O2Reg(Register r) {
|
|
return O2RegImmShift(r, LSL, 0);
|
|
}
|
|
|
|
O2RegImmShift
|
|
jit::lsl(Register r, int amt)
|
|
{
|
|
MOZ_ASSERT(0 <= amt && amt <= 31);
|
|
return O2RegImmShift(r, LSL, amt);
|
|
}
|
|
|
|
O2RegImmShift
|
|
jit::lsr(Register r, int amt)
|
|
{
|
|
MOZ_ASSERT(1 <= amt && amt <= 32);
|
|
return O2RegImmShift(r, LSR, amt);
|
|
}
|
|
|
|
O2RegImmShift
|
|
jit::ror(Register r, int amt)
|
|
{
|
|
MOZ_ASSERT(1 <= amt && amt <= 31);
|
|
return O2RegImmShift(r, ROR, amt);
|
|
}
|
|
O2RegImmShift
|
|
jit::rol(Register r, int amt)
|
|
{
|
|
MOZ_ASSERT(1 <= amt && amt <= 31);
|
|
return O2RegImmShift(r, ROR, 32 - amt);
|
|
}
|
|
|
|
O2RegImmShift
|
|
jit::asr(Register r, int amt)
|
|
{
|
|
MOZ_ASSERT(1 <= amt && amt <= 32);
|
|
return O2RegImmShift(r, ASR, amt);
|
|
}
|
|
|
|
|
|
O2RegRegShift
|
|
jit::lsl(Register r, Register amt)
|
|
{
|
|
return O2RegRegShift(r, LSL, amt);
|
|
}
|
|
|
|
O2RegRegShift
|
|
jit::lsr(Register r, Register amt)
|
|
{
|
|
return O2RegRegShift(r, LSR, amt);
|
|
}
|
|
|
|
O2RegRegShift
|
|
jit::ror(Register r, Register amt)
|
|
{
|
|
return O2RegRegShift(r, ROR, amt);
|
|
}
|
|
|
|
O2RegRegShift
|
|
jit::asr(Register r, Register amt)
|
|
{
|
|
return O2RegRegShift(r, ASR, amt);
|
|
}
|
|
|
|
static js::jit::DoubleEncoder doubleEncoder;
|
|
|
|
/* static */ const js::jit::VFPImm js::jit::VFPImm::One(0x3FF00000);
|
|
|
|
js::jit::VFPImm::VFPImm(uint32_t top)
|
|
{
|
|
data_ = -1;
|
|
datastore::Imm8VFPImmData tmp;
|
|
if (doubleEncoder.lookup(top, &tmp))
|
|
data_ = tmp.encode();
|
|
}
|
|
|
|
BOffImm::BOffImm(const Instruction& inst)
|
|
: data_(inst.encode() & 0x00ffffff)
|
|
{
|
|
}
|
|
|
|
Instruction*
|
|
BOffImm::getDest(Instruction* src) const
|
|
{
|
|
// TODO: It is probably worthwhile to verify that src is actually a branch.
|
|
// NOTE: This does not explicitly shift the offset of the destination left by 2,
|
|
// since it is indexing into an array of instruction sized objects.
|
|
return &src[((int32_t(data_) << 8) >> 8) + 2];
|
|
}
|
|
|
|
const js::jit::DoubleEncoder::DoubleEntry js::jit::DoubleEncoder::table[256] = {
|
|
#include "jit/arm/DoubleEntryTable.tbl"
|
|
};
|
|
|
|
// VFPRegister implementation
|
|
VFPRegister
|
|
VFPRegister::doubleOverlay(unsigned int which) const
|
|
{
|
|
MOZ_ASSERT(!_isInvalid);
|
|
MOZ_ASSERT(which == 0);
|
|
if (kind != Double)
|
|
return VFPRegister(code_ >> 1, Double);
|
|
return *this;
|
|
}
|
|
VFPRegister
|
|
VFPRegister::singleOverlay(unsigned int which) const
|
|
{
|
|
MOZ_ASSERT(!_isInvalid);
|
|
if (kind == Double) {
|
|
// There are no corresponding float registers for d16-d31.
|
|
MOZ_ASSERT(code_ < 16);
|
|
MOZ_ASSERT(which < 2);
|
|
return VFPRegister((code_ << 1) + which, Single);
|
|
}
|
|
MOZ_ASSERT(which == 0);
|
|
return VFPRegister(code_, Single);
|
|
}
|
|
|
|
VFPRegister
|
|
VFPRegister::sintOverlay(unsigned int which) const
|
|
{
|
|
MOZ_ASSERT(!_isInvalid);
|
|
if (kind == Double) {
|
|
// There are no corresponding float registers for d16-d31.
|
|
MOZ_ASSERT(code_ < 16);
|
|
MOZ_ASSERT(which < 2);
|
|
return VFPRegister((code_ << 1) + which, Int);
|
|
}
|
|
MOZ_ASSERT(which == 0);
|
|
return VFPRegister(code_, Int);
|
|
}
|
|
VFPRegister
|
|
VFPRegister::uintOverlay(unsigned int which) const
|
|
{
|
|
MOZ_ASSERT(!_isInvalid);
|
|
if (kind == Double) {
|
|
// There are no corresponding float registers for d16-d31.
|
|
MOZ_ASSERT(code_ < 16);
|
|
MOZ_ASSERT(which < 2);
|
|
return VFPRegister((code_ << 1) + which, UInt);
|
|
}
|
|
MOZ_ASSERT(which == 0);
|
|
return VFPRegister(code_, UInt);
|
|
}
|
|
|
|
bool
|
|
VFPRegister::isInvalid() const
|
|
{
|
|
return _isInvalid;
|
|
}
|
|
|
|
bool
|
|
VFPRegister::isMissing() const
|
|
{
|
|
MOZ_ASSERT(!_isInvalid);
|
|
return _isMissing;
|
|
}
|
|
|
|
|
|
bool
|
|
Assembler::oom() const
|
|
{
|
|
return AssemblerShared::oom() ||
|
|
m_buffer.oom() ||
|
|
jumpRelocations_.oom() ||
|
|
dataRelocations_.oom() ||
|
|
preBarriers_.oom();
|
|
}
|
|
|
|
// Size of the instruction stream, in bytes. Including pools. This function
|
|
// expects all pools that need to be placed have been placed. If they haven't
|
|
// then we need to go an flush the pools :(
|
|
size_t
|
|
Assembler::size() const
|
|
{
|
|
return m_buffer.size();
|
|
}
|
|
// Size of the relocation table, in bytes.
|
|
size_t
|
|
Assembler::jumpRelocationTableBytes() const
|
|
{
|
|
return jumpRelocations_.length();
|
|
}
|
|
size_t
|
|
Assembler::dataRelocationTableBytes() const
|
|
{
|
|
return dataRelocations_.length();
|
|
}
|
|
|
|
size_t
|
|
Assembler::preBarrierTableBytes() const
|
|
{
|
|
return preBarriers_.length();
|
|
}
|
|
|
|
// Size of the data table, in bytes.
|
|
size_t
|
|
Assembler::bytesNeeded() const
|
|
{
|
|
return size() +
|
|
jumpRelocationTableBytes() +
|
|
dataRelocationTableBytes() +
|
|
preBarrierTableBytes();
|
|
}
|
|
|
|
#ifdef JS_DISASM_ARM
|
|
|
|
void
|
|
Assembler::spewInst(Instruction* i)
|
|
{
|
|
disasm::NameConverter converter;
|
|
disasm::Disassembler dasm(converter);
|
|
disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
|
|
uint8_t* loc = reinterpret_cast<uint8_t*>(const_cast<uint32_t*>(i->raw()));
|
|
dasm.InstructionDecode(buffer, loc);
|
|
printf(" %08x %s\n", reinterpret_cast<uint32_t>(loc), buffer.start());
|
|
}
|
|
|
|
// Labels are named as they are encountered by adding names to a
|
|
// table, using the Label address as the key. This is made tricky by
|
|
// the (memory for) Label objects being reused, but reused label
|
|
// objects are recognizable from being marked as not used or not
|
|
// bound. See spewResolve().
|
|
//
|
|
// In a number of cases there is no information about the target, and
|
|
// we just end up printing "patchable constant load to PC". This is
|
|
// true especially for jumps to bailout handlers (which have no
|
|
// names). See spewData() and its callers. In some cases (loop back
|
|
// edges) some information about the intended target may be propagated
|
|
// from higher levels, and if so it's printed here.
|
|
|
|
void
|
|
Assembler::spew(Instruction* i)
|
|
{
|
|
if (spewDisabled() || !i)
|
|
return;
|
|
disasm::NameConverter converter;
|
|
disasm::Disassembler dasm(converter);
|
|
disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
|
|
uint8_t* loc = reinterpret_cast<uint8_t*>(const_cast<uint32_t*>(i->raw()));
|
|
dasm.InstructionDecode(buffer, loc);
|
|
spew(" %08x %s", reinterpret_cast<uint32_t>(loc), buffer.start());
|
|
}
|
|
|
|
void
|
|
Assembler::spewTarget(Label* target)
|
|
{
|
|
if (spewDisabled())
|
|
return;
|
|
spew(" -> %d%s", spewResolve(target), !target->bound() ? "f" : "");
|
|
}
|
|
|
|
// If a target label is known, always print that and do not attempt to
|
|
// disassemble the branch operands, as they will often be encoding
|
|
// metainformation (pointers for a chain of jump instructions), and
|
|
// not actual branch targets.
|
|
|
|
void
|
|
Assembler::spewBranch(Instruction* i, Label* target /* may be nullptr */)
|
|
{
|
|
if (spewDisabled() || !i)
|
|
return;
|
|
disasm::NameConverter converter;
|
|
disasm::Disassembler dasm(converter);
|
|
disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
|
|
uint8_t* loc = reinterpret_cast<uint8_t*>(const_cast<uint32_t*>(i->raw()));
|
|
dasm.InstructionDecode(buffer, loc);
|
|
char labelBuf[128];
|
|
labelBuf[0] = 0;
|
|
if (!target)
|
|
snprintf(labelBuf, sizeof(labelBuf), " -> (link-time target)");
|
|
if (InstBranchImm::IsTHIS(*i)) {
|
|
InstBranchImm* bimm = InstBranchImm::AsTHIS(*i);
|
|
BOffImm destOff;
|
|
bimm->extractImm(&destOff);
|
|
if (destOff.isInvalid() || target) {
|
|
// The target information in the instruction is likely garbage, so remove it.
|
|
// The target label will in any case be printed if we have it.
|
|
//
|
|
// The format of the instruction disassembly is [0-9a-f]{8}\s+\S+\s+.*,
|
|
// where the \S+ string is the opcode. Strip everything after the opcode,
|
|
// and attach the label if we have it.
|
|
int i;
|
|
for ( i=8 ; i < buffer.length() && buffer[i] == ' ' ; i++ )
|
|
;
|
|
for ( ; i < buffer.length() && buffer[i] != ' ' ; i++ )
|
|
;
|
|
buffer[i] = 0;
|
|
if (target) {
|
|
snprintf(labelBuf, sizeof(labelBuf), " -> %d%s", spewResolve(target),
|
|
!target->bound() ? "f" : "");
|
|
target = nullptr;
|
|
}
|
|
}
|
|
}
|
|
spew(" %08x %s%s", reinterpret_cast<uint32_t>(loc), buffer.start(), labelBuf);
|
|
if (target)
|
|
spewTarget(target);
|
|
}
|
|
|
|
void
|
|
Assembler::spewLabel(Label* l)
|
|
{
|
|
if (spewDisabled())
|
|
return;
|
|
spew(" %d:", spewResolve(l));
|
|
}
|
|
|
|
void
|
|
Assembler::spewRetarget(Label* label, Label* target)
|
|
{
|
|
if (spewDisabled())
|
|
return;
|
|
spew(" %d: .retarget -> %d%s",
|
|
spewResolve(label), spewResolve(target), !target->bound() ? "f" : "");
|
|
}
|
|
|
|
void
|
|
Assembler::spewData(BufferOffset addr, size_t numInstr, bool loadToPC)
|
|
{
|
|
if (spewDisabled())
|
|
return;
|
|
Instruction* inst = m_buffer.getInstOrNull(addr);
|
|
if (!inst)
|
|
return;
|
|
uint32_t *instr = reinterpret_cast<uint32_t*>(inst);
|
|
for ( size_t k=0 ; k < numInstr ; k++ ) {
|
|
spew(" %08x %08x (patchable constant load%s)",
|
|
reinterpret_cast<uint32_t>(instr+k), *(instr+k), loadToPC ? " to PC" : "");
|
|
}
|
|
}
|
|
|
|
bool
|
|
Assembler::spewDisabled()
|
|
{
|
|
return !(JitSpewEnabled(JitSpew_Codegen) || printer_);
|
|
}
|
|
|
|
void
|
|
Assembler::spew(const char* fmt, ...)
|
|
{
|
|
va_list args;
|
|
va_start(args, fmt);
|
|
spew(fmt, args);
|
|
va_end(args);
|
|
}
|
|
|
|
void
|
|
Assembler::spew(const char* fmt, va_list va)
|
|
{
|
|
if (printer_) {
|
|
printer_->vprintf(fmt, va);
|
|
printer_->put("\n");
|
|
}
|
|
js::jit::JitSpewVA(js::jit::JitSpew_Codegen, fmt, va);
|
|
}
|
|
|
|
uint32_t
|
|
Assembler::spewResolve(Label* l)
|
|
{
|
|
// Note, spewResolve will sometimes return 0 when it is triggered
|
|
// by the profiler and not by a full disassembly, since in that
|
|
// case a label can be used or bound but not previously have been
|
|
// defined.
|
|
return l->used() || l->bound() ? spewProbe(l) : spewDefine(l);
|
|
}
|
|
|
|
uint32_t
|
|
Assembler::spewProbe(Label* l)
|
|
{
|
|
uint32_t key = reinterpret_cast<uint32_t>(l);
|
|
uint32_t value = 0;
|
|
spewNodes_.lookup(key, &value);
|
|
return value;
|
|
}
|
|
|
|
uint32_t
|
|
Assembler::spewDefine(Label* l)
|
|
{
|
|
uint32_t key = reinterpret_cast<uint32_t>(l);
|
|
spewNodes_.remove(key);
|
|
uint32_t value = spewNext_++;
|
|
if (!spewNodes_.add(key, value))
|
|
return 0;
|
|
return value;
|
|
}
|
|
|
|
Assembler::SpewNodes::~SpewNodes()
|
|
{
|
|
Node* p = nodes;
|
|
while (p) {
|
|
Node* victim = p;
|
|
p = p->next;
|
|
js_free(victim);
|
|
}
|
|
}
|
|
|
|
bool
|
|
Assembler::SpewNodes::lookup(uint32_t key, uint32_t* value)
|
|
{
|
|
for ( Node* p = nodes ; p ; p = p->next ) {
|
|
if (p->key == key) {
|
|
*value = p->value;
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
Assembler::SpewNodes::add(uint32_t key, uint32_t value)
|
|
{
|
|
Node* node = (Node*)js_malloc(sizeof(Node));
|
|
if (!node)
|
|
return false;
|
|
node->key = key;
|
|
node->value = value;
|
|
node->next = nodes;
|
|
nodes = node;
|
|
return true;
|
|
}
|
|
|
|
bool
|
|
Assembler::SpewNodes::remove(uint32_t key)
|
|
{
|
|
for ( Node* p = nodes, *pp = nullptr ; p ; pp = p, p = p->next ) {
|
|
if (p->key == key) {
|
|
if (pp)
|
|
pp->next = p->next;
|
|
else
|
|
nodes = p->next;
|
|
js_free(p);
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
#endif // JS_DISASM_ARM
|
|
|
|
// Write a blob of binary into the instruction stream.
|
|
BufferOffset
|
|
Assembler::writeInst(uint32_t x)
|
|
{
|
|
BufferOffset offs = m_buffer.putInt(x);
|
|
#ifdef JS_DISASM_ARM
|
|
spew(m_buffer.getInstOrNull(offs));
|
|
#endif
|
|
return offs;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::writeBranchInst(uint32_t x, Label* documentation)
|
|
{
|
|
BufferOffset offs = m_buffer.putInt(x, /* markAsBranch = */ true);
|
|
#ifdef JS_DISASM_ARM
|
|
spewBranch(m_buffer.getInstOrNull(offs), documentation);
|
|
#endif
|
|
return offs;
|
|
}
|
|
|
|
// Allocate memory for a branch instruction, it will be overwritten
|
|
// subsequently and should not be disassembled.
|
|
|
|
BufferOffset
|
|
Assembler::allocBranchInst()
|
|
{
|
|
return m_buffer.putInt(Always | InstNOP::NopInst, /* markAsBranch = */ true);
|
|
}
|
|
|
|
void
|
|
Assembler::WriteInstStatic(uint32_t x, uint32_t* dest)
|
|
{
|
|
MOZ_ASSERT(dest != nullptr);
|
|
*dest = x;
|
|
}
|
|
|
|
void
|
|
Assembler::haltingAlign(int alignment)
|
|
{
|
|
// TODO: Implement a proper halting align.
|
|
nopAlign(alignment);
|
|
}
|
|
|
|
void
|
|
Assembler::nopAlign(int alignment)
|
|
{
|
|
m_buffer.align(alignment);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_nop()
|
|
{
|
|
return writeInst(0xe320f000);
|
|
}
|
|
|
|
static uint32_t
|
|
EncodeAlu(Register dest, Register src1, Operand2 op2, ALUOp op, SBit s, Assembler::Condition c)
|
|
{
|
|
return (int)op | (int)s | (int)c | op2.encode() |
|
|
((dest == InvalidReg) ? 0 : RD(dest)) |
|
|
((src1 == InvalidReg) ? 0 : RN(src1));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_alu(Register dest, Register src1, Operand2 op2,
|
|
ALUOp op, SBit s, Condition c)
|
|
{
|
|
return writeInst(EncodeAlu(dest, src1, op2, op, s, c));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_mov(Register dest, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, InvalidReg, op2, OpMov, s, c);
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::as_alu_patch(Register dest, Register src1, Operand2 op2, ALUOp op, SBit s,
|
|
Condition c, uint32_t* pos)
|
|
{
|
|
WriteInstStatic(EncodeAlu(dest, src1, op2, op, s, c), pos);
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::as_mov_patch(Register dest, Operand2 op2, SBit s, Condition c, uint32_t* pos)
|
|
{
|
|
as_alu_patch(dest, InvalidReg, op2, OpMov, s, c, pos);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_mvn(Register dest, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, InvalidReg, op2, OpMvn, s, c);
|
|
}
|
|
|
|
// Logical operations.
|
|
BufferOffset
|
|
Assembler::as_and(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpAnd, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_bic(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpBic, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_eor(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpEor, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_orr(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpOrr, s, c);
|
|
}
|
|
|
|
// Mathematical operations.
|
|
BufferOffset
|
|
Assembler::as_adc(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpAdc, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_add(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpAdd, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_sbc(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpSbc, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_sub(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpSub, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_rsb(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpRsb, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_rsc(Register dest, Register src1, Operand2 op2, SBit s, Condition c)
|
|
{
|
|
return as_alu(dest, src1, op2, OpRsc, s, c);
|
|
}
|
|
|
|
// Test operations.
|
|
BufferOffset
|
|
Assembler::as_cmn(Register src1, Operand2 op2, Condition c)
|
|
{
|
|
return as_alu(InvalidReg, src1, op2, OpCmn, SetCC, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_cmp(Register src1, Operand2 op2, Condition c)
|
|
{
|
|
return as_alu(InvalidReg, src1, op2, OpCmp, SetCC, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_teq(Register src1, Operand2 op2, Condition c)
|
|
{
|
|
return as_alu(InvalidReg, src1, op2, OpTeq, SetCC, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_tst(Register src1, Operand2 op2, Condition c)
|
|
{
|
|
return as_alu(InvalidReg, src1, op2, OpTst, SetCC, c);
|
|
}
|
|
|
|
static constexpr Register NoAddend = { Registers::pc };
|
|
|
|
static const int SignExtend = 0x06000070;
|
|
|
|
enum SignExtend {
|
|
SxSxtb = 10 << 20,
|
|
SxSxth = 11 << 20,
|
|
SxUxtb = 14 << 20,
|
|
SxUxth = 15 << 20
|
|
};
|
|
|
|
// Sign extension operations.
|
|
BufferOffset
|
|
Assembler::as_sxtb(Register dest, Register src, int rotate, Condition c)
|
|
{
|
|
return writeInst((int)c | SignExtend | SxSxtb | RN(NoAddend) | RD(dest) | ((rotate & 3) << 10) | src.code());
|
|
}
|
|
BufferOffset
|
|
Assembler::as_sxth(Register dest, Register src, int rotate, Condition c)
|
|
{
|
|
return writeInst((int)c | SignExtend | SxSxth | RN(NoAddend) | RD(dest) | ((rotate & 3) << 10) | src.code());
|
|
}
|
|
BufferOffset
|
|
Assembler::as_uxtb(Register dest, Register src, int rotate, Condition c)
|
|
{
|
|
return writeInst((int)c | SignExtend | SxUxtb | RN(NoAddend) | RD(dest) | ((rotate & 3) << 10) | src.code());
|
|
}
|
|
BufferOffset
|
|
Assembler::as_uxth(Register dest, Register src, int rotate, Condition c)
|
|
{
|
|
return writeInst((int)c | SignExtend | SxUxth | RN(NoAddend) | RD(dest) | ((rotate & 3) << 10) | src.code());
|
|
}
|
|
|
|
static uint32_t
|
|
EncodeMovW(Register dest, Imm16 imm, Assembler::Condition c)
|
|
{
|
|
MOZ_ASSERT(HasMOVWT());
|
|
return 0x03000000 | c | imm.encode() | RD(dest);
|
|
}
|
|
|
|
static uint32_t
|
|
EncodeMovT(Register dest, Imm16 imm, Assembler::Condition c)
|
|
{
|
|
MOZ_ASSERT(HasMOVWT());
|
|
return 0x03400000 | c | imm.encode() | RD(dest);
|
|
}
|
|
|
|
// Not quite ALU worthy, but these are useful none the less. These also have
|
|
// the isue of these being formatted completly differently from the standard ALU
|
|
// operations.
|
|
BufferOffset
|
|
Assembler::as_movw(Register dest, Imm16 imm, Condition c)
|
|
{
|
|
return writeInst(EncodeMovW(dest, imm, c));
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::as_movw_patch(Register dest, Imm16 imm, Condition c, Instruction* pos)
|
|
{
|
|
WriteInstStatic(EncodeMovW(dest, imm, c), (uint32_t*)pos);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_movt(Register dest, Imm16 imm, Condition c)
|
|
{
|
|
return writeInst(EncodeMovT(dest, imm, c));
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::as_movt_patch(Register dest, Imm16 imm, Condition c, Instruction* pos)
|
|
{
|
|
WriteInstStatic(EncodeMovT(dest, imm, c), (uint32_t*)pos);
|
|
}
|
|
|
|
static const int mull_tag = 0x90;
|
|
|
|
BufferOffset
|
|
Assembler::as_genmul(Register dhi, Register dlo, Register rm, Register rn,
|
|
MULOp op, SBit s, Condition c)
|
|
{
|
|
|
|
return writeInst(RN(dhi) | maybeRD(dlo) | RM(rm) | rn.code() | op | s | c | mull_tag);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_mul(Register dest, Register src1, Register src2, SBit s, Condition c)
|
|
{
|
|
return as_genmul(dest, InvalidReg, src1, src2, OpmMul, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_mla(Register dest, Register acc, Register src1, Register src2,
|
|
SBit s, Condition c)
|
|
{
|
|
return as_genmul(dest, acc, src1, src2, OpmMla, s, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_umaal(Register destHI, Register destLO, Register src1, Register src2, Condition c)
|
|
{
|
|
return as_genmul(destHI, destLO, src1, src2, OpmUmaal, LeaveCC, c);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_mls(Register dest, Register acc, Register src1, Register src2, Condition c)
|
|
{
|
|
return as_genmul(dest, acc, src1, src2, OpmMls, LeaveCC, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_umull(Register destHI, Register destLO, Register src1, Register src2,
|
|
SBit s, Condition c)
|
|
{
|
|
return as_genmul(destHI, destLO, src1, src2, OpmUmull, s, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_umlal(Register destHI, Register destLO, Register src1, Register src2,
|
|
SBit s, Condition c)
|
|
{
|
|
return as_genmul(destHI, destLO, src1, src2, OpmUmlal, s, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_smull(Register destHI, Register destLO, Register src1, Register src2,
|
|
SBit s, Condition c)
|
|
{
|
|
return as_genmul(destHI, destLO, src1, src2, OpmSmull, s, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_smlal(Register destHI, Register destLO, Register src1, Register src2,
|
|
SBit s, Condition c)
|
|
{
|
|
return as_genmul(destHI, destLO, src1, src2, OpmSmlal, s, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_sdiv(Register rd, Register rn, Register rm, Condition c)
|
|
{
|
|
return writeInst(0x0710f010 | c | RN(rd) | RM(rm) | rn.code());
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_udiv(Register rd, Register rn, Register rm, Condition c)
|
|
{
|
|
return writeInst(0x0730f010 | c | RN(rd) | RM(rm) | rn.code());
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_clz(Register dest, Register src, Condition c)
|
|
{
|
|
MOZ_ASSERT(src != pc && dest != pc);
|
|
return writeInst(RD(dest) | src.code() | c | 0x016f0f10);
|
|
}
|
|
|
|
// Data transfer instructions: ldr, str, ldrb, strb. Using an int to
|
|
// differentiate between 8 bits and 32 bits is overkill, but meh.
|
|
|
|
static uint32_t
|
|
EncodeDtr(LoadStore ls, int size, Index mode, Register rt, DTRAddr addr, Assembler::Condition c)
|
|
{
|
|
MOZ_ASSERT(mode == Offset || (rt != addr.getBase() && pc != addr.getBase()));
|
|
MOZ_ASSERT(size == 32 || size == 8);
|
|
return 0x04000000 | ls | (size == 8 ? 0x00400000 : 0) | mode | c | RT(rt) | addr.encode();
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_dtr(LoadStore ls, int size, Index mode, Register rt, DTRAddr addr, Condition c)
|
|
{
|
|
return writeInst(EncodeDtr(ls, size, mode, rt, addr, c));
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::as_dtr_patch(LoadStore ls, int size, Index mode, Register rt, DTRAddr addr, Condition c,
|
|
uint32_t* dest)
|
|
{
|
|
WriteInstStatic(EncodeDtr(ls, size, mode, rt, addr, c), dest);
|
|
}
|
|
|
|
class PoolHintData
|
|
{
|
|
public:
|
|
enum LoadType {
|
|
// Set 0 to bogus, since that is the value most likely to be
|
|
// accidentally left somewhere.
|
|
PoolBOGUS = 0,
|
|
PoolDTR = 1,
|
|
PoolBranch = 2,
|
|
PoolVDTR = 3
|
|
};
|
|
|
|
private:
|
|
uint32_t index_ : 16;
|
|
uint32_t cond_ : 4;
|
|
LoadType loadType_ : 2;
|
|
uint32_t destReg_ : 5;
|
|
uint32_t destType_ : 1;
|
|
uint32_t ONES : 4;
|
|
|
|
static const uint32_t ExpectedOnes = 0xfu;
|
|
|
|
public:
|
|
void init(uint32_t index, Assembler::Condition cond, LoadType lt, Register destReg) {
|
|
index_ = index;
|
|
MOZ_ASSERT(index_ == index);
|
|
cond_ = cond >> 28;
|
|
MOZ_ASSERT(cond_ == cond >> 28);
|
|
loadType_ = lt;
|
|
ONES = ExpectedOnes;
|
|
destReg_ = destReg.code();
|
|
destType_ = 0;
|
|
}
|
|
void init(uint32_t index, Assembler::Condition cond, LoadType lt, const VFPRegister& destReg) {
|
|
MOZ_ASSERT(destReg.isFloat());
|
|
index_ = index;
|
|
MOZ_ASSERT(index_ == index);
|
|
cond_ = cond >> 28;
|
|
MOZ_ASSERT(cond_ == cond >> 28);
|
|
loadType_ = lt;
|
|
ONES = ExpectedOnes;
|
|
destReg_ = destReg.id();
|
|
destType_ = destReg.isDouble();
|
|
}
|
|
Assembler::Condition getCond() const {
|
|
return Assembler::Condition(cond_ << 28);
|
|
}
|
|
|
|
Register getReg() const {
|
|
return Register::FromCode(destReg_);
|
|
}
|
|
VFPRegister getVFPReg() const {
|
|
VFPRegister r = VFPRegister(destReg_, destType_ ? VFPRegister::Double : VFPRegister::Single);
|
|
return r;
|
|
}
|
|
|
|
int32_t getIndex() const {
|
|
return index_;
|
|
}
|
|
void setIndex(uint32_t index) {
|
|
MOZ_ASSERT(ONES == ExpectedOnes && loadType_ != PoolBOGUS);
|
|
index_ = index;
|
|
MOZ_ASSERT(index_ == index);
|
|
}
|
|
|
|
LoadType getLoadType() const {
|
|
// If this *was* a PoolBranch, but the branch has already been bound
|
|
// then this isn't going to look like a real poolhintdata, but we still
|
|
// want to lie about it so everyone knows it *used* to be a branch.
|
|
if (ONES != ExpectedOnes)
|
|
return PoolHintData::PoolBranch;
|
|
return loadType_;
|
|
}
|
|
|
|
bool isValidPoolHint() const {
|
|
// Most instructions cannot have a condition that is 0xf. Notable
|
|
// exceptions are blx and the entire NEON instruction set. For the
|
|
// purposes of pool loads, and possibly patched branches, the possible
|
|
// instructions are ldr and b, neither of which can have a condition
|
|
// code of 0xf.
|
|
return ONES == ExpectedOnes;
|
|
}
|
|
};
|
|
|
|
union PoolHintPun
|
|
{
|
|
PoolHintData phd;
|
|
uint32_t raw;
|
|
};
|
|
|
|
// Handles all of the other integral data transferring functions: ldrsb, ldrsh,
|
|
// ldrd, etc. The size is given in bits.
|
|
BufferOffset
|
|
Assembler::as_extdtr(LoadStore ls, int size, bool IsSigned, Index mode,
|
|
Register rt, EDtrAddr addr, Condition c)
|
|
{
|
|
int extra_bits2 = 0;
|
|
int extra_bits1 = 0;
|
|
switch(size) {
|
|
case 8:
|
|
MOZ_ASSERT(IsSigned);
|
|
MOZ_ASSERT(ls != IsStore);
|
|
extra_bits1 = 0x1;
|
|
extra_bits2 = 0x2;
|
|
break;
|
|
case 16:
|
|
// 'case 32' doesn't need to be handled, it is handled by the default
|
|
// ldr/str.
|
|
extra_bits2 = 0x01;
|
|
extra_bits1 = (ls == IsStore) ? 0 : 1;
|
|
if (IsSigned) {
|
|
MOZ_ASSERT(ls != IsStore);
|
|
extra_bits2 |= 0x2;
|
|
}
|
|
break;
|
|
case 64:
|
|
extra_bits2 = (ls == IsStore) ? 0x3 : 0x2;
|
|
extra_bits1 = 0;
|
|
break;
|
|
default:
|
|
MOZ_CRASH("unexpected size in as_extdtr");
|
|
}
|
|
return writeInst(extra_bits2 << 5 | extra_bits1 << 20 | 0x90 |
|
|
addr.encode() | RT(rt) | mode | c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_dtm(LoadStore ls, Register rn, uint32_t mask,
|
|
DTMMode mode, DTMWriteBack wb, Condition c)
|
|
{
|
|
return writeInst(0x08000000 | RN(rn) | ls | mode | mask | c | wb);
|
|
}
|
|
|
|
// Note, it's possible for markAsBranch and loadToPC to disagree,
|
|
// because some loads to the PC are not necessarily encoding
|
|
// instructions that should be marked as branches: only patchable
|
|
// near branch instructions should be marked.
|
|
|
|
BufferOffset
|
|
Assembler::allocEntry(size_t numInst, unsigned numPoolEntries,
|
|
uint8_t* inst, uint8_t* data, ARMBuffer::PoolEntry* pe,
|
|
bool markAsBranch, bool loadToPC)
|
|
{
|
|
BufferOffset offs = m_buffer.allocEntry(numInst, numPoolEntries, inst, data, pe, markAsBranch);
|
|
propagateOOM(offs.assigned());
|
|
#ifdef JS_DISASM_ARM
|
|
spewData(offs, numInst, loadToPC);
|
|
#endif
|
|
return offs;
|
|
}
|
|
|
|
// This is also used for instructions that might be resolved into branches,
|
|
// or might not. If dest==pc then it is effectively a branch.
|
|
|
|
BufferOffset
|
|
Assembler::as_Imm32Pool(Register dest, uint32_t value, Condition c)
|
|
{
|
|
PoolHintPun php;
|
|
php.phd.init(0, c, PoolHintData::PoolDTR, dest);
|
|
BufferOffset offs = allocEntry(1, 1, (uint8_t*)&php.raw, (uint8_t*)&value, nullptr, false,
|
|
dest == pc);
|
|
return offs;
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::WritePoolEntry(Instruction* addr, Condition c, uint32_t data)
|
|
{
|
|
MOZ_ASSERT(addr->is<InstLDR>());
|
|
*addr->as<InstLDR>()->dest() = data;
|
|
MOZ_ASSERT(addr->extractCond() == c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_BranchPool(uint32_t value, RepatchLabel* label, ARMBuffer::PoolEntry* pe, Condition c,
|
|
Label* documentation)
|
|
{
|
|
PoolHintPun php;
|
|
php.phd.init(0, c, PoolHintData::PoolBranch, pc);
|
|
BufferOffset ret = allocEntry(1, 1, (uint8_t*)&php.raw, (uint8_t*)&value, pe,
|
|
/* markAsBranch = */ true, /* loadToPC = */ true);
|
|
// If this label is already bound, then immediately replace the stub load
|
|
// with a correct branch.
|
|
if (label->bound()) {
|
|
BufferOffset dest(label);
|
|
BOffImm offset = dest.diffB<BOffImm>(ret);
|
|
if (offset.isInvalid()) {
|
|
m_buffer.fail_bail();
|
|
return ret;
|
|
}
|
|
as_b(offset, c, ret);
|
|
} else if (!oom()) {
|
|
label->use(ret.getOffset());
|
|
}
|
|
#ifdef JS_DISASM_ARM
|
|
if (documentation)
|
|
spewTarget(documentation);
|
|
#endif
|
|
return ret;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_FImm64Pool(VFPRegister dest, wasm::RawF64 value, Condition c)
|
|
{
|
|
MOZ_ASSERT(dest.isDouble());
|
|
PoolHintPun php;
|
|
php.phd.init(0, c, PoolHintData::PoolVDTR, dest);
|
|
uint64_t d = value.bits();
|
|
return allocEntry(1, 2, (uint8_t*)&php.raw, (uint8_t*)&d);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_FImm32Pool(VFPRegister dest, wasm::RawF32 value, Condition c)
|
|
{
|
|
// Insert floats into the double pool as they have the same limitations on
|
|
// immediate offset. This wastes 4 bytes padding per float. An alternative
|
|
// would be to have a separate pool for floats.
|
|
MOZ_ASSERT(dest.isSingle());
|
|
PoolHintPun php;
|
|
php.phd.init(0, c, PoolHintData::PoolVDTR, dest);
|
|
uint32_t f = value.bits();
|
|
return allocEntry(1, 1, (uint8_t*)&php.raw, (uint8_t*)&f);
|
|
}
|
|
|
|
// Pool callbacks stuff:
|
|
void
|
|
Assembler::InsertIndexIntoTag(uint8_t* load_, uint32_t index)
|
|
{
|
|
uint32_t* load = (uint32_t*)load_;
|
|
PoolHintPun php;
|
|
php.raw = *load;
|
|
php.phd.setIndex(index);
|
|
*load = php.raw;
|
|
}
|
|
|
|
// patchConstantPoolLoad takes the address of the instruction that wants to be
|
|
// patched, and the address of the start of the constant pool, and figures
|
|
// things out from there.
|
|
void
|
|
Assembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr)
|
|
{
|
|
PoolHintData data = *(PoolHintData*)loadAddr;
|
|
uint32_t* instAddr = (uint32_t*) loadAddr;
|
|
int offset = (char*)constPoolAddr - (char*)loadAddr;
|
|
switch(data.getLoadType()) {
|
|
case PoolHintData::PoolBOGUS:
|
|
MOZ_CRASH("bogus load type!");
|
|
case PoolHintData::PoolDTR:
|
|
Assembler::as_dtr_patch(IsLoad, 32, Offset, data.getReg(),
|
|
DTRAddr(pc, DtrOffImm(offset+4*data.getIndex() - 8)),
|
|
data.getCond(), instAddr);
|
|
break;
|
|
case PoolHintData::PoolBranch:
|
|
// Either this used to be a poolBranch, and the label was already bound,
|
|
// so it was replaced with a real branch, or this may happen in the
|
|
// future. If this is going to happen in the future, then the actual
|
|
// bits that are written here don't matter (except the condition code,
|
|
// since that is always preserved across patchings) but if it does not
|
|
// get bound later, then we want to make sure this is a load from the
|
|
// pool entry (and the pool entry should be nullptr so it will crash).
|
|
if (data.isValidPoolHint()) {
|
|
Assembler::as_dtr_patch(IsLoad, 32, Offset, pc,
|
|
DTRAddr(pc, DtrOffImm(offset+4*data.getIndex() - 8)),
|
|
data.getCond(), instAddr);
|
|
}
|
|
break;
|
|
case PoolHintData::PoolVDTR: {
|
|
VFPRegister dest = data.getVFPReg();
|
|
int32_t imm = offset + (data.getIndex() * 4) - 8;
|
|
MOZ_ASSERT(-1024 < imm && imm < 1024);
|
|
Assembler::as_vdtr_patch(IsLoad, dest, VFPAddr(pc, VFPOffImm(imm)), data.getCond(),
|
|
instAddr);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Atomic instruction stuff:
|
|
|
|
BufferOffset
|
|
Assembler::as_ldrex(Register rt, Register rn, Condition c)
|
|
{
|
|
return writeInst(0x01900f9f | (int)c | RT(rt) | RN(rn));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_ldrexh(Register rt, Register rn, Condition c)
|
|
{
|
|
return writeInst(0x01f00f9f | (int)c | RT(rt) | RN(rn));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_ldrexb(Register rt, Register rn, Condition c)
|
|
{
|
|
return writeInst(0x01d00f9f | (int)c | RT(rt) | RN(rn));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_strex(Register rd, Register rt, Register rn, Condition c)
|
|
{
|
|
MOZ_ASSERT(rd != rn && rd != rt); // True restriction on Cortex-A7 (RPi2)
|
|
return writeInst(0x01800f90 | (int)c | RD(rd) | RN(rn) | rt.code());
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_strexh(Register rd, Register rt, Register rn, Condition c)
|
|
{
|
|
MOZ_ASSERT(rd != rn && rd != rt); // True restriction on Cortex-A7 (RPi2)
|
|
return writeInst(0x01e00f90 | (int)c | RD(rd) | RN(rn) | rt.code());
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_strexb(Register rd, Register rt, Register rn, Condition c)
|
|
{
|
|
MOZ_ASSERT(rd != rn && rd != rt); // True restriction on Cortex-A7 (RPi2)
|
|
return writeInst(0x01c00f90 | (int)c | RD(rd) | RN(rn) | rt.code());
|
|
}
|
|
|
|
// Memory barrier stuff:
|
|
|
|
BufferOffset
|
|
Assembler::as_dmb(BarrierOption option)
|
|
{
|
|
return writeInst(0xf57ff050U | (int)option);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_dsb(BarrierOption option)
|
|
{
|
|
return writeInst(0xf57ff040U | (int)option);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_isb()
|
|
{
|
|
return writeInst(0xf57ff06fU); // option == SY
|
|
}
|
|
BufferOffset
|
|
Assembler::as_dsb_trap()
|
|
{
|
|
// DSB is "mcr 15, 0, r0, c7, c10, 4".
|
|
// See eg https://bugs.kde.org/show_bug.cgi?id=228060.
|
|
// ARMv7 manual, "VMSA CP15 c7 register summary".
|
|
// Flagged as "legacy" starting with ARMv8, may be disabled on chip, see
|
|
// ARMv8 manual E2.7.3 and G3.18.16.
|
|
return writeInst(0xee070f9a);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_dmb_trap()
|
|
{
|
|
// DMB is "mcr 15, 0, r0, c7, c10, 5".
|
|
// ARMv7 manual, "VMSA CP15 c7 register summary".
|
|
// Flagged as "legacy" starting with ARMv8, may be disabled on chip, see
|
|
// ARMv8 manual E2.7.3 and G3.18.16.
|
|
return writeInst(0xee070fba);
|
|
}
|
|
BufferOffset
|
|
Assembler::as_isb_trap()
|
|
{
|
|
// ISB is "mcr 15, 0, r0, c7, c5, 4".
|
|
// ARMv7 manual, "VMSA CP15 c7 register summary".
|
|
// Flagged as "legacy" starting with ARMv8, may be disabled on chip, see
|
|
// ARMv8 manual E2.7.3 and G3.18.16.
|
|
return writeInst(0xee070f94);
|
|
}
|
|
|
|
// Control flow stuff:
|
|
|
|
// bx can *only* branch to a register, never to an immediate.
|
|
BufferOffset
|
|
Assembler::as_bx(Register r, Condition c)
|
|
{
|
|
BufferOffset ret = writeInst(((int) c) | OpBx | r.code());
|
|
return ret;
|
|
}
|
|
|
|
void
|
|
Assembler::WritePoolGuard(BufferOffset branch, Instruction* dest, BufferOffset afterPool)
|
|
{
|
|
BOffImm off = afterPool.diffB<BOffImm>(branch);
|
|
if (off.isInvalid())
|
|
MOZ_CRASH("BOffImm invalid");
|
|
*dest = InstBImm(off, Always);
|
|
}
|
|
|
|
// Branch can branch to an immediate *or* to a register.
|
|
// Branches to immediates are pc relative, branches to registers are absolute.
|
|
BufferOffset
|
|
Assembler::as_b(BOffImm off, Condition c, Label* documentation)
|
|
{
|
|
BufferOffset ret = writeBranchInst(((int)c) | OpB | off.encode(), documentation);
|
|
return ret;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_b(Label* l, Condition c)
|
|
{
|
|
if (l->bound()) {
|
|
// Note only one instruction is emitted here, the NOP is overwritten.
|
|
BufferOffset ret = allocBranchInst();
|
|
if (oom())
|
|
return BufferOffset();
|
|
|
|
BOffImm off = BufferOffset(l).diffB<BOffImm>(ret);
|
|
if (off.isInvalid()) {
|
|
m_buffer.fail_bail();
|
|
return BufferOffset();
|
|
}
|
|
as_b(off, c, ret);
|
|
#ifdef JS_DISASM_ARM
|
|
spewBranch(m_buffer.getInstOrNull(ret), l);
|
|
#endif
|
|
return ret;
|
|
}
|
|
|
|
if (oom())
|
|
return BufferOffset();
|
|
|
|
int32_t old;
|
|
BufferOffset ret;
|
|
if (l->used()) {
|
|
old = l->offset();
|
|
// This will currently throw an assertion if we couldn't actually
|
|
// encode the offset of the branch.
|
|
if (!BOffImm::IsInRange(old)) {
|
|
m_buffer.fail_bail();
|
|
return ret;
|
|
}
|
|
ret = as_b(BOffImm(old), c, l);
|
|
} else {
|
|
old = LabelBase::INVALID_OFFSET;
|
|
BOffImm inv;
|
|
ret = as_b(inv, c, l);
|
|
}
|
|
|
|
if (oom())
|
|
return BufferOffset();
|
|
|
|
DebugOnly<int32_t> check = l->use(ret.getOffset());
|
|
MOZ_ASSERT(check == old);
|
|
return ret;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_b(wasm::TrapDesc target, Condition c)
|
|
{
|
|
Label l;
|
|
BufferOffset ret = as_b(&l, c);
|
|
bindLater(&l, target);
|
|
return ret;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_b(BOffImm off, Condition c, BufferOffset inst)
|
|
{
|
|
// JS_DISASM_ARM NOTE: Can't disassemble here, because numerous callers use this to
|
|
// patchup old code. Must disassemble in caller where it makes sense. Not many callers.
|
|
*editSrc(inst) = InstBImm(off, c);
|
|
return inst;
|
|
}
|
|
|
|
// blx can go to either an immediate or a register.
|
|
// When blx'ing to a register, we change processor state depending on the low
|
|
// bit of the register when blx'ing to an immediate, we *always* change
|
|
// processor state.
|
|
|
|
BufferOffset
|
|
Assembler::as_blx(Register r, Condition c)
|
|
{
|
|
return writeInst(((int) c) | OpBlx | r.code());
|
|
}
|
|
|
|
// bl can only branch to an pc-relative immediate offset
|
|
// It cannot change the processor state.
|
|
BufferOffset
|
|
Assembler::as_bl(BOffImm off, Condition c, Label* documentation)
|
|
{
|
|
return writeBranchInst(((int)c) | OpBl | off.encode(), documentation);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_bl(Label* l, Condition c)
|
|
{
|
|
if (l->bound()) {
|
|
// Note only one instruction is emitted here, the NOP is overwritten.
|
|
BufferOffset ret = allocBranchInst();
|
|
if (oom())
|
|
return BufferOffset();
|
|
|
|
BOffImm offset = BufferOffset(l).diffB<BOffImm>(ret);
|
|
if (offset.isInvalid()) {
|
|
m_buffer.fail_bail();
|
|
return BufferOffset();
|
|
}
|
|
|
|
as_bl(offset, c, ret);
|
|
#ifdef JS_DISASM_ARM
|
|
spewBranch(m_buffer.getInstOrNull(ret), l);
|
|
#endif
|
|
return ret;
|
|
}
|
|
|
|
if (oom())
|
|
return BufferOffset();
|
|
|
|
int32_t old;
|
|
BufferOffset ret;
|
|
// See if the list was empty :(
|
|
if (l->used()) {
|
|
// This will currently throw an assertion if we couldn't actually encode
|
|
// the offset of the branch.
|
|
old = l->offset();
|
|
if (!BOffImm::IsInRange(old)) {
|
|
m_buffer.fail_bail();
|
|
return ret;
|
|
}
|
|
ret = as_bl(BOffImm(old), c, l);
|
|
} else {
|
|
old = LabelBase::INVALID_OFFSET;
|
|
BOffImm inv;
|
|
ret = as_bl(inv, c, l);
|
|
}
|
|
|
|
if (oom())
|
|
return BufferOffset();
|
|
|
|
DebugOnly<int32_t> check = l->use(ret.getOffset());
|
|
MOZ_ASSERT(check == old);
|
|
return ret;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_bl(BOffImm off, Condition c, BufferOffset inst)
|
|
{
|
|
*editSrc(inst) = InstBLImm(off, c);
|
|
return inst;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_mrs(Register r, Condition c)
|
|
{
|
|
return writeInst(0x010f0000 | int(c) | RD(r));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_msr(Register r, Condition c)
|
|
{
|
|
// Hardcode the 'mask' field to 0b11 for now. It is bits 18 and 19, which
|
|
// are the two high bits of the 'c' in this constant.
|
|
MOZ_ASSERT((r.code() & ~0xf) == 0);
|
|
return writeInst(0x012cf000 | int(c) | r.code());
|
|
}
|
|
|
|
// VFP instructions!
|
|
enum vfp_tags {
|
|
VfpTag = 0x0C000A00,
|
|
VfpArith = 0x02000000
|
|
};
|
|
|
|
BufferOffset
|
|
Assembler::writeVFPInst(vfp_size sz, uint32_t blob)
|
|
{
|
|
MOZ_ASSERT((sz & blob) == 0);
|
|
MOZ_ASSERT((VfpTag & blob) == 0);
|
|
return writeInst(VfpTag | sz | blob);
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::WriteVFPInstStatic(vfp_size sz, uint32_t blob, uint32_t* dest)
|
|
{
|
|
MOZ_ASSERT((sz & blob) == 0);
|
|
MOZ_ASSERT((VfpTag & blob) == 0);
|
|
WriteInstStatic(VfpTag | sz | blob, dest);
|
|
}
|
|
|
|
// Unityped variants: all registers hold the same (ieee754 single/double)
|
|
// notably not included are vcvt; vmov vd, #imm; vmov rt, vn.
|
|
BufferOffset
|
|
Assembler::as_vfp_float(VFPRegister vd, VFPRegister vn, VFPRegister vm,
|
|
VFPOp op, Condition c)
|
|
{
|
|
// Make sure we believe that all of our operands are the same kind.
|
|
MOZ_ASSERT_IF(!vn.isMissing(), vd.equiv(vn));
|
|
MOZ_ASSERT_IF(!vm.isMissing(), vd.equiv(vm));
|
|
vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
|
|
return writeVFPInst(sz, VD(vd) | VN(vn) | VM(vm) | op | VfpArith | c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vadd(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, vn, vm, OpvAdd, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vdiv(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, vn, vm, OpvDiv, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vmul(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, vn, vm, OpvMul, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vnmul(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, vn, vm, OpvMul, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vnmla(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
MOZ_CRASH("Feature NYI");
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vnmls(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
MOZ_CRASH("Feature NYI");
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vneg(VFPRegister vd, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, NoVFPRegister, vm, OpvNeg, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vsqrt(VFPRegister vd, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, NoVFPRegister, vm, OpvSqrt, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vabs(VFPRegister vd, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, NoVFPRegister, vm, OpvAbs, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vsub(VFPRegister vd, VFPRegister vn, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, vn, vm, OpvSub, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vcmp(VFPRegister vd, VFPRegister vm, Condition c)
|
|
{
|
|
return as_vfp_float(vd, NoVFPRegister, vm, OpvCmp, c);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vcmpz(VFPRegister vd, Condition c)
|
|
{
|
|
return as_vfp_float(vd, NoVFPRegister, NoVFPRegister, OpvCmpz, c);
|
|
}
|
|
|
|
// Specifically, a move between two same sized-registers.
|
|
BufferOffset
|
|
Assembler::as_vmov(VFPRegister vd, VFPRegister vsrc, Condition c)
|
|
{
|
|
return as_vfp_float(vd, NoVFPRegister, vsrc, OpvMov, c);
|
|
}
|
|
|
|
// Transfer between Core and VFP.
|
|
|
|
// Unlike the next function, moving between the core registers and vfp registers
|
|
// can't be *that* properly typed. Namely, since I don't want to munge the type
|
|
// VFPRegister to also include core registers. Thus, the core and vfp registers
|
|
// are passed in based on their type, and src/dest is determined by the
|
|
// float2core.
|
|
|
|
BufferOffset
|
|
Assembler::as_vxfer(Register vt1, Register vt2, VFPRegister vm, FloatToCore_ f2c,
|
|
Condition c, int idx)
|
|
{
|
|
vfp_size sz = IsSingle;
|
|
if (vm.isDouble()) {
|
|
// Technically, this can be done with a vmov à la ARM ARM under vmov
|
|
// however, that requires at least an extra bit saying if the operation
|
|
// should be performed on the lower or upper half of the double. Moving
|
|
// a single to/from 2N/2N+1 isn't equivalent, since there are 32 single
|
|
// registers, and 32 double registers so there is no way to encode the
|
|
// last 16 double registers.
|
|
sz = IsDouble;
|
|
MOZ_ASSERT(idx == 0 || idx == 1);
|
|
// If we are transferring a single half of the double then it must be
|
|
// moving a VFP reg to a core reg.
|
|
MOZ_ASSERT_IF(vt2 == InvalidReg, f2c == FloatToCore);
|
|
idx = idx << 21;
|
|
} else {
|
|
MOZ_ASSERT(idx == 0);
|
|
}
|
|
|
|
if (vt2 == InvalidReg)
|
|
return writeVFPInst(sz, WordTransfer | f2c | c | RT(vt1) | maybeRN(vt2) | VN(vm) | idx);
|
|
|
|
// We are doing a 64 bit transfer.
|
|
return writeVFPInst(sz, DoubleTransfer | f2c | c | RT(vt1) | maybeRN(vt2) | VM(vm) | idx);
|
|
}
|
|
|
|
enum vcvt_destFloatness {
|
|
VcvtToInteger = 1 << 18,
|
|
VcvtToFloat = 0 << 18
|
|
};
|
|
enum vcvt_toZero {
|
|
VcvtToZero = 1 << 7, // Use the default rounding mode, which rounds truncates.
|
|
VcvtToFPSCR = 0 << 7 // Use whatever rounding mode the fpscr specifies.
|
|
};
|
|
enum vcvt_Signedness {
|
|
VcvtToSigned = 1 << 16,
|
|
VcvtToUnsigned = 0 << 16,
|
|
VcvtFromSigned = 1 << 7,
|
|
VcvtFromUnsigned = 0 << 7
|
|
};
|
|
|
|
// Our encoding actually allows just the src and the dest (and their types) to
|
|
// uniquely specify the encoding that we are going to use.
|
|
BufferOffset
|
|
Assembler::as_vcvt(VFPRegister vd, VFPRegister vm, bool useFPSCR,
|
|
Condition c)
|
|
{
|
|
// Unlike other cases, the source and dest types cannot be the same.
|
|
MOZ_ASSERT(!vd.equiv(vm));
|
|
vfp_size sz = IsDouble;
|
|
if (vd.isFloat() && vm.isFloat()) {
|
|
// Doing a float -> float conversion.
|
|
if (vm.isSingle())
|
|
sz = IsSingle;
|
|
return writeVFPInst(sz, c | 0x02B700C0 | VM(vm) | VD(vd));
|
|
}
|
|
|
|
// At least one of the registers should be a float.
|
|
vcvt_destFloatness destFloat;
|
|
vcvt_Signedness opSign;
|
|
vcvt_toZero doToZero = VcvtToFPSCR;
|
|
MOZ_ASSERT(vd.isFloat() || vm.isFloat());
|
|
if (vd.isSingle() || vm.isSingle())
|
|
sz = IsSingle;
|
|
|
|
if (vd.isFloat()) {
|
|
destFloat = VcvtToFloat;
|
|
opSign = (vm.isSInt()) ? VcvtFromSigned : VcvtFromUnsigned;
|
|
} else {
|
|
destFloat = VcvtToInteger;
|
|
opSign = (vd.isSInt()) ? VcvtToSigned : VcvtToUnsigned;
|
|
doToZero = useFPSCR ? VcvtToFPSCR : VcvtToZero;
|
|
}
|
|
return writeVFPInst(sz, c | 0x02B80040 | VD(vd) | VM(vm) | destFloat | opSign | doToZero);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vcvtFixed(VFPRegister vd, bool isSigned, uint32_t fixedPoint, bool toFixed, Condition c)
|
|
{
|
|
MOZ_ASSERT(vd.isFloat());
|
|
uint32_t sx = 0x1;
|
|
vfp_size sf = vd.isDouble() ? IsDouble : IsSingle;
|
|
int32_t imm5 = fixedPoint;
|
|
imm5 = (sx ? 32 : 16) - imm5;
|
|
MOZ_ASSERT(imm5 >= 0);
|
|
imm5 = imm5 >> 1 | (imm5 & 1) << 5;
|
|
return writeVFPInst(sf, 0x02BA0040 | VD(vd) | toFixed << 18 | sx << 7 |
|
|
(!isSigned) << 16 | imm5 | c);
|
|
}
|
|
|
|
// Transfer between VFP and memory.
|
|
static uint32_t
|
|
EncodeVdtr(LoadStore ls, VFPRegister vd, VFPAddr addr, Assembler::Condition c)
|
|
{
|
|
return ls | 0x01000000 | addr.encode() | VD(vd) | c;
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vdtr(LoadStore ls, VFPRegister vd, VFPAddr addr,
|
|
Condition c /* vfp doesn't have a wb option */)
|
|
{
|
|
vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
|
|
return writeVFPInst(sz, EncodeVdtr(ls, vd, addr, c));
|
|
}
|
|
|
|
/* static */ void
|
|
Assembler::as_vdtr_patch(LoadStore ls, VFPRegister vd, VFPAddr addr, Condition c, uint32_t* dest)
|
|
{
|
|
vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
|
|
WriteVFPInstStatic(sz, EncodeVdtr(ls, vd, addr, c), dest);
|
|
}
|
|
|
|
// VFP's ldm/stm work differently from the standard arm ones. You can only
|
|
// transfer a range.
|
|
|
|
BufferOffset
|
|
Assembler::as_vdtm(LoadStore st, Register rn, VFPRegister vd, int length,
|
|
/* also has update conditions */ Condition c)
|
|
{
|
|
MOZ_ASSERT(length <= 16 && length >= 0);
|
|
vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
|
|
|
|
if (vd.isDouble())
|
|
length *= 2;
|
|
|
|
return writeVFPInst(sz, dtmLoadStore | RN(rn) | VD(vd) | length |
|
|
dtmMode | dtmUpdate | dtmCond);
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vimm(VFPRegister vd, VFPImm imm, Condition c)
|
|
{
|
|
MOZ_ASSERT(imm.isValid());
|
|
vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
|
|
return writeVFPInst(sz, c | imm.encode() | VD(vd) | 0x02B00000);
|
|
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vmrs(Register r, Condition c)
|
|
{
|
|
return writeInst(c | 0x0ef10a10 | RT(r));
|
|
}
|
|
|
|
BufferOffset
|
|
Assembler::as_vmsr(Register r, Condition c)
|
|
{
|
|
return writeInst(c | 0x0ee10a10 | RT(r));
|
|
}
|
|
|
|
bool
|
|
Assembler::nextLink(BufferOffset b, BufferOffset* next)
|
|
{
|
|
Instruction branch = *editSrc(b);
|
|
MOZ_ASSERT(branch.is<InstBranchImm>());
|
|
|
|
BOffImm destOff;
|
|
branch.as<InstBranchImm>()->extractImm(&destOff);
|
|
if (destOff.isInvalid())
|
|
return false;
|
|
|
|
// Propagate the next link back to the caller, by constructing a new
|
|
// BufferOffset into the space they provided.
|
|
new (next) BufferOffset(destOff.decode());
|
|
return true;
|
|
}
|
|
|
|
void
|
|
Assembler::bind(Label* label, BufferOffset boff)
|
|
{
|
|
#ifdef JS_DISASM_ARM
|
|
spewLabel(label);
|
|
#endif
|
|
if (oom()) {
|
|
// Ensure we always bind the label. This matches what we do on
|
|
// x86/x64 and silences the assert in ~Label.
|
|
label->bind(0);
|
|
return;
|
|
}
|
|
|
|
if (label->used()) {
|
|
bool more;
|
|
// If our caller didn't give us an explicit target to bind to then we
|
|
// want to bind to the location of the next instruction.
|
|
BufferOffset dest = boff.assigned() ? boff : nextOffset();
|
|
BufferOffset b(label);
|
|
do {
|
|
BufferOffset next;
|
|
more = nextLink(b, &next);
|
|
Instruction branch = *editSrc(b);
|
|
Condition c = branch.extractCond();
|
|
BOffImm offset = dest.diffB<BOffImm>(b);
|
|
if (offset.isInvalid()) {
|
|
m_buffer.fail_bail();
|
|
return;
|
|
}
|
|
if (branch.is<InstBImm>())
|
|
as_b(offset, c, b);
|
|
else if (branch.is<InstBLImm>())
|
|
as_bl(offset, c, b);
|
|
else
|
|
MOZ_CRASH("crazy fixup!");
|
|
b = next;
|
|
} while (more);
|
|
}
|
|
label->bind(nextOffset().getOffset());
|
|
MOZ_ASSERT(!oom());
|
|
}
|
|
|
|
void
|
|
Assembler::bindLater(Label* label, wasm::TrapDesc target)
|
|
{
|
|
if (label->used()) {
|
|
BufferOffset b(label);
|
|
do {
|
|
append(wasm::TrapSite(target, b.getOffset()));
|
|
} while (nextLink(b, &b));
|
|
}
|
|
label->reset();
|
|
}
|
|
|
|
void
|
|
Assembler::bind(RepatchLabel* label)
|
|
{
|
|
// It does not seem to be useful to record this label for
|
|
// disassembly, as the value that is bound to the label is often
|
|
// effectively garbage and is replaced by something else later.
|
|
BufferOffset dest = nextOffset();
|
|
if (label->used() && !oom()) {
|
|
// If the label has a use, then change this use to refer to the bound
|
|
// label.
|
|
BufferOffset branchOff(label->offset());
|
|
// Since this was created with a RepatchLabel, the value written in the
|
|
// instruction stream is not branch shaped, it is PoolHintData shaped.
|
|
Instruction* branch = editSrc(branchOff);
|
|
PoolHintPun p;
|
|
p.raw = branch->encode();
|
|
Condition cond;
|
|
if (p.phd.isValidPoolHint())
|
|
cond = p.phd.getCond();
|
|
else
|
|
cond = branch->extractCond();
|
|
|
|
BOffImm offset = dest.diffB<BOffImm>(branchOff);
|
|
if (offset.isInvalid()) {
|
|
m_buffer.fail_bail();
|
|
return;
|
|
}
|
|
as_b(offset, cond, branchOff);
|
|
}
|
|
label->bind(dest.getOffset());
|
|
}
|
|
|
|
void
|
|
Assembler::retarget(Label* label, Label* target)
|
|
{
|
|
#ifdef JS_DISASM_ARM
|
|
spewRetarget(label, target);
|
|
#endif
|
|
if (label->used() && !oom()) {
|
|
if (target->bound()) {
|
|
bind(label, BufferOffset(target));
|
|
} else if (target->used()) {
|
|
// The target is not bound but used. Prepend label's branch list
|
|
// onto target's.
|
|
BufferOffset labelBranchOffset(label);
|
|
BufferOffset next;
|
|
|
|
// Find the head of the use chain for label.
|
|
while (nextLink(labelBranchOffset, &next))
|
|
labelBranchOffset = next;
|
|
|
|
// Then patch the head of label's use chain to the tail of target's
|
|
// use chain, prepending the entire use chain of target.
|
|
Instruction branch = *editSrc(labelBranchOffset);
|
|
Condition c = branch.extractCond();
|
|
int32_t prev = target->use(label->offset());
|
|
if (branch.is<InstBImm>())
|
|
as_b(BOffImm(prev), c, labelBranchOffset);
|
|
else if (branch.is<InstBLImm>())
|
|
as_bl(BOffImm(prev), c, labelBranchOffset);
|
|
else
|
|
MOZ_CRASH("crazy fixup!");
|
|
} else {
|
|
// The target is unbound and unused. We can just take the head of
|
|
// the list hanging off of label, and dump that into target.
|
|
DebugOnly<uint32_t> prev = target->use(label->offset());
|
|
MOZ_ASSERT((int32_t)prev == Label::INVALID_OFFSET);
|
|
}
|
|
}
|
|
label->reset();
|
|
|
|
}
|
|
|
|
static int stopBKPT = -1;
|
|
void
|
|
Assembler::as_bkpt()
|
|
{
|
|
// This is a count of how many times a breakpoint instruction has been
|
|
// generated. It is embedded into the instruction for debugging
|
|
// purposes. Gdb will print "bkpt xxx" when you attempt to dissassemble a
|
|
// breakpoint with the number xxx embedded into it. If this breakpoint is
|
|
// being hit, then you can run (in gdb):
|
|
// >b dbg_break
|
|
// >b main
|
|
// >commands
|
|
// >set stopBKPT = xxx
|
|
// >c
|
|
// >end
|
|
// which will set a breakpoint on the function dbg_break above set a
|
|
// scripted breakpoint on main that will set the (otherwise unmodified)
|
|
// value to the number of the breakpoint, so dbg_break will actuall be
|
|
// called and finally, when you run the executable, execution will halt when
|
|
// that breakpoint is generated.
|
|
static int hit = 0;
|
|
if (stopBKPT == hit)
|
|
dbg_break();
|
|
writeInst(0xe1200070 | (hit & 0xf) | ((hit & 0xfff0) << 4));
|
|
hit++;
|
|
}
|
|
|
|
void
|
|
Assembler::flushBuffer()
|
|
{
|
|
m_buffer.flushPool();
|
|
}
|
|
|
|
void
|
|
Assembler::enterNoPool(size_t maxInst)
|
|
{
|
|
m_buffer.enterNoPool(maxInst);
|
|
}
|
|
|
|
void
|
|
Assembler::leaveNoPool()
|
|
{
|
|
m_buffer.leaveNoPool();
|
|
}
|
|
|
|
ptrdiff_t
|
|
Assembler::GetBranchOffset(const Instruction* i_)
|
|
{
|
|
MOZ_ASSERT(i_->is<InstBranchImm>());
|
|
InstBranchImm* i = i_->as<InstBranchImm>();
|
|
BOffImm dest;
|
|
i->extractImm(&dest);
|
|
return dest.decode();
|
|
}
|
|
|
|
void
|
|
Assembler::RetargetNearBranch(Instruction* i, int offset, bool final)
|
|
{
|
|
Assembler::Condition c = i->extractCond();
|
|
RetargetNearBranch(i, offset, c, final);
|
|
}
|
|
|
|
void
|
|
Assembler::RetargetNearBranch(Instruction* i, int offset, Condition cond, bool final)
|
|
{
|
|
// Retargeting calls is totally unsupported!
|
|
MOZ_ASSERT_IF(i->is<InstBranchImm>(), i->is<InstBImm>() || i->is<InstBLImm>());
|
|
if (i->is<InstBLImm>())
|
|
new (i) InstBLImm(BOffImm(offset), cond);
|
|
else
|
|
new (i) InstBImm(BOffImm(offset), cond);
|
|
|
|
// Flush the cache, since an instruction was overwritten.
|
|
if (final)
|
|
AutoFlushICache::flush(uintptr_t(i), 4);
|
|
}
|
|
|
|
void
|
|
Assembler::RetargetFarBranch(Instruction* i, uint8_t** slot, uint8_t* dest, Condition cond)
|
|
{
|
|
int32_t offset = reinterpret_cast<uint8_t*>(slot) - reinterpret_cast<uint8_t*>(i);
|
|
if (!i->is<InstLDR>()) {
|
|
new (i) InstLDR(Offset, pc, DTRAddr(pc, DtrOffImm(offset - 8)), cond);
|
|
AutoFlushICache::flush(uintptr_t(i), 4);
|
|
}
|
|
*slot = dest;
|
|
}
|
|
|
|
struct PoolHeader : Instruction
|
|
{
|
|
struct Header
|
|
{
|
|
// The size should take into account the pool header.
|
|
// The size is in units of Instruction (4 bytes), not byte.
|
|
uint32_t size : 15;
|
|
bool isNatural : 1;
|
|
uint32_t ONES : 16;
|
|
|
|
Header(int size_, bool isNatural_)
|
|
: size(size_),
|
|
isNatural(isNatural_),
|
|
ONES(0xffff)
|
|
{ }
|
|
|
|
Header(const Instruction* i) {
|
|
JS_STATIC_ASSERT(sizeof(Header) == sizeof(uint32_t));
|
|
memcpy(this, i, sizeof(Header));
|
|
MOZ_ASSERT(ONES == 0xffff);
|
|
}
|
|
|
|
uint32_t raw() const {
|
|
JS_STATIC_ASSERT(sizeof(Header) == sizeof(uint32_t));
|
|
uint32_t dest;
|
|
memcpy(&dest, this, sizeof(Header));
|
|
return dest;
|
|
}
|
|
};
|
|
|
|
PoolHeader(int size_, bool isNatural_)
|
|
: Instruction(Header(size_, isNatural_).raw(), true)
|
|
{ }
|
|
|
|
uint32_t size() const {
|
|
Header tmp(this);
|
|
return tmp.size;
|
|
}
|
|
uint32_t isNatural() const {
|
|
Header tmp(this);
|
|
return tmp.isNatural;
|
|
}
|
|
|
|
static bool IsTHIS(const Instruction& i) {
|
|
return (*i.raw() & 0xffff0000) == 0xffff0000;
|
|
}
|
|
static const PoolHeader* AsTHIS(const Instruction& i) {
|
|
if (!IsTHIS(i))
|
|
return nullptr;
|
|
return static_cast<const PoolHeader*>(&i);
|
|
}
|
|
};
|
|
|
|
void
|
|
Assembler::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural)
|
|
{
|
|
static_assert(sizeof(PoolHeader) == 4, "PoolHandler must have the correct size.");
|
|
uint8_t* pool = start + 4;
|
|
// Go through the usual rigmarole to get the size of the pool.
|
|
pool += p->getPoolSize();
|
|
uint32_t size = pool - start;
|
|
MOZ_ASSERT((size & 3) == 0);
|
|
size = size >> 2;
|
|
MOZ_ASSERT(size < (1 << 15));
|
|
PoolHeader header(size, isNatural);
|
|
*(PoolHeader*)start = header;
|
|
}
|
|
|
|
// The size of an arbitrary 32-bit call in the instruction stream. On ARM this
|
|
// sequence is |pc = ldr pc - 4; imm32| given that we never reach the imm32.
|
|
uint32_t
|
|
Assembler::PatchWrite_NearCallSize()
|
|
{
|
|
return sizeof(uint32_t);
|
|
}
|
|
|
|
void
|
|
Assembler::PatchWrite_NearCall(CodeLocationLabel start, CodeLocationLabel toCall)
|
|
{
|
|
Instruction* inst = (Instruction*) start.raw();
|
|
// Overwrite whatever instruction used to be here with a call. Since the
|
|
// destination is in the same function, it will be within range of the
|
|
// 24 << 2 byte bl instruction.
|
|
uint8_t* dest = toCall.raw();
|
|
new (inst) InstBLImm(BOffImm(dest - (uint8_t*)inst) , Always);
|
|
// Ensure everyone sees the code that was just written into memory.
|
|
AutoFlushICache::flush(uintptr_t(inst), 4);
|
|
}
|
|
|
|
void
|
|
Assembler::PatchDataWithValueCheck(CodeLocationLabel label, PatchedImmPtr newValue,
|
|
PatchedImmPtr expectedValue)
|
|
{
|
|
Instruction* ptr = reinterpret_cast<Instruction*>(label.raw());
|
|
InstructionIterator iter(ptr);
|
|
Register dest;
|
|
Assembler::RelocStyle rs;
|
|
|
|
DebugOnly<const uint32_t*> val = GetPtr32Target(&iter, &dest, &rs);
|
|
MOZ_ASSERT(uint32_t((const uint32_t*)val) == uint32_t(expectedValue.value));
|
|
|
|
MacroAssembler::ma_mov_patch(Imm32(int32_t(newValue.value)), dest, Always, rs, ptr);
|
|
|
|
// L_LDR won't cause any instructions to be updated.
|
|
if (rs != L_LDR) {
|
|
AutoFlushICache::flush(uintptr_t(ptr), 4);
|
|
AutoFlushICache::flush(uintptr_t(ptr->next()), 4);
|
|
}
|
|
}
|
|
|
|
void
|
|
Assembler::PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue, ImmPtr expectedValue)
|
|
{
|
|
PatchDataWithValueCheck(label,
|
|
PatchedImmPtr(newValue.value),
|
|
PatchedImmPtr(expectedValue.value));
|
|
}
|
|
|
|
// This just stomps over memory with 32 bits of raw data. Its purpose is to
|
|
// overwrite the call of JITed code with 32 bits worth of an offset. This will
|
|
// is only meant to function on code that has been invalidated, so it should be
|
|
// totally safe. Since that instruction will never be executed again, a ICache
|
|
// flush should not be necessary
|
|
void
|
|
Assembler::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
|
|
// Raw is going to be the return address.
|
|
uint32_t* raw = (uint32_t*)label.raw();
|
|
// Overwrite the 4 bytes before the return address, which will end up being
|
|
// the call instruction.
|
|
*(raw - 1) = imm.value;
|
|
}
|
|
|
|
uint8_t*
|
|
Assembler::NextInstruction(uint8_t* inst_, uint32_t* count)
|
|
{
|
|
Instruction* inst = reinterpret_cast<Instruction*>(inst_);
|
|
if (count != nullptr)
|
|
*count += sizeof(Instruction);
|
|
return reinterpret_cast<uint8_t*>(inst->next());
|
|
}
|
|
|
|
static bool
|
|
InstIsGuard(Instruction* inst, const PoolHeader** ph)
|
|
{
|
|
Assembler::Condition c = inst->extractCond();
|
|
if (c != Assembler::Always)
|
|
return false;
|
|
if (!(inst->is<InstBXReg>() || inst->is<InstBImm>()))
|
|
return false;
|
|
// See if the next instruction is a pool header.
|
|
*ph = (inst + 1)->as<const PoolHeader>();
|
|
return *ph != nullptr;
|
|
}
|
|
|
|
static bool
|
|
InstIsBNop(Instruction* inst)
|
|
{
|
|
// In some special situations, it is necessary to insert a NOP into the
|
|
// instruction stream that nobody knows about, since nobody should know
|
|
// about it, make sure it gets skipped when Instruction::next() is called.
|
|
// this generates a very specific nop, namely a branch to the next
|
|
// instruction.
|
|
Assembler::Condition c = inst->extractCond();
|
|
if (c != Assembler::Always)
|
|
return false;
|
|
if (!inst->is<InstBImm>())
|
|
return false;
|
|
InstBImm* b = inst->as<InstBImm>();
|
|
BOffImm offset;
|
|
b->extractImm(&offset);
|
|
return offset.decode() == 4;
|
|
}
|
|
|
|
static bool
|
|
InstIsArtificialGuard(Instruction* inst, const PoolHeader** ph)
|
|
{
|
|
if (!InstIsGuard(inst, ph))
|
|
return false;
|
|
return !(*ph)->isNatural();
|
|
}
|
|
|
|
// If the instruction points to a artificial pool guard then skip the pool.
|
|
Instruction*
|
|
Instruction::skipPool()
|
|
{
|
|
const PoolHeader* ph;
|
|
// If this is a guard, and the next instruction is a header, always work
|
|
// around the pool. If it isn't a guard, then start looking ahead.
|
|
if (InstIsGuard(this, &ph)) {
|
|
// Don't skip a natural guard.
|
|
if (ph->isNatural())
|
|
return this;
|
|
return (this + 1 + ph->size())->skipPool();
|
|
}
|
|
if (InstIsBNop(this))
|
|
return (this + 1)->skipPool();
|
|
return this;
|
|
}
|
|
|
|
// Cases to be handled:
|
|
// 1) no pools or branches in sight => return this+1
|
|
// 2) branch to next instruction => return this+2, because a nop needed to be inserted into the stream.
|
|
// 3) this+1 is an artificial guard for a pool => return first instruction after the pool
|
|
// 4) this+1 is a natural guard => return the branch
|
|
// 5) this is a branch, right before a pool => return first instruction after the pool
|
|
// in assembly form:
|
|
// 1) add r0, r0, r0 <= this
|
|
// add r1, r1, r1 <= returned value
|
|
// add r2, r2, r2
|
|
//
|
|
// 2) add r0, r0, r0 <= this
|
|
// b foo
|
|
// foo:
|
|
// add r2, r2, r2 <= returned value
|
|
//
|
|
// 3) add r0, r0, r0 <= this
|
|
// b after_pool;
|
|
// .word 0xffff0002 # bit 15 being 0 indicates that the branch was not requested by the assembler
|
|
// 0xdeadbeef # the 2 indicates that there is 1 pool entry, and the pool header
|
|
// add r4, r4, r4 <= returned value
|
|
// 4) add r0, r0, r0 <= this
|
|
// b after_pool <= returned value
|
|
// .word 0xffff8002 # bit 15 being 1 indicates that the branch was requested by the assembler
|
|
// 0xdeadbeef
|
|
// add r4, r4, r4
|
|
// 5) b after_pool <= this
|
|
// .word 0xffff8002 # bit 15 has no bearing on the returned value
|
|
// 0xdeadbeef
|
|
// add r4, r4, r4 <= returned value
|
|
|
|
Instruction*
|
|
Instruction::next()
|
|
{
|
|
Instruction* ret = this+1;
|
|
const PoolHeader* ph;
|
|
// If this is a guard, and the next instruction is a header, always work
|
|
// around the pool. If it isn't a guard, then start looking ahead.
|
|
if (InstIsGuard(this, &ph))
|
|
return (ret + ph->size())->skipPool();
|
|
if (InstIsArtificialGuard(ret, &ph))
|
|
return (ret + 1 + ph->size())->skipPool();
|
|
return ret->skipPool();
|
|
}
|
|
|
|
void
|
|
Assembler::ToggleToJmp(CodeLocationLabel inst_)
|
|
{
|
|
uint32_t* ptr = (uint32_t*)inst_.raw();
|
|
|
|
DebugOnly<Instruction*> inst = (Instruction*)inst_.raw();
|
|
MOZ_ASSERT(inst->is<InstCMP>());
|
|
|
|
// Zero bits 20-27, then set 24-27 to be correct for a branch.
|
|
// 20-23 will be party of the B's immediate, and should be 0.
|
|
*ptr = (*ptr & ~(0xff << 20)) | (0xa0 << 20);
|
|
AutoFlushICache::flush(uintptr_t(ptr), 4);
|
|
}
|
|
|
|
void
|
|
Assembler::ToggleToCmp(CodeLocationLabel inst_)
|
|
{
|
|
uint32_t* ptr = (uint32_t*)inst_.raw();
|
|
|
|
DebugOnly<Instruction*> inst = (Instruction*)inst_.raw();
|
|
MOZ_ASSERT(inst->is<InstBImm>());
|
|
|
|
// Ensure that this masking operation doesn't affect the offset of the
|
|
// branch instruction when it gets toggled back.
|
|
MOZ_ASSERT((*ptr & (0xf << 20)) == 0);
|
|
|
|
// Also make sure that the CMP is valid. Part of having a valid CMP is that
|
|
// all of the bits describing the destination in most ALU instructions are
|
|
// all unset (looks like it is encoding r0).
|
|
MOZ_ASSERT(toRD(*inst) == r0);
|
|
|
|
// Zero out bits 20-27, then set them to be correct for a compare.
|
|
*ptr = (*ptr & ~(0xff << 20)) | (0x35 << 20);
|
|
|
|
AutoFlushICache::flush(uintptr_t(ptr), 4);
|
|
}
|
|
|
|
void
|
|
Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled)
|
|
{
|
|
Instruction* inst = (Instruction*)inst_.raw();
|
|
// Skip a pool with an artificial guard.
|
|
inst = inst->skipPool();
|
|
MOZ_ASSERT(inst->is<InstMovW>() || inst->is<InstLDR>());
|
|
|
|
if (inst->is<InstMovW>()) {
|
|
// If it looks like the start of a movw/movt sequence, then make sure we
|
|
// have all of it (and advance the iterator past the full sequence).
|
|
inst = inst->next();
|
|
MOZ_ASSERT(inst->is<InstMovT>());
|
|
}
|
|
|
|
inst = inst->next();
|
|
MOZ_ASSERT(inst->is<InstNOP>() || inst->is<InstBLXReg>());
|
|
|
|
if (enabled == inst->is<InstBLXReg>()) {
|
|
// Nothing to do.
|
|
return;
|
|
}
|
|
|
|
if (enabled)
|
|
*inst = InstBLXReg(ScratchRegister, Always);
|
|
else
|
|
*inst = InstNOP();
|
|
|
|
AutoFlushICache::flush(uintptr_t(inst), 4);
|
|
}
|
|
|
|
size_t
|
|
Assembler::ToggledCallSize(uint8_t* code)
|
|
{
|
|
Instruction* inst = (Instruction*)code;
|
|
// Skip a pool with an artificial guard.
|
|
inst = inst->skipPool();
|
|
MOZ_ASSERT(inst->is<InstMovW>() || inst->is<InstLDR>());
|
|
|
|
if (inst->is<InstMovW>()) {
|
|
// If it looks like the start of a movw/movt sequence, then make sure we
|
|
// have all of it (and advance the iterator past the full sequence).
|
|
inst = inst->next();
|
|
MOZ_ASSERT(inst->is<InstMovT>());
|
|
}
|
|
|
|
inst = inst->next();
|
|
MOZ_ASSERT(inst->is<InstNOP>() || inst->is<InstBLXReg>());
|
|
return uintptr_t(inst) + 4 - uintptr_t(code);
|
|
}
|
|
|
|
uint8_t*
|
|
Assembler::BailoutTableStart(uint8_t* code)
|
|
{
|
|
Instruction* inst = (Instruction*)code;
|
|
// Skip a pool with an artificial guard or NOP fill.
|
|
inst = inst->skipPool();
|
|
MOZ_ASSERT(inst->is<InstBLImm>());
|
|
return (uint8_t*) inst;
|
|
}
|
|
|
|
InstructionIterator::InstructionIterator(Instruction* i_)
|
|
: i(i_)
|
|
{
|
|
// Work around pools with an artificial pool guard and around nop-fill.
|
|
i = i->skipPool();
|
|
}
|
|
|
|
uint32_t Assembler::NopFill = 0;
|
|
|
|
uint32_t
|
|
Assembler::GetNopFill()
|
|
{
|
|
static bool isSet = false;
|
|
if (!isSet) {
|
|
char* fillStr = getenv("ARM_ASM_NOP_FILL");
|
|
uint32_t fill;
|
|
if (fillStr && sscanf(fillStr, "%u", &fill) == 1)
|
|
NopFill = fill;
|
|
if (NopFill > 8)
|
|
MOZ_CRASH("Nop fill > 8 is not supported");
|
|
isSet = true;
|
|
}
|
|
return NopFill;
|
|
}
|
|
|
|
uint32_t Assembler::AsmPoolMaxOffset = 1024;
|
|
|
|
uint32_t
|
|
Assembler::GetPoolMaxOffset()
|
|
{
|
|
static bool isSet = false;
|
|
if (!isSet) {
|
|
char* poolMaxOffsetStr = getenv("ASM_POOL_MAX_OFFSET");
|
|
uint32_t poolMaxOffset;
|
|
if (poolMaxOffsetStr && sscanf(poolMaxOffsetStr, "%u", &poolMaxOffset) == 1)
|
|
AsmPoolMaxOffset = poolMaxOffset;
|
|
isSet = true;
|
|
}
|
|
return AsmPoolMaxOffset;
|
|
}
|
|
|
|
SecondScratchRegisterScope::SecondScratchRegisterScope(MacroAssembler &masm)
|
|
: AutoRegisterScope(masm, masm.getSecondScratchReg())
|
|
{
|
|
}
|