/* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_ #include "base/bit_field.h" #include "base/macros.h" #include "class_root.h" #include "code_generator.h" #include "common_arm64.h" #include "dex/dex_file_types.h" #include "dex/string_reference.h" #include "dex/type_reference.h" #include "driver/compiler_options.h" #include "jit_patches_arm64.h" #include "nodes.h" #include "parallel_move_resolver.h" #include "utils/arm64/assembler_arm64.h" // TODO(VIXL): Make VIXL compile cleanly with -Wshadow, -Wdeprecated-declarations. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include "aarch64/disasm-aarch64.h" #include "aarch64/macro-assembler-aarch64.h" #pragma GCC diagnostic pop namespace art HIDDEN { namespace linker { class Arm64RelativePatcherTest; } // namespace linker namespace arm64 { class CodeGeneratorARM64; // Use a local definition to prevent copying mistakes. static constexpr size_t kArm64WordSize = static_cast(kArm64PointerSize); // This constant is used as an approximate margin when emission of veneer and literal pools // must be blocked. static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize; // Reference load (except object array loads) is using LDR Wt, [Xn, #offset] which can handle // offset < 16KiB. For offsets >= 16KiB, the load shall be emitted as two or more instructions. // For the Baker read barrier implementation using link-time generated thunks we need to split // the offset explicitly. static constexpr uint32_t kReferenceLoadMinFarOffset = 16 * KB; static const vixl::aarch64::Register kParameterCoreRegisters[] = { vixl::aarch64::x1, vixl::aarch64::x2, vixl::aarch64::x3, vixl::aarch64::x4, vixl::aarch64::x5, vixl::aarch64::x6, vixl::aarch64::x7 }; static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters); static const vixl::aarch64::VRegister kParameterFPRegisters[] = { vixl::aarch64::d0, vixl::aarch64::d1, vixl::aarch64::d2, vixl::aarch64::d3, vixl::aarch64::d4, vixl::aarch64::d5, vixl::aarch64::d6, vixl::aarch64::d7 }; static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters); // Thread Register. const vixl::aarch64::Register tr = vixl::aarch64::x19; // Marking Register. const vixl::aarch64::Register mr = vixl::aarch64::x20; // Implicit suspend check register. const vixl::aarch64::Register kImplicitSuspendCheckRegister = vixl::aarch64::x21; // Method register on invoke. static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0; const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0, vixl::aarch64::ip1); const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31); const vixl::aarch64::CPURegList runtime_reserved_core_registers = vixl::aarch64::CPURegList( tr, // Reserve X20 as Marking Register when emitting Baker read barriers. // TODO: We don't need to reserve marking-register for userfaultfd GC. But // that would require some work in the assembler code as the right GC is // chosen at load-time and not compile time. (kReserveMarkingRegister ? mr : vixl::aarch64::NoCPUReg), kImplicitSuspendCheckRegister, vixl::aarch64::lr); // Some instructions have special requirements for a temporary, for example // LoadClass/kBssEntry and LoadString/kBssEntry for Baker read barrier require // temp that's not an R0 (to avoid an extra move) and Baker read barrier field // loads with large offsets need a fixed register to limit the number of link-time // thunks we generate. For these and similar cases, we want to reserve a specific // register that's neither callee-save nor an argument register. We choose x15. inline Location FixedTempLocation() { return Location::RegisterLocation(vixl::aarch64::x15.GetCode()); } // Callee-save registers AAPCS64, without x19 (Thread Register) (nor // x20 (Marking Register) when emitting Baker read barriers). const vixl::aarch64::CPURegList callee_saved_core_registers( vixl::aarch64::CPURegister::kRegister, vixl::aarch64::kXRegSize, (kReserveMarkingRegister ? vixl::aarch64::x21.GetCode() : vixl::aarch64::x20.GetCode()), vixl::aarch64::x30.GetCode()); const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kVRegister, vixl::aarch64::kDRegSize, vixl::aarch64::d8.GetCode(), vixl::aarch64::d15.GetCode()); Location ARM64ReturnLocation(DataType::Type return_type); vixl::aarch64::Condition ARM64PCondition(HVecPredToBoolean::PCondKind cond); #define UNIMPLEMENTED_INTRINSIC_LIST_ARM64(V) \ V(MathSignumFloat) \ V(MathSignumDouble) \ V(MathCopySignFloat) \ V(MathCopySignDouble) \ V(IntegerRemainderUnsigned) \ V(LongRemainderUnsigned) \ V(StringStringIndexOf) \ V(StringStringIndexOfAfter) \ V(StringBufferAppend) \ V(StringBufferLength) \ V(StringBufferToString) \ V(StringBuilderAppendObject) \ V(StringBuilderAppendString) \ V(StringBuilderAppendCharSequence) \ V(StringBuilderAppendCharArray) \ V(StringBuilderAppendBoolean) \ V(StringBuilderAppendChar) \ V(StringBuilderAppendInt) \ V(StringBuilderAppendLong) \ V(StringBuilderAppendFloat) \ V(StringBuilderAppendDouble) \ V(StringBuilderLength) \ V(StringBuilderToString) \ V(SystemArrayCopyByte) \ V(SystemArrayCopyInt) \ V(UnsafeArrayBaseOffset) \ /* 1.8 */ \ V(MethodHandleInvokeExact) \ V(MethodHandleInvoke) \ /* OpenJDK 11 */ \ V(JdkUnsafeArrayBaseOffset) class SlowPathCodeARM64 : public SlowPathCode { public: explicit SlowPathCodeARM64(HInstruction* instruction) : SlowPathCode(instruction), entry_label_(), exit_label_() {} vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; } vixl::aarch64::Label* GetExitLabel() { return &exit_label_; } void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override; void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override; private: vixl::aarch64::Label entry_label_; vixl::aarch64::Label exit_label_; DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64); }; class JumpTableARM64 : public DeletableArenaObject { public: using VIXLInt32Literal = vixl::aarch64::Literal; explicit JumpTableARM64(HPackedSwitch* switch_instr) : switch_instr_(switch_instr), table_start_(), jump_targets_(switch_instr->GetAllocator()->Adapter(kArenaAllocCodeGenerator)) { uint32_t num_entries = switch_instr_->GetNumEntries(); for (uint32_t i = 0; i < num_entries; i++) { VIXLInt32Literal* lit = new VIXLInt32Literal(0); jump_targets_.emplace_back(lit); } } vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; } // Emits the jump table into the code buffer; jump target offsets are not yet known. void EmitTable(CodeGeneratorARM64* codegen); // Updates the offsets in the jump table, to be used when the jump targets basic blocks // addresses are resolved. void FixTable(CodeGeneratorARM64* codegen); private: HPackedSwitch* const switch_instr_; vixl::aarch64::Label table_start_; // Contains literals for the switch's jump targets. ArenaVector> jump_targets_; DISALLOW_COPY_AND_ASSIGN(JumpTableARM64); }; static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] = { vixl::aarch64::x0, vixl::aarch64::x1, vixl::aarch64::x2, vixl::aarch64::x3, vixl::aarch64::x4, vixl::aarch64::x5, vixl::aarch64::x6, vixl::aarch64::x7 }; static constexpr size_t kRuntimeParameterCoreRegistersLength = arraysize(kRuntimeParameterCoreRegisters); static const vixl::aarch64::VRegister kRuntimeParameterFpuRegisters[] = { vixl::aarch64::d0, vixl::aarch64::d1, vixl::aarch64::d2, vixl::aarch64::d3, vixl::aarch64::d4, vixl::aarch64::d5, vixl::aarch64::d6, vixl::aarch64::d7 }; static constexpr size_t kRuntimeParameterFpuRegistersLength = arraysize(kRuntimeParameterCoreRegisters); class InvokeRuntimeCallingConvention : public CallingConvention { public: InvokeRuntimeCallingConvention() : CallingConvention(kRuntimeParameterCoreRegisters, kRuntimeParameterCoreRegistersLength, kRuntimeParameterFpuRegisters, kRuntimeParameterFpuRegistersLength, kArm64PointerSize) {} Location GetReturnLocation(DataType::Type return_type); private: DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention); }; class InvokeDexCallingConvention : public CallingConvention { public: InvokeDexCallingConvention() : CallingConvention(kParameterCoreRegisters, kParameterCoreRegistersLength, kParameterFPRegisters, kParameterFPRegistersLength, kArm64PointerSize) {} Location GetReturnLocation(DataType::Type return_type) const { return ARM64ReturnLocation(return_type); } private: DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention); }; class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor { public: InvokeDexCallingConventionVisitorARM64() {} virtual ~InvokeDexCallingConventionVisitorARM64() {} Location GetNextLocation(DataType::Type type) override; Location GetReturnLocation(DataType::Type return_type) const override { return calling_convention.GetReturnLocation(return_type); } Location GetMethodLocation() const override; private: InvokeDexCallingConvention calling_convention; DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64); }; class CriticalNativeCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor { public: explicit CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation) : for_register_allocation_(for_register_allocation) {} virtual ~CriticalNativeCallingConventionVisitorARM64() {} Location GetNextLocation(DataType::Type type) override; Location GetReturnLocation(DataType::Type type) const override; Location GetMethodLocation() const override; size_t GetStackOffset() const { return stack_offset_; } private: // Register allocator does not support adjusting frame size, so we cannot provide final locations // of stack arguments for register allocation. We ask the register allocator for any location and // move these arguments to the right place after adjusting the SP when generating the call. const bool for_register_allocation_; size_t gpr_index_ = 0u; size_t fpr_index_ = 0u; size_t stack_offset_ = 0u; DISALLOW_COPY_AND_ASSIGN(CriticalNativeCallingConventionVisitorARM64); }; class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention { public: FieldAccessCallingConventionARM64() {} Location GetObjectLocation() const override { return helpers::LocationFrom(vixl::aarch64::x1); } Location GetFieldIndexLocation() const override { return helpers::LocationFrom(vixl::aarch64::x0); } Location GetReturnLocation([[maybe_unused]] DataType::Type type) const override { return helpers::LocationFrom(vixl::aarch64::x0); } Location GetSetValueLocation([[maybe_unused]] DataType::Type type, bool is_instance) const override { return is_instance ? helpers::LocationFrom(vixl::aarch64::x2) : helpers::LocationFrom(vixl::aarch64::x1); } Location GetFpuLocation([[maybe_unused]] DataType::Type type) const override { return helpers::LocationFrom(vixl::aarch64::d0); } private: DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64); }; class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator { public: InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen); #define DECLARE_VISIT_INSTRUCTION(name, super) \ void Visit##name(H##name* instr) override; FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION) FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION) FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION void VisitInstruction(HInstruction* instruction) override { LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() << " (id " << instruction->GetId() << ")"; } Arm64Assembler* GetAssembler() const { return assembler_; } vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); } // SIMD helpers. virtual Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) = 0; virtual void FreeSIMDScratchLocation(Location loc, vixl::aarch64::UseScratchRegisterScope* scope) = 0; virtual void LoadSIMDRegFromStack(Location destination, Location source) = 0; virtual void MoveSIMDRegToSIMDReg(Location destination, Location source) = 0; virtual void MoveToSIMDStackSlot(Location destination, Location source) = 0; virtual void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) = 0; virtual void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) = 0; protected: void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path, vixl::aarch64::Register class_reg); void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check, vixl::aarch64::Register temp); void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor); void HandleBinaryOp(HBinaryOperation* instr); void HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info, bool value_can_be_null, WriteBarrierKind write_barrier_kind); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); void HandleCondition(HCondition* instruction); // Generate a heap reference load using one register `out`: // // out <- *(out + offset) // // while honoring heap poisoning and/or read barriers (if any). // // Location `maybe_temp` is used when generating a read barrier and // shall be a register in that case; it may be an invalid location // otherwise. void GenerateReferenceLoadOneRegister(HInstruction* instruction, Location out, uint32_t offset, Location maybe_temp, ReadBarrierOption read_barrier_option); // Generate a heap reference load using two different registers // `out` and `obj`: // // out <- *(obj + offset) // // while honoring heap poisoning and/or read barriers (if any). // // Location `maybe_temp` is used when generating a Baker's (fast // path) read barrier and shall be a register in that case; it may // be an invalid location otherwise. void GenerateReferenceLoadTwoRegisters(HInstruction* instruction, Location out, Location obj, uint32_t offset, Location maybe_temp, ReadBarrierOption read_barrier_option); // Generate a floating-point comparison. void GenerateFcmp(HInstruction* instruction); void HandleShift(HBinaryOperation* instr); void GenerateTestAndBranch(HInstruction* instruction, size_t condition_input_index, vixl::aarch64::Label* true_target, vixl::aarch64::Label* false_target); void DivRemOneOrMinusOne(HBinaryOperation* instruction); void DivRemByPowerOfTwo(HBinaryOperation* instruction); void GenerateIncrementNegativeByOne(vixl::aarch64::Register out, vixl::aarch64::Register in, bool use_cond_inc); void GenerateResultRemWithAnyConstant(vixl::aarch64::Register out, vixl::aarch64::Register dividend, vixl::aarch64::Register quotient, int64_t divisor, // This function may acquire a scratch register. vixl::aarch64::UseScratchRegisterScope* temps_scope); void GenerateInt64UnsignedDivRemWithAnyPositiveConstant(HBinaryOperation* instruction); void GenerateInt64DivRemWithAnyConstant(HBinaryOperation* instruction); void GenerateInt32DivRemWithAnyConstant(HBinaryOperation* instruction); void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction, int64_t divisor); void GenerateIntDiv(HDiv* instruction); void GenerateIntDivForConstDenom(HDiv *instruction); void GenerateIntDivForPower2Denom(HDiv *instruction); void GenerateIntRem(HRem* instruction); void GenerateIntRemForConstDenom(HRem *instruction); void GenerateIntRemForPower2Denom(HRem *instruction); void HandleGoto(HInstruction* got, HBasicBlock* successor); void GenerateMethodEntryExitHook(HInstruction* instruction); // Helpers to set up locations for vector memory operations. Returns the memory operand and, // if used, sets the output parameter scratch to a temporary register used in this operand, // so that the client can release it right after the memory operand use. // Neon version. vixl::aarch64::MemOperand VecNEONAddress( HVecMemoryOperation* instruction, // This function may acquire a scratch register. vixl::aarch64::UseScratchRegisterScope* temps_scope, size_t size, bool is_string_char_at, /*out*/ vixl::aarch64::Register* scratch); // SVE version. vixl::aarch64::SVEMemOperand VecSVEAddress( HVecMemoryOperation* instruction, // This function may acquire a scratch register. vixl::aarch64::UseScratchRegisterScope* temps_scope, size_t size, bool is_string_char_at, /*out*/ vixl::aarch64::Register* scratch); Arm64Assembler* const assembler_; CodeGeneratorARM64* const codegen_; DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64); }; class LocationsBuilderARM64 : public HGraphVisitor { public: LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen) : HGraphVisitor(graph), codegen_(codegen) {} #define DECLARE_VISIT_INSTRUCTION(name, super) \ void Visit##name(H##name* instr) override; FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION) FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION) FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION void VisitInstruction(HInstruction* instruction) override { LOG(FATAL) << "Unreachable instruction " << instruction->DebugName() << " (id " << instruction->GetId() << ")"; } protected: void HandleBinaryOp(HBinaryOperation* instr); void HandleFieldSet(HInstruction* instruction); void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info); void HandleInvoke(HInvoke* instr); void HandleCondition(HCondition* instruction); void HandleShift(HBinaryOperation* instr); CodeGeneratorARM64* const codegen_; InvokeDexCallingConventionVisitorARM64 parameter_visitor_; DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64); }; class InstructionCodeGeneratorARM64Neon : public InstructionCodeGeneratorARM64 { public: InstructionCodeGeneratorARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) : InstructionCodeGeneratorARM64(graph, codegen) {} #define DECLARE_VISIT_INSTRUCTION(name, super) \ void Visit##name(H##name* instr) override; FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override; void FreeSIMDScratchLocation(Location loc, vixl::aarch64::UseScratchRegisterScope* scope) override; void LoadSIMDRegFromStack(Location destination, Location source) override; void MoveSIMDRegToSIMDReg(Location destination, Location source) override; void MoveToSIMDStackSlot(Location destination, Location source) override; void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override; void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override; }; class LocationsBuilderARM64Neon : public LocationsBuilderARM64 { public: LocationsBuilderARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) : LocationsBuilderARM64(graph, codegen) {} #define DECLARE_VISIT_INSTRUCTION(name, super) \ void Visit##name(H##name* instr) override; FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION }; class InstructionCodeGeneratorARM64Sve : public InstructionCodeGeneratorARM64 { public: InstructionCodeGeneratorARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) : InstructionCodeGeneratorARM64(graph, codegen) {} #define DECLARE_VISIT_INSTRUCTION(name, super) \ void Visit##name(H##name* instr) override; FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override; void FreeSIMDScratchLocation(Location loc, vixl::aarch64::UseScratchRegisterScope* scope) override; void LoadSIMDRegFromStack(Location destination, Location source) override; void MoveSIMDRegToSIMDReg(Location destination, Location source) override; void MoveToSIMDStackSlot(Location destination, Location source) override; void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override; void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override; private: // Validate that instruction vector length and packed type are compliant with the SIMD // register size (full SIMD register is used). void ValidateVectorLength(HVecOperation* instr) const; vixl::aarch64::PRegister GetVecGoverningPReg(HVecOperation* instr) { return GetVecPredSetFixedOutPReg(instr->GetGoverningPredicate()); } // Returns a fixed p-reg for predicate setting instruction. // // Currently we only support diamond CF loops for predicated vectorization; also we don't have // register allocator support for vector predicates. Thus we use fixed P-regs for loop main, // True and False predicates as a temporary solution. // // TODO: Support SIMD types and registers in ART. static vixl::aarch64::PRegister GetVecPredSetFixedOutPReg(HVecPredSetOperation* instr) { if (instr->IsVecPredWhile() || instr->IsVecPredSetAll()) { // VecPredWhile and VecPredSetAll live ranges never overlap due to the current vectorization // scheme: the former only is live inside a vectorized loop and the later is never in a // loop and never spans across loops. return vixl::aarch64::p0; } else if (instr->IsVecPredNot()) { // This relies on the fact that we only use PredNot manually in the autovectorizer, // so there is only one of them in each loop. return vixl::aarch64::p1; } else { DCHECK(instr->IsVecCondition()); return vixl::aarch64::p2; } } // Generate a vector comparison instruction based on the IfCondition. void GenerateIntegerVecComparison(const vixl::aarch64::PRegisterWithLaneSize& pd, const vixl::aarch64::PRegisterZ& pg, const vixl::aarch64::ZRegister& zn, const vixl::aarch64::ZRegister& zm, IfCondition cond); void HandleVecCondition(HVecCondition* instruction); }; class LocationsBuilderARM64Sve : public LocationsBuilderARM64 { public: LocationsBuilderARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) : LocationsBuilderARM64(graph, codegen) {} #define DECLARE_VISIT_INSTRUCTION(name, super) \ void Visit##name(H##name* instr) override; FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION) #undef DECLARE_VISIT_INSTRUCTION private: void HandleVecCondition(HVecCondition* instruction); }; class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap { public: ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen) : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {} protected: void PrepareForEmitNativeCode() override; void FinishEmitNativeCode() override; Location AllocateScratchLocationFor(Location::Kind kind) override; void FreeScratchLocation(Location loc) override; void EmitMove(size_t index) override; private: Arm64Assembler* GetAssembler() const; vixl::aarch64::MacroAssembler* GetVIXLAssembler() const { return GetAssembler()->GetVIXLAssembler(); } CodeGeneratorARM64* const codegen_; vixl::aarch64::UseScratchRegisterScope vixl_temps_; DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64); }; class CodeGeneratorARM64 : public CodeGenerator { public: CodeGeneratorARM64(HGraph* graph, const CompilerOptions& compiler_options, OptimizingCompilerStats* stats = nullptr); virtual ~CodeGeneratorARM64() {} void GenerateFrameEntry() override; void GenerateFrameExit() override; vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const; vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const; void Bind(HBasicBlock* block) override; vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) { block = FirstNonEmptyBlock(block); return &(block_labels_[block->GetBlockId()]); } size_t GetWordSize() const override { return kArm64WordSize; } bool SupportsPredicatedSIMD() const override { return ShouldUseSVE(); } size_t GetSlowPathFPWidth() const override { return GetGraph()->HasSIMD() ? GetSIMDRegisterWidth() : vixl::aarch64::kDRegSizeInBytes; } size_t GetCalleePreservedFPWidth() const override { return vixl::aarch64::kDRegSizeInBytes; } size_t GetSIMDRegisterWidth() const override; uintptr_t GetAddressOf(HBasicBlock* block) override { vixl::aarch64::Label* block_entry_label = GetLabelOf(block); DCHECK(block_entry_label->IsBound()); return block_entry_label->GetLocation(); } HGraphVisitor* GetLocationBuilder() override { return location_builder_; } InstructionCodeGeneratorARM64* GetInstructionCodeGeneratorArm64() { return instruction_visitor_; } HGraphVisitor* GetInstructionVisitor() override { return GetInstructionCodeGeneratorArm64(); } Arm64Assembler* GetAssembler() override { return &assembler_; } const Arm64Assembler& GetAssembler() const override { return assembler_; } vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); } // Emit a write barrier if: // A) emit_null_check is false // B) emit_null_check is true, and value is not null. void MaybeMarkGCCard(vixl::aarch64::Register object, vixl::aarch64::Register value, bool emit_null_check); // Emit a write barrier unconditionally. void MarkGCCard(vixl::aarch64::Register object); // Crash if the card table is not valid. This check is only emitted for the CC GC. We assert // `(!clean || !self->is_gc_marking)`, since the card table should not be set to clean when the CC // GC is marking for eliminated write barriers. void CheckGCCardIsValid(vixl::aarch64::Register object); void GenerateMemoryBarrier(MemBarrierKind kind); // Register allocation. void SetupBlockedRegisters() const override; size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override; size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override; size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override; size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override; // The number of registers that can be allocated. The register allocator may // decide to reserve and not use a few of them. // We do not consider registers sp, xzr, wzr. They are either not allocatable // (xzr, wzr), or make for poor allocatable registers (sp alignment // requirements, etc.). This also facilitates our task as all other registers // can easily be mapped via to or from their type and index or code. static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1; static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfVRegisters; static constexpr int kNumberOfAllocatableRegisterPairs = 0; void DumpCoreRegister(std::ostream& stream, int reg) const override; void DumpFloatingPointRegister(std::ostream& stream, int reg) const override; InstructionSet GetInstructionSet() const override { return InstructionSet::kArm64; } const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const; void Initialize() override { block_labels_.resize(GetGraph()->GetBlocks().size()); } // We want to use the STP and LDP instructions to spill and restore registers for slow paths. // These instructions can only encode offsets that are multiples of the register size accessed. uint32_t GetPreferredSlotsAlignment() const override { return vixl::aarch64::kXRegSizeInBytes; } JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) { jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARM64(switch_instr)); return jump_tables_.back().get(); } void Finalize() override; // Code generation helpers. void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant); void MoveConstant(Location destination, int32_t value) override; void MoveLocation(Location dst, Location src, DataType::Type dst_type) override; void AddLocationAsTemp(Location location, LocationSummary* locations) override; void Load(DataType::Type type, vixl::aarch64::CPURegister dst, const vixl::aarch64::MemOperand& src); void Store(DataType::Type type, vixl::aarch64::CPURegister src, const vixl::aarch64::MemOperand& dst); void LoadAcquire(HInstruction* instruction, DataType::Type type, vixl::aarch64::CPURegister dst, const vixl::aarch64::MemOperand& src, bool needs_null_check); void StoreRelease(HInstruction* instruction, DataType::Type type, vixl::aarch64::CPURegister src, const vixl::aarch64::MemOperand& dst, bool needs_null_check); // Generate code to invoke a runtime entry point. void InvokeRuntime(QuickEntrypointEnum entrypoint, HInstruction* instruction, uint32_t dex_pc, SlowPathCode* slow_path = nullptr) override; // Generate code to invoke a runtime entry point, but do not record // PC-related information in a stack map. void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, HInstruction* instruction, SlowPathCode* slow_path); ParallelMoveResolverARM64* GetMoveResolver() override { return &move_resolver_; } bool NeedsTwoRegisters([[maybe_unused]] DataType::Type type) const override { return false; } // Check if the desired_string_load_kind is supported. If it is, return it, // otherwise return a fall-back kind that should be used instead. HLoadString::LoadKind GetSupportedLoadStringKind( HLoadString::LoadKind desired_string_load_kind) override; // Check if the desired_class_load_kind is supported. If it is, return it, // otherwise return a fall-back kind that should be used instead. HLoadClass::LoadKind GetSupportedLoadClassKind( HLoadClass::LoadKind desired_class_load_kind) override; // Check if the desired_dispatch_info is supported. If it is, return it, // otherwise return a fall-back info that should be used instead. HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch( const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, ArtMethod* method) override; void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke); void GenerateStaticOrDirectCall( HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override; void GenerateVirtualCall( HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override; void MoveFromReturnRegister(Location trg, DataType::Type type) override; // Add a new boot image intrinsic patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBootImageIntrinsicPatch(uint32_t intrinsic_data, vixl::aarch64::Label* adrp_label = nullptr); // Add a new boot image relocation patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset, vixl::aarch64::Label* adrp_label = nullptr); // Add a new boot image method patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBootImageMethodPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); // Add a new app image method patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewAppImageMethodPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); // Add a new .bss entry method patch for an instruction and return // the label to be bound before the instruction. The instruction will be // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` // pointing to the associated ADRP patch label). vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); // Add a new boot image type patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBootImageTypePatch(const DexFile& dex_file, dex::TypeIndex type_index, vixl::aarch64::Label* adrp_label = nullptr); // Add a new app image type patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewAppImageTypePatch(const DexFile& dex_file, dex::TypeIndex type_index, vixl::aarch64::Label* adrp_label = nullptr); // Add a new .bss entry type patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBssEntryTypePatch(HLoadClass* load_class, vixl::aarch64::Label* adrp_label = nullptr); // Add a new boot image string patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBootImageStringPatch(const DexFile& dex_file, dex::StringIndex string_index, vixl::aarch64::Label* adrp_label = nullptr); // Add a new .bss entry string patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewStringBssEntryPatch(const DexFile& dex_file, dex::StringIndex string_index, vixl::aarch64::Label* adrp_label = nullptr); // Add a new boot image JNI entrypoint patch for an instruction and return the label // to be bound before the instruction. The instruction will be either the // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing // to the associated ADRP patch label). vixl::aarch64::Label* NewBootImageJniEntrypointPatch(MethodReference target_method, vixl::aarch64::Label* adrp_label = nullptr); // Emit the BL instruction for entrypoint thunk call and record the associated patch for AOT. void EmitEntrypointThunkCall(ThreadOffset64 entrypoint_offset); // Emit the CBNZ instruction for baker read barrier and record // the associated patch for AOT or slow path for JIT. void EmitBakerReadBarrierCbnz(uint32_t custom_data); vixl::aarch64::Literal* DeduplicateBootImageAddressLiteral(uint64_t address) { return jit_patches_.DeduplicateBootImageAddressLiteral(address); } vixl::aarch64::Literal* DeduplicateJitStringLiteral(const DexFile& dex_file, dex::StringIndex string_index, Handle handle) { return jit_patches_.DeduplicateJitStringLiteral( dex_file, string_index, handle, GetCodeGenerationData()); } vixl::aarch64::Literal* DeduplicateJitClassLiteral(const DexFile& dex_file, dex::TypeIndex class_index, Handle handle) { return jit_patches_.DeduplicateJitClassLiteral( dex_file, class_index, handle, GetCodeGenerationData()); } void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg); void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register out, vixl::aarch64::Register base); void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register out, vixl::aarch64::Register base); void LoadBootImageRelRoEntry(vixl::aarch64::Register reg, uint32_t boot_image_offset); void LoadBootImageAddress(vixl::aarch64::Register reg, uint32_t boot_image_reference); void LoadTypeForBootImageIntrinsic(vixl::aarch64::Register reg, TypeReference type_reference); void LoadIntrinsicDeclaringClass(vixl::aarch64::Register reg, HInvoke* invoke); void LoadClassRootForIntrinsic(vixl::aarch64::Register reg, ClassRoot class_root); void EmitLinkerPatches(ArenaVector* linker_patches) override; bool NeedsThunkCode(const linker::LinkerPatch& patch) const override; void EmitThunkCode(const linker::LinkerPatch& patch, /*out*/ ArenaVector* code, /*out*/ std::string* debug_name) override; void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override; // Generate a GC root reference load: // // root <- *(obj + offset) // // while honoring read barriers based on read_barrier_option. void GenerateGcRootFieldLoad(HInstruction* instruction, Location root, vixl::aarch64::Register obj, uint32_t offset, vixl::aarch64::Label* fixup_label, ReadBarrierOption read_barrier_option); // Generate MOV for the `old_value` in intrinsic and mark it with Baker read barrier. void GenerateIntrinsicMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value, vixl::aarch64::Register old_value); // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. // Overload suitable for Unsafe.getObject/-Volatile() intrinsic. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, Location ref, vixl::aarch64::Register obj, const vixl::aarch64::MemOperand& src, bool needs_null_check, bool use_load_acquire); // Fast path implementation of ReadBarrier::Barrier for a heap // reference field load when Baker's read barriers are used. void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, Location ref, vixl::aarch64::Register obj, uint32_t offset, Location maybe_temp, bool needs_null_check, bool use_load_acquire); // Fast path implementation of ReadBarrier::Barrier for a heap // reference array load when Baker's read barriers are used. void GenerateArrayLoadWithBakerReadBarrier(HArrayGet* instruction, Location ref, vixl::aarch64::Register obj, uint32_t data_offset, Location index, bool needs_null_check); // Emit code checking the status of the Marking Register, and // aborting the program if MR does not match the value stored in the // art::Thread object. Code is only emitted in debug mode and if // CompilerOptions::EmitRunTimeChecksInDebugMode returns true. // // Argument `code` is used to identify the different occurrences of // MaybeGenerateMarkingRegisterCheck in the code generator, and is // passed to the BRK instruction. // // If `temp_loc` is a valid location, it is expected to be a // register and will be used as a temporary to generate code; // otherwise, a temporary will be fetched from the core register // scratch pool. virtual void MaybeGenerateMarkingRegisterCheck(int code, Location temp_loc = Location::NoLocation()); // Create slow path for a read barrier for a heap reference within `instruction`. // // This is a helper function for GenerateReadBarrierSlow() that has the same // arguments. The creation and adding of the slow path is exposed for intrinsics // that cannot use GenerateReadBarrierSlow() from their own slow paths. SlowPathCodeARM64* AddReadBarrierSlowPath(HInstruction* instruction, Location out, Location ref, Location obj, uint32_t offset, Location index); // Generate a read barrier for a heap reference within `instruction` // using a slow path. // // A read barrier for an object reference read from the heap is // implemented as a call to the artReadBarrierSlow runtime entry // point, which is passed the values in locations `ref`, `obj`, and // `offset`: // // mirror::Object* artReadBarrierSlow(mirror::Object* ref, // mirror::Object* obj, // uint32_t offset); // // The `out` location contains the value returned by // artReadBarrierSlow. // // When `index` is provided (i.e. for array accesses), the offset // value passed to artReadBarrierSlow is adjusted to take `index` // into account. void GenerateReadBarrierSlow(HInstruction* instruction, Location out, Location ref, Location obj, uint32_t offset, Location index = Location::NoLocation()); // If read barriers are enabled, generate a read barrier for a heap // reference using a slow path. If heap poisoning is enabled, also // unpoison the reference in `out`. void MaybeGenerateReadBarrierSlow(HInstruction* instruction, Location out, Location ref, Location obj, uint32_t offset, Location index = Location::NoLocation()); // Generate a read barrier for a GC root within `instruction` using // a slow path. // // A read barrier for an object reference GC root is implemented as // a call to the artReadBarrierForRootSlow runtime entry point, // which is passed the value in location `root`: // // mirror::Object* artReadBarrierForRootSlow(GcRoot* root); // // The `out` location contains the value returned by // artReadBarrierForRootSlow. void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root); void IncreaseFrame(size_t adjustment) override; void DecreaseFrame(size_t adjustment) override; void GenerateNop() override; void GenerateImplicitNullCheck(HNullCheck* instruction) override; void GenerateExplicitNullCheck(HNullCheck* instruction) override; void MaybeRecordImplicitNullCheck(HInstruction* instr) final { // The function must be only called within special scopes // (EmissionCheckScope, ExactAssemblyScope) which prevent generation of // veneer/literal pools by VIXL assembler. CHECK_EQ(GetVIXLAssembler()->ArePoolsBlocked(), true) << "The function must only be called within EmissionCheckScope or ExactAssemblyScope"; CodeGenerator::MaybeRecordImplicitNullCheck(instr); } void MaybeGenerateInlineCacheCheck(HInstruction* instruction, vixl::aarch64::Register klass); void MaybeIncrementHotness(HSuspendCheck* suspend_check, bool is_frame_entry); void MaybeRecordTraceEvent(bool is_method_entry); bool CanUseImplicitSuspendCheck() const; private: // Encoding of thunk type and data for link-time generated thunks for Baker read barriers. enum class BakerReadBarrierKind : uint8_t { kField, // Field get or array get with constant offset (i.e. constant index). kAcquire, // Volatile field get. kArray, // Array get with index in register. kGcRoot, // GC root load. kLast = kGcRoot }; static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u; static constexpr size_t kBitsForBakerReadBarrierKind = MinimumBitsToStore(static_cast(BakerReadBarrierKind::kLast)); static constexpr size_t kBakerReadBarrierBitsForRegister = MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg); using BakerReadBarrierKindField = BitField; using BakerReadBarrierFirstRegField = BitField; using BakerReadBarrierSecondRegField = BitField; static void CheckValidReg(uint32_t reg) { DCHECK(reg < vixl::aarch64::lr.GetCode() && reg != vixl::aarch64::ip0.GetCode() && reg != vixl::aarch64::ip1.GetCode()) << reg; } static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) { CheckValidReg(base_reg); CheckValidReg(holder_reg); return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) | BakerReadBarrierFirstRegField::Encode(base_reg) | BakerReadBarrierSecondRegField::Encode(holder_reg); } static inline uint32_t EncodeBakerReadBarrierAcquireData(uint32_t base_reg, uint32_t holder_reg) { CheckValidReg(base_reg); CheckValidReg(holder_reg); DCHECK_NE(base_reg, holder_reg); return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kAcquire) | BakerReadBarrierFirstRegField::Encode(base_reg) | BakerReadBarrierSecondRegField::Encode(holder_reg); } static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) { CheckValidReg(base_reg); return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) | BakerReadBarrierFirstRegField::Encode(base_reg) | BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg); } static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) { CheckValidReg(root_reg); return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) | BakerReadBarrierFirstRegField::Encode(root_reg) | BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg); } void CompileBakerReadBarrierThunk(Arm64Assembler& assembler, uint32_t encoded_data, /*out*/ std::string* debug_name); // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types, // whether through .data.img.rel.ro, .bss, or directly in the boot image. struct PcRelativePatchInfo : PatchInfo { PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx) : PatchInfo(dex_file, off_or_idx), pc_insn_label() { } vixl::aarch64::Label* pc_insn_label; }; struct BakerReadBarrierPatchInfo { explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { } vixl::aarch64::Label label; uint32_t custom_data; }; vixl::aarch64::Label* NewPcRelativePatch(const DexFile* dex_file, uint32_t offset_or_index, vixl::aarch64::Label* adrp_label, ArenaDeque* patches); void FixJumpTables(); template static void EmitPcRelativeLinkerPatches(const ArenaDeque& infos, ArenaVector* linker_patches); // Returns whether SVE features are supported and should be used. bool ShouldUseSVE() const; // Labels for each block that will be compiled. // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory. ArenaDeque block_labels_; // Indexed by block id. vixl::aarch64::Label frame_entry_label_; ArenaVector> jump_tables_; LocationsBuilderARM64Neon location_builder_neon_; InstructionCodeGeneratorARM64Neon instruction_visitor_neon_; LocationsBuilderARM64Sve location_builder_sve_; InstructionCodeGeneratorARM64Sve instruction_visitor_sve_; LocationsBuilderARM64* location_builder_; InstructionCodeGeneratorARM64* instruction_visitor_; ParallelMoveResolverARM64 move_resolver_; Arm64Assembler assembler_; // PC-relative method patch info for kBootImageLinkTimePcRelative. ArenaDeque boot_image_method_patches_; // PC-relative method patch info for kAppImageRelRo. ArenaDeque app_image_method_patches_; // PC-relative method patch info for kBssEntry. ArenaDeque method_bss_entry_patches_; // PC-relative type patch info for kBootImageLinkTimePcRelative. ArenaDeque boot_image_type_patches_; // PC-relative type patch info for kAppImageRelRo. ArenaDeque app_image_type_patches_; // PC-relative type patch info for kBssEntry. ArenaDeque type_bss_entry_patches_; // PC-relative public type patch info for kBssEntryPublic. ArenaDeque public_type_bss_entry_patches_; // PC-relative package type patch info for kBssEntryPackage. ArenaDeque package_type_bss_entry_patches_; // PC-relative String patch info for kBootImageLinkTimePcRelative. ArenaDeque boot_image_string_patches_; // PC-relative String patch info for kBssEntry. ArenaDeque string_bss_entry_patches_; // PC-relative method patch info for kBootImageLinkTimePcRelative+kCallCriticalNative. ArenaDeque boot_image_jni_entrypoint_patches_; // PC-relative patch info for IntrinsicObjects for the boot image, // and for method/type/string patches for kBootImageRelRo otherwise. ArenaDeque boot_image_other_patches_; // Patch info for calls to entrypoint dispatch thunks. Used for slow paths. ArenaDeque> call_entrypoint_patches_; // Baker read barrier patch info. ArenaDeque baker_read_barrier_patches_; JitPatchesARM64 jit_patches_; // Baker read barrier slow paths, mapping custom data (uint32_t) to label. // Wrap the label to work around vixl::aarch64::Label being non-copyable // and non-moveable and as such unusable in ArenaSafeMap<>. struct LabelWrapper { LabelWrapper(const LabelWrapper& src) : label() { DCHECK(!src.label.IsLinked() && !src.label.IsBound()); } LabelWrapper() = default; vixl::aarch64::Label label; }; ArenaSafeMap jit_baker_read_barrier_slow_paths_; friend class linker::Arm64RelativePatcherTest; DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64); }; inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const { return codegen_->GetAssembler(); } } // namespace arm64 } // namespace art #endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_