V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
code-generator-arm64.cc
1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/backend/code-generator.h"
6 
7 #include "src/arm64/assembler-arm64-inl.h"
8 #include "src/arm64/macro-assembler-arm64-inl.h"
9 #include "src/compiler/backend/code-generator-impl.h"
10 #include "src/compiler/backend/gap-resolver.h"
11 #include "src/compiler/node-matchers.h"
12 #include "src/compiler/osr.h"
13 #include "src/frame-constants.h"
14 #include "src/heap/heap-inl.h" // crbug.com/v8/8499
15 #include "src/optimized-compilation-info.h"
16 #include "src/wasm/wasm-code-manager.h"
17 #include "src/wasm/wasm-objects.h"
18 
19 namespace v8 {
20 namespace internal {
21 namespace compiler {
22 
23 #define __ tasm()->
24 
25 // Adds Arm64-specific methods to convert InstructionOperands.
26 class Arm64OperandConverter final : public InstructionOperandConverter {
27  public:
28  Arm64OperandConverter(CodeGenerator* gen, Instruction* instr)
29  : InstructionOperandConverter(gen, instr) {}
30 
31  DoubleRegister InputFloat32Register(size_t index) {
32  return InputDoubleRegister(index).S();
33  }
34 
35  DoubleRegister InputFloat64Register(size_t index) {
36  return InputDoubleRegister(index);
37  }
38 
39  DoubleRegister InputSimd128Register(size_t index) {
40  return InputDoubleRegister(index).Q();
41  }
42 
43  CPURegister InputFloat32OrZeroRegister(size_t index) {
44  if (instr_->InputAt(index)->IsImmediate()) {
45  DCHECK_EQ(0, bit_cast<int32_t>(InputFloat32(index)));
46  return wzr;
47  }
48  DCHECK(instr_->InputAt(index)->IsFPRegister());
49  return InputDoubleRegister(index).S();
50  }
51 
52  CPURegister InputFloat64OrZeroRegister(size_t index) {
53  if (instr_->InputAt(index)->IsImmediate()) {
54  DCHECK_EQ(0, bit_cast<int64_t>(InputDouble(index)));
55  return xzr;
56  }
57  DCHECK(instr_->InputAt(index)->IsDoubleRegister());
58  return InputDoubleRegister(index);
59  }
60 
61  size_t OutputCount() { return instr_->OutputCount(); }
62 
63  DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
64 
65  DoubleRegister OutputFloat64Register() { return OutputDoubleRegister(); }
66 
67  DoubleRegister OutputSimd128Register() { return OutputDoubleRegister().Q(); }
68 
69  Register InputRegister32(size_t index) {
70  return ToRegister(instr_->InputAt(index)).W();
71  }
72 
73  Register InputOrZeroRegister32(size_t index) {
74  DCHECK(instr_->InputAt(index)->IsRegister() ||
75  (instr_->InputAt(index)->IsImmediate() && (InputInt32(index) == 0)));
76  if (instr_->InputAt(index)->IsImmediate()) {
77  return wzr;
78  }
79  return InputRegister32(index);
80  }
81 
82  Register InputRegister64(size_t index) { return InputRegister(index); }
83 
84  Register InputOrZeroRegister64(size_t index) {
85  DCHECK(instr_->InputAt(index)->IsRegister() ||
86  (instr_->InputAt(index)->IsImmediate() && (InputInt64(index) == 0)));
87  if (instr_->InputAt(index)->IsImmediate()) {
88  return xzr;
89  }
90  return InputRegister64(index);
91  }
92 
93  Operand InputOperand(size_t index) {
94  return ToOperand(instr_->InputAt(index));
95  }
96 
97  Operand InputOperand64(size_t index) { return InputOperand(index); }
98 
99  Operand InputOperand32(size_t index) {
100  return ToOperand32(instr_->InputAt(index));
101  }
102 
103  Register OutputRegister64() { return OutputRegister(); }
104 
105  Register OutputRegister32() { return ToRegister(instr_->Output()).W(); }
106 
107  Register TempRegister32(size_t index) {
108  return ToRegister(instr_->TempAt(index)).W();
109  }
110 
111  Operand InputOperand2_32(size_t index) {
112  switch (AddressingModeField::decode(instr_->opcode())) {
113  case kMode_None:
114  return InputOperand32(index);
115  case kMode_Operand2_R_LSL_I:
116  return Operand(InputRegister32(index), LSL, InputInt5(index + 1));
117  case kMode_Operand2_R_LSR_I:
118  return Operand(InputRegister32(index), LSR, InputInt5(index + 1));
119  case kMode_Operand2_R_ASR_I:
120  return Operand(InputRegister32(index), ASR, InputInt5(index + 1));
121  case kMode_Operand2_R_ROR_I:
122  return Operand(InputRegister32(index), ROR, InputInt5(index + 1));
123  case kMode_Operand2_R_UXTB:
124  return Operand(InputRegister32(index), UXTB);
125  case kMode_Operand2_R_UXTH:
126  return Operand(InputRegister32(index), UXTH);
127  case kMode_Operand2_R_SXTB:
128  return Operand(InputRegister32(index), SXTB);
129  case kMode_Operand2_R_SXTH:
130  return Operand(InputRegister32(index), SXTH);
131  case kMode_Operand2_R_SXTW:
132  return Operand(InputRegister32(index), SXTW);
133  case kMode_MRI:
134  case kMode_MRR:
135  case kMode_Root:
136  break;
137  }
138  UNREACHABLE();
139  }
140 
141  Operand InputOperand2_64(size_t index) {
142  switch (AddressingModeField::decode(instr_->opcode())) {
143  case kMode_None:
144  return InputOperand64(index);
145  case kMode_Operand2_R_LSL_I:
146  return Operand(InputRegister64(index), LSL, InputInt6(index + 1));
147  case kMode_Operand2_R_LSR_I:
148  return Operand(InputRegister64(index), LSR, InputInt6(index + 1));
149  case kMode_Operand2_R_ASR_I:
150  return Operand(InputRegister64(index), ASR, InputInt6(index + 1));
151  case kMode_Operand2_R_ROR_I:
152  return Operand(InputRegister64(index), ROR, InputInt6(index + 1));
153  case kMode_Operand2_R_UXTB:
154  return Operand(InputRegister64(index), UXTB);
155  case kMode_Operand2_R_UXTH:
156  return Operand(InputRegister64(index), UXTH);
157  case kMode_Operand2_R_SXTB:
158  return Operand(InputRegister64(index), SXTB);
159  case kMode_Operand2_R_SXTH:
160  return Operand(InputRegister64(index), SXTH);
161  case kMode_Operand2_R_SXTW:
162  return Operand(InputRegister64(index), SXTW);
163  case kMode_MRI:
164  case kMode_MRR:
165  case kMode_Root:
166  break;
167  }
168  UNREACHABLE();
169  }
170 
171  MemOperand MemoryOperand(size_t index = 0) {
172  switch (AddressingModeField::decode(instr_->opcode())) {
173  case kMode_None:
174  case kMode_Operand2_R_LSR_I:
175  case kMode_Operand2_R_ASR_I:
176  case kMode_Operand2_R_ROR_I:
177  case kMode_Operand2_R_UXTB:
178  case kMode_Operand2_R_UXTH:
179  case kMode_Operand2_R_SXTB:
180  case kMode_Operand2_R_SXTH:
181  case kMode_Operand2_R_SXTW:
182  break;
183  case kMode_Root:
184  return MemOperand(kRootRegister, InputInt64(index));
185  case kMode_Operand2_R_LSL_I:
186  return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
187  LSL, InputInt32(index + 2));
188  case kMode_MRI:
189  return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
190  case kMode_MRR:
191  return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
192  }
193  UNREACHABLE();
194  }
195 
196  Operand ToOperand(InstructionOperand* op) {
197  if (op->IsRegister()) {
198  return Operand(ToRegister(op));
199  }
200  return ToImmediate(op);
201  }
202 
203  Operand ToOperand32(InstructionOperand* op) {
204  if (op->IsRegister()) {
205  return Operand(ToRegister(op).W());
206  }
207  return ToImmediate(op);
208  }
209 
210  Operand ToImmediate(InstructionOperand* operand) {
211  Constant constant = ToConstant(operand);
212  switch (constant.type()) {
213  case Constant::kInt32:
214  return Operand(constant.ToInt32());
215  case Constant::kInt64:
216  if (RelocInfo::IsWasmReference(constant.rmode())) {
217  return Operand(constant.ToInt64(), constant.rmode());
218  } else {
219  return Operand(constant.ToInt64());
220  }
221  case Constant::kFloat32:
222  return Operand(Operand::EmbeddedNumber(constant.ToFloat32()));
223  case Constant::kFloat64:
224  return Operand(Operand::EmbeddedNumber(constant.ToFloat64().value()));
225  case Constant::kExternalReference:
226  return Operand(constant.ToExternalReference());
227  case Constant::kHeapObject:
228  return Operand(constant.ToHeapObject());
229  case Constant::kDelayedStringConstant:
230  return Operand::EmbeddedStringConstant(
231  constant.ToDelayedStringConstant());
232  case Constant::kRpoNumber:
233  UNREACHABLE(); // TODO(dcarney): RPO immediates on arm64.
234  break;
235  }
236  UNREACHABLE();
237  }
238 
239  MemOperand ToMemOperand(InstructionOperand* op, TurboAssembler* tasm) const {
240  DCHECK_NOT_NULL(op);
241  DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
242  return SlotToMemOperand(AllocatedOperand::cast(op)->index(), tasm);
243  }
244 
245  MemOperand SlotToMemOperand(int slot, TurboAssembler* tasm) const {
246  FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
247  if (offset.from_frame_pointer()) {
248  int from_sp = offset.offset() + frame_access_state()->GetSPToFPOffset();
249  // Convert FP-offsets to SP-offsets if it results in better code.
250  if (Assembler::IsImmLSUnscaled(from_sp) ||
251  Assembler::IsImmLSScaled(from_sp, 3)) {
252  offset = FrameOffset::FromStackPointer(from_sp);
253  }
254  }
255  return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
256  }
257 };
258 
259 namespace {
260 
261 class OutOfLineRecordWrite final : public OutOfLineCode {
262  public:
263  OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index,
264  Register value, Register scratch0, Register scratch1,
265  RecordWriteMode mode, StubCallMode stub_mode,
266  UnwindingInfoWriter* unwinding_info_writer)
267  : OutOfLineCode(gen),
268  object_(object),
269  index_(index),
270  value_(value),
271  scratch0_(scratch0),
272  scratch1_(scratch1),
273  mode_(mode),
274  stub_mode_(stub_mode),
275  must_save_lr_(!gen->frame_access_state()->has_frame()),
276  unwinding_info_writer_(unwinding_info_writer),
277  zone_(gen->zone()) {}
278 
279  void Generate() final {
280  if (mode_ > RecordWriteMode::kValueIsPointer) {
281  __ JumpIfSmi(value_, exit());
282  }
283  __ CheckPageFlagClear(value_, scratch0_,
284  MemoryChunk::kPointersToHereAreInterestingMask,
285  exit());
286  __ Add(scratch1_, object_, index_);
287  RememberedSetAction const remembered_set_action =
288  mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
289  : OMIT_REMEMBERED_SET;
290  SaveFPRegsMode const save_fp_mode =
291  frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
292  if (must_save_lr_) {
293  // We need to save and restore lr if the frame was elided.
294  __ Push(lr, padreg);
295  unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(), sp);
296  }
297  if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
298  // A direct call to a wasm runtime stub defined in this module.
299  // Just encode the stub index. This will be patched when the code
300  // is added to the native module and copied into wasm code space.
301  __ CallRecordWriteStub(object_, scratch1_, remembered_set_action,
302  save_fp_mode, wasm::WasmCode::kWasmRecordWrite);
303  } else {
304  __ CallRecordWriteStub(object_, scratch1_, remembered_set_action,
305  save_fp_mode);
306  }
307  if (must_save_lr_) {
308  __ Pop(padreg, lr);
309  unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
310  }
311  }
312 
313  private:
314  Register const object_;
315  Operand const index_;
316  Register const value_;
317  Register const scratch0_;
318  Register const scratch1_;
319  RecordWriteMode const mode_;
320  StubCallMode const stub_mode_;
321  bool must_save_lr_;
322  UnwindingInfoWriter* const unwinding_info_writer_;
323  Zone* zone_;
324 };
325 
326 Condition FlagsConditionToCondition(FlagsCondition condition) {
327  switch (condition) {
328  case kEqual:
329  return eq;
330  case kNotEqual:
331  return ne;
332  case kSignedLessThan:
333  return lt;
334  case kSignedGreaterThanOrEqual:
335  return ge;
336  case kSignedLessThanOrEqual:
337  return le;
338  case kSignedGreaterThan:
339  return gt;
340  case kUnsignedLessThan:
341  return lo;
342  case kUnsignedGreaterThanOrEqual:
343  return hs;
344  case kUnsignedLessThanOrEqual:
345  return ls;
346  case kUnsignedGreaterThan:
347  return hi;
348  case kFloatLessThanOrUnordered:
349  return lt;
350  case kFloatGreaterThanOrEqual:
351  return ge;
352  case kFloatLessThanOrEqual:
353  return ls;
354  case kFloatGreaterThanOrUnordered:
355  return hi;
356  case kFloatLessThan:
357  return lo;
358  case kFloatGreaterThanOrEqualOrUnordered:
359  return hs;
360  case kFloatLessThanOrEqualOrUnordered:
361  return le;
362  case kFloatGreaterThan:
363  return gt;
364  case kOverflow:
365  return vs;
366  case kNotOverflow:
367  return vc;
368  case kUnorderedEqual:
369  case kUnorderedNotEqual:
370  break;
371  case kPositiveOrZero:
372  return pl;
373  case kNegative:
374  return mi;
375  }
376  UNREACHABLE();
377 }
378 
379 void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen,
380  InstructionCode opcode, Instruction* instr,
381  Arm64OperandConverter& i) {
382  const MemoryAccessMode access_mode =
383  static_cast<MemoryAccessMode>(MiscField::decode(opcode));
384  if (access_mode == kMemoryAccessPoisoned) {
385  Register value = i.OutputRegister();
386  Register poison = value.Is64Bits() ? kSpeculationPoisonRegister
387  : kSpeculationPoisonRegister.W();
388  codegen->tasm()->And(value, value, Operand(poison));
389  }
390 }
391 
392 } // namespace
393 
394 #define ASSEMBLE_SHIFT(asm_instr, width) \
395  do { \
396  if (instr->InputAt(1)->IsRegister()) { \
397  __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
398  i.InputRegister##width(1)); \
399  } else { \
400  uint32_t imm = \
401  static_cast<uint32_t>(i.InputOperand##width(1).ImmediateValue()); \
402  __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), \
403  imm % (width)); \
404  } \
405  } while (0)
406 
407 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr, reg) \
408  do { \
409  __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
410  __ asm_instr(i.Output##reg(), i.TempRegister(0)); \
411  } while (0)
412 
413 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr, reg) \
414  do { \
415  __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
416  __ asm_instr(i.Input##reg(2), i.TempRegister(0)); \
417  } while (0)
418 
419 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr, reg) \
420  do { \
421  Label exchange; \
422  __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
423  __ Bind(&exchange); \
424  __ load_instr(i.Output##reg(), i.TempRegister(0)); \
425  __ store_instr(i.TempRegister32(1), i.Input##reg(2), i.TempRegister(0)); \
426  __ Cbnz(i.TempRegister32(1), &exchange); \
427  } while (0)
428 
429 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(load_instr, store_instr, ext, \
430  reg) \
431  do { \
432  Label compareExchange; \
433  Label exit; \
434  __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
435  __ Bind(&compareExchange); \
436  __ load_instr(i.Output##reg(), i.TempRegister(0)); \
437  __ Cmp(i.Output##reg(), Operand(i.Input##reg(2), ext)); \
438  __ B(ne, &exit); \
439  __ store_instr(i.TempRegister32(1), i.Input##reg(3), i.TempRegister(0)); \
440  __ Cbnz(i.TempRegister32(1), &compareExchange); \
441  __ Bind(&exit); \
442  } while (0)
443 
444 #define ASSEMBLE_ATOMIC_BINOP(load_instr, store_instr, bin_instr, reg) \
445  do { \
446  Label binop; \
447  __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
448  __ Bind(&binop); \
449  __ load_instr(i.Output##reg(), i.TempRegister(0)); \
450  __ bin_instr(i.Temp##reg(1), i.Output##reg(), Operand(i.Input##reg(2))); \
451  __ store_instr(i.TempRegister32(2), i.Temp##reg(1), i.TempRegister(0)); \
452  __ Cbnz(i.TempRegister32(2), &binop); \
453  } while (0)
454 
455 #define ASSEMBLE_IEEE754_BINOP(name) \
456  do { \
457  FrameScope scope(tasm(), StackFrame::MANUAL); \
458  __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
459  } while (0)
460 
461 #define ASSEMBLE_IEEE754_UNOP(name) \
462  do { \
463  FrameScope scope(tasm(), StackFrame::MANUAL); \
464  __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
465  } while (0)
466 
467 void CodeGenerator::AssembleDeconstructFrame() {
468  __ Mov(sp, fp);
469  __ Pop(fp, lr);
470 
471  unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
472 }
473 
474 void CodeGenerator::AssemblePrepareTailCall() {
475  if (frame_access_state()->has_frame()) {
476  __ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
477  __ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
478  }
479  frame_access_state()->SetFrameAccessToSP();
480 }
481 
482 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
483  Register scratch1,
484  Register scratch2,
485  Register scratch3) {
486  DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
487  Label done;
488 
489  // Check if current frame is an arguments adaptor frame.
490  __ Ldr(scratch1, MemOperand(fp, StandardFrameConstants::kContextOffset));
491  __ Cmp(scratch1,
492  Operand(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
493  __ B(ne, &done);
494 
495  // Load arguments count from current arguments adaptor frame (note, it
496  // does not include receiver).
497  Register caller_args_count_reg = scratch1;
498  __ Ldr(caller_args_count_reg,
499  MemOperand(fp, ArgumentsAdaptorFrameConstants::kLengthOffset));
500  __ SmiUntag(caller_args_count_reg);
501 
502  ParameterCount callee_args_count(args_reg);
503  __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
504  scratch3);
505  __ bind(&done);
506 }
507 
508 namespace {
509 
510 void AdjustStackPointerForTailCall(TurboAssembler* tasm,
511  FrameAccessState* state,
512  int new_slot_above_sp,
513  bool allow_shrinkage = true) {
514  int current_sp_offset = state->GetSPToFPSlotCount() +
515  StandardFrameConstants::kFixedSlotCountAboveFp;
516  int stack_slot_delta = new_slot_above_sp - current_sp_offset;
517  DCHECK_EQ(stack_slot_delta % 2, 0);
518  if (stack_slot_delta > 0) {
519  tasm->Claim(stack_slot_delta);
520  state->IncreaseSPDelta(stack_slot_delta);
521  } else if (allow_shrinkage && stack_slot_delta < 0) {
522  tasm->Drop(-stack_slot_delta);
523  state->IncreaseSPDelta(stack_slot_delta);
524  }
525 }
526 
527 } // namespace
528 
529 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
530  int first_unused_stack_slot) {
531  AdjustStackPointerForTailCall(tasm(), frame_access_state(),
532  first_unused_stack_slot, false);
533 }
534 
535 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
536  int first_unused_stack_slot) {
537  DCHECK_EQ(first_unused_stack_slot % 2, 0);
538  AdjustStackPointerForTailCall(tasm(), frame_access_state(),
539  first_unused_stack_slot);
540  DCHECK(instr->IsTailCall());
541  InstructionOperandConverter g(this, instr);
542  int optional_padding_slot = g.InputInt32(instr->InputCount() - 2);
543  if (optional_padding_slot % 2) {
544  __ Poke(padreg, optional_padding_slot * kPointerSize);
545  }
546 }
547 
548 // Check that {kJavaScriptCallCodeStartRegister} is correct.
549 void CodeGenerator::AssembleCodeStartRegisterCheck() {
550  UseScratchRegisterScope temps(tasm());
551  Register scratch = temps.AcquireX();
552  __ ComputeCodeStartAddress(scratch);
553  __ cmp(scratch, kJavaScriptCallCodeStartRegister);
554  __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
555 }
556 
557 // Check if the code object is marked for deoptimization. If it is, then it
558 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
559 // to:
560 // 1. read from memory the word that contains that bit, which can be found in
561 // the flags in the referenced {CodeDataContainer} object;
562 // 2. test kMarkedForDeoptimizationBit in those flags; and
563 // 3. if it is not zero then it jumps to the builtin.
564 void CodeGenerator::BailoutIfDeoptimized() {
565  UseScratchRegisterScope temps(tasm());
566  Register scratch = temps.AcquireX();
567  int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
568  __ Ldr(scratch, MemOperand(kJavaScriptCallCodeStartRegister, offset));
569  __ Ldr(scratch,
570  FieldMemOperand(scratch, CodeDataContainer::kKindSpecificFlagsOffset));
571  Label not_deoptimized;
572  __ Tbz(scratch, Code::kMarkedForDeoptimizationBit, &not_deoptimized);
573  // Ensure we're not serializing (otherwise we'd need to use an indirection to
574  // access the builtin below).
575  DCHECK(!isolate()->ShouldLoadConstantsFromRootList());
576  Handle<Code> code = isolate()->builtins()->builtin_handle(
577  Builtins::kCompileLazyDeoptimizedCode);
578  __ Jump(code, RelocInfo::CODE_TARGET);
579  __ Bind(&not_deoptimized);
580 }
581 
582 void CodeGenerator::GenerateSpeculationPoisonFromCodeStartRegister() {
583  UseScratchRegisterScope temps(tasm());
584  Register scratch = temps.AcquireX();
585 
586  // Set a mask which has all bits set in the normal case, but has all
587  // bits cleared if we are speculatively executing the wrong PC.
588  __ ComputeCodeStartAddress(scratch);
589  __ Cmp(kJavaScriptCallCodeStartRegister, scratch);
590  __ Csetm(kSpeculationPoisonRegister, eq);
591  __ Csdb();
592 }
593 
594 void CodeGenerator::AssembleRegisterArgumentPoisoning() {
595  UseScratchRegisterScope temps(tasm());
596  Register scratch = temps.AcquireX();
597 
598  __ Mov(scratch, sp);
599  __ And(kJSFunctionRegister, kJSFunctionRegister, kSpeculationPoisonRegister);
600  __ And(kContextRegister, kContextRegister, kSpeculationPoisonRegister);
601  __ And(scratch, scratch, kSpeculationPoisonRegister);
602  __ Mov(sp, scratch);
603 }
604 
605 // Assembles an instruction after register allocation, producing machine code.
606 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
607  Instruction* instr) {
608  Arm64OperandConverter i(this, instr);
609  InstructionCode opcode = instr->opcode();
610  ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
611  switch (arch_opcode) {
612  case kArchCallCodeObject: {
613  if (instr->InputAt(0)->IsImmediate()) {
614  __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
615  } else {
616  Register reg = i.InputRegister(0);
617  DCHECK_IMPLIES(
618  HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister),
619  reg == kJavaScriptCallCodeStartRegister);
620  __ Add(reg, reg, Code::kHeaderSize - kHeapObjectTag);
621  __ Call(reg);
622  }
623  RecordCallPosition(instr);
624  frame_access_state()->ClearSPDelta();
625  break;
626  }
627  case kArchCallWasmFunction: {
628  if (instr->InputAt(0)->IsImmediate()) {
629  Constant constant = i.ToConstant(instr->InputAt(0));
630  Address wasm_code = static_cast<Address>(constant.ToInt64());
631  __ Call(wasm_code, constant.rmode());
632  } else {
633  Register target = i.InputRegister(0);
634  __ Call(target);
635  }
636  RecordCallPosition(instr);
637  frame_access_state()->ClearSPDelta();
638  break;
639  }
640  case kArchTailCallCodeObjectFromJSFunction:
641  case kArchTailCallCodeObject: {
642  if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
643  AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
644  i.TempRegister(0), i.TempRegister(1),
645  i.TempRegister(2));
646  }
647  if (instr->InputAt(0)->IsImmediate()) {
648  __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
649  } else {
650  Register reg = i.InputRegister(0);
651  DCHECK_IMPLIES(
652  HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister),
653  reg == kJavaScriptCallCodeStartRegister);
654  __ Add(reg, reg, Code::kHeaderSize - kHeapObjectTag);
655  __ Jump(reg);
656  }
657  unwinding_info_writer_.MarkBlockWillExit();
658  frame_access_state()->ClearSPDelta();
659  frame_access_state()->SetFrameAccessToDefault();
660  break;
661  }
662  case kArchTailCallWasm: {
663  if (instr->InputAt(0)->IsImmediate()) {
664  Constant constant = i.ToConstant(instr->InputAt(0));
665  Address wasm_code = static_cast<Address>(constant.ToInt64());
666  __ Jump(wasm_code, constant.rmode());
667  } else {
668  Register target = i.InputRegister(0);
669  __ Jump(target);
670  }
671  unwinding_info_writer_.MarkBlockWillExit();
672  frame_access_state()->ClearSPDelta();
673  frame_access_state()->SetFrameAccessToDefault();
674  break;
675  }
676  case kArchTailCallAddress: {
677  CHECK(!instr->InputAt(0)->IsImmediate());
678  Register reg = i.InputRegister(0);
679  DCHECK_IMPLIES(
680  HasCallDescriptorFlag(instr, CallDescriptor::kFixedTargetRegister),
681  reg == kJavaScriptCallCodeStartRegister);
682  __ Jump(reg);
683  unwinding_info_writer_.MarkBlockWillExit();
684  frame_access_state()->ClearSPDelta();
685  frame_access_state()->SetFrameAccessToDefault();
686  break;
687  }
688  case kArchCallJSFunction: {
689  Register func = i.InputRegister(0);
690  if (FLAG_debug_code) {
691  // Check the function's context matches the context argument.
692  UseScratchRegisterScope scope(tasm());
693  Register temp = scope.AcquireX();
694  __ Ldr(temp, FieldMemOperand(func, JSFunction::kContextOffset));
695  __ cmp(cp, temp);
696  __ Assert(eq, AbortReason::kWrongFunctionContext);
697  }
698  static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
699  __ Ldr(x2, FieldMemOperand(func, JSFunction::kCodeOffset));
700  __ Add(x2, x2, Operand(Code::kHeaderSize - kHeapObjectTag));
701  __ Call(x2);
702  RecordCallPosition(instr);
703  frame_access_state()->ClearSPDelta();
704  break;
705  }
706  case kArchPrepareCallCFunction:
707  // We don't need kArchPrepareCallCFunction on arm64 as the instruction
708  // selector has already performed a Claim to reserve space on the stack.
709  // Frame alignment is always 16 bytes, and the stack pointer is already
710  // 16-byte aligned, therefore we do not need to align the stack pointer
711  // by an unknown value, and it is safe to continue accessing the frame
712  // via the stack pointer.
713  UNREACHABLE();
714  break;
715  case kArchSaveCallerRegisters: {
716  fp_mode_ =
717  static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
718  DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
719  // kReturnRegister0 should have been saved before entering the stub.
720  int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
721  DCHECK_EQ(0, bytes % kPointerSize);
722  DCHECK_EQ(0, frame_access_state()->sp_delta());
723  frame_access_state()->IncreaseSPDelta(bytes / kPointerSize);
724  DCHECK(!caller_registers_saved_);
725  caller_registers_saved_ = true;
726  break;
727  }
728  case kArchRestoreCallerRegisters: {
729  DCHECK(fp_mode_ ==
730  static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
731  DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
732  // Don't overwrite the returned value.
733  int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
734  frame_access_state()->IncreaseSPDelta(-(bytes / kPointerSize));
735  DCHECK_EQ(0, frame_access_state()->sp_delta());
736  DCHECK(caller_registers_saved_);
737  caller_registers_saved_ = false;
738  break;
739  }
740  case kArchPrepareTailCall:
741  AssemblePrepareTailCall();
742  break;
743  case kArchCallCFunction: {
744  int const num_parameters = MiscField::decode(instr->opcode());
745  if (instr->InputAt(0)->IsImmediate()) {
746  ExternalReference ref = i.InputExternalReference(0);
747  __ CallCFunction(ref, num_parameters, 0);
748  } else {
749  Register func = i.InputRegister(0);
750  __ CallCFunction(func, num_parameters, 0);
751  }
752  frame_access_state()->SetFrameAccessToDefault();
753  // Ideally, we should decrement SP delta to match the change of stack
754  // pointer in CallCFunction. However, for certain architectures (e.g.
755  // ARM), there may be more strict alignment requirement, causing old SP
756  // to be saved on the stack. In those cases, we can not calculate the SP
757  // delta statically.
758  frame_access_state()->ClearSPDelta();
759  if (caller_registers_saved_) {
760  // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
761  // Here, we assume the sequence to be:
762  // kArchSaveCallerRegisters;
763  // kArchCallCFunction;
764  // kArchRestoreCallerRegisters;
765  int bytes =
766  __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
767  frame_access_state()->IncreaseSPDelta(bytes / kPointerSize);
768  }
769  break;
770  }
771  case kArchJmp:
772  AssembleArchJump(i.InputRpo(0));
773  break;
774  case kArchTableSwitch:
775  AssembleArchTableSwitch(instr);
776  break;
777  case kArchBinarySearchSwitch:
778  AssembleArchBinarySearchSwitch(instr);
779  break;
780  case kArchLookupSwitch:
781  AssembleArchLookupSwitch(instr);
782  break;
783  case kArchDebugAbort:
784  DCHECK(i.InputRegister(0).is(x1));
785  if (!frame_access_state()->has_frame()) {
786  // We don't actually want to generate a pile of code for this, so just
787  // claim there is a stack frame, without generating one.
788  FrameScope scope(tasm(), StackFrame::NONE);
789  __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS),
790  RelocInfo::CODE_TARGET);
791  } else {
792  __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS),
793  RelocInfo::CODE_TARGET);
794  }
795  __ Debug("kArchDebugAbort", 0, BREAK);
796  unwinding_info_writer_.MarkBlockWillExit();
797  break;
798  case kArchDebugBreak:
799  __ Debug("kArchDebugBreak", 0, BREAK);
800  break;
801  case kArchComment:
802  __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
803  break;
804  case kArchThrowTerminator:
805  unwinding_info_writer_.MarkBlockWillExit();
806  break;
807  case kArchNop:
808  // don't emit code for nops.
809  break;
810  case kArchDeoptimize: {
811  int deopt_state_id =
812  BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
813  CodeGenResult result =
814  AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
815  if (result != kSuccess) return result;
816  unwinding_info_writer_.MarkBlockWillExit();
817  break;
818  }
819  case kArchRet:
820  AssembleReturn(instr->InputAt(0));
821  break;
822  case kArchStackPointer:
823  __ mov(i.OutputRegister(), sp);
824  break;
825  case kArchFramePointer:
826  __ mov(i.OutputRegister(), fp);
827  break;
828  case kArchParentFramePointer:
829  if (frame_access_state()->has_frame()) {
830  __ ldr(i.OutputRegister(), MemOperand(fp, 0));
831  } else {
832  __ mov(i.OutputRegister(), fp);
833  }
834  break;
835  case kArchTruncateDoubleToI:
836  __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
837  i.InputDoubleRegister(0), DetermineStubCallMode());
838  break;
839  case kArchStoreWithWriteBarrier: {
840  RecordWriteMode mode =
841  static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
842  AddressingMode addressing_mode =
843  AddressingModeField::decode(instr->opcode());
844  Register object = i.InputRegister(0);
845  Operand index(0);
846  if (addressing_mode == kMode_MRI) {
847  index = Operand(i.InputInt64(1));
848  } else {
849  DCHECK_EQ(addressing_mode, kMode_MRR);
850  index = Operand(i.InputRegister(1));
851  }
852  Register value = i.InputRegister(2);
853  Register scratch0 = i.TempRegister(0);
854  Register scratch1 = i.TempRegister(1);
855  auto ool = new (zone()) OutOfLineRecordWrite(
856  this, object, index, value, scratch0, scratch1, mode,
857  DetermineStubCallMode(), &unwinding_info_writer_);
858  __ Str(value, MemOperand(object, index));
859  __ CheckPageFlagSet(object, scratch0,
860  MemoryChunk::kPointersFromHereAreInterestingMask,
861  ool->entry());
862  __ Bind(ool->exit());
863  break;
864  }
865  case kArchStackSlot: {
866  FrameOffset offset =
867  frame_access_state()->GetFrameOffset(i.InputInt32(0));
868  Register base = offset.from_stack_pointer() ? sp : fp;
869  __ Add(i.OutputRegister(0), base, Operand(offset.offset()));
870  break;
871  }
872  case kIeee754Float64Acos:
873  ASSEMBLE_IEEE754_UNOP(acos);
874  break;
875  case kIeee754Float64Acosh:
876  ASSEMBLE_IEEE754_UNOP(acosh);
877  break;
878  case kIeee754Float64Asin:
879  ASSEMBLE_IEEE754_UNOP(asin);
880  break;
881  case kIeee754Float64Asinh:
882  ASSEMBLE_IEEE754_UNOP(asinh);
883  break;
884  case kIeee754Float64Atan:
885  ASSEMBLE_IEEE754_UNOP(atan);
886  break;
887  case kIeee754Float64Atanh:
888  ASSEMBLE_IEEE754_UNOP(atanh);
889  break;
890  case kIeee754Float64Atan2:
891  ASSEMBLE_IEEE754_BINOP(atan2);
892  break;
893  case kIeee754Float64Cos:
894  ASSEMBLE_IEEE754_UNOP(cos);
895  break;
896  case kIeee754Float64Cosh:
897  ASSEMBLE_IEEE754_UNOP(cosh);
898  break;
899  case kIeee754Float64Cbrt:
900  ASSEMBLE_IEEE754_UNOP(cbrt);
901  break;
902  case kIeee754Float64Exp:
903  ASSEMBLE_IEEE754_UNOP(exp);
904  break;
905  case kIeee754Float64Expm1:
906  ASSEMBLE_IEEE754_UNOP(expm1);
907  break;
908  case kIeee754Float64Log:
909  ASSEMBLE_IEEE754_UNOP(log);
910  break;
911  case kIeee754Float64Log1p:
912  ASSEMBLE_IEEE754_UNOP(log1p);
913  break;
914  case kIeee754Float64Log2:
915  ASSEMBLE_IEEE754_UNOP(log2);
916  break;
917  case kIeee754Float64Log10:
918  ASSEMBLE_IEEE754_UNOP(log10);
919  break;
920  case kIeee754Float64Pow: {
921  __ Call(BUILTIN_CODE(isolate(), MathPowInternal), RelocInfo::CODE_TARGET);
922  break;
923  }
924  case kIeee754Float64Sin:
925  ASSEMBLE_IEEE754_UNOP(sin);
926  break;
927  case kIeee754Float64Sinh:
928  ASSEMBLE_IEEE754_UNOP(sinh);
929  break;
930  case kIeee754Float64Tan:
931  ASSEMBLE_IEEE754_UNOP(tan);
932  break;
933  case kIeee754Float64Tanh:
934  ASSEMBLE_IEEE754_UNOP(tanh);
935  break;
936  case kArm64Float32RoundDown:
937  __ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0));
938  break;
939  case kArm64Float64RoundDown:
940  __ Frintm(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
941  break;
942  case kArm64Float32RoundUp:
943  __ Frintp(i.OutputFloat32Register(), i.InputFloat32Register(0));
944  break;
945  case kArm64Float64RoundUp:
946  __ Frintp(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
947  break;
948  case kArm64Float64RoundTiesAway:
949  __ Frinta(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
950  break;
951  case kArm64Float32RoundTruncate:
952  __ Frintz(i.OutputFloat32Register(), i.InputFloat32Register(0));
953  break;
954  case kArm64Float64RoundTruncate:
955  __ Frintz(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
956  break;
957  case kArm64Float32RoundTiesEven:
958  __ Frintn(i.OutputFloat32Register(), i.InputFloat32Register(0));
959  break;
960  case kArm64Float64RoundTiesEven:
961  __ Frintn(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
962  break;
963  case kArm64Add:
964  if (FlagsModeField::decode(opcode) != kFlags_none) {
965  __ Adds(i.OutputRegister(), i.InputOrZeroRegister64(0),
966  i.InputOperand2_64(1));
967  } else {
968  __ Add(i.OutputRegister(), i.InputOrZeroRegister64(0),
969  i.InputOperand2_64(1));
970  }
971  break;
972  case kArm64Add32:
973  if (FlagsModeField::decode(opcode) != kFlags_none) {
974  __ Adds(i.OutputRegister32(), i.InputOrZeroRegister32(0),
975  i.InputOperand2_32(1));
976  } else {
977  __ Add(i.OutputRegister32(), i.InputOrZeroRegister32(0),
978  i.InputOperand2_32(1));
979  }
980  break;
981  case kArm64And:
982  if (FlagsModeField::decode(opcode) != kFlags_none) {
983  // The ands instruction only sets N and Z, so only the following
984  // conditions make sense.
985  DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
986  FlagsConditionField::decode(opcode) == kNotEqual ||
987  FlagsConditionField::decode(opcode) == kPositiveOrZero ||
988  FlagsConditionField::decode(opcode) == kNegative);
989  __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
990  i.InputOperand2_64(1));
991  } else {
992  __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
993  i.InputOperand2_64(1));
994  }
995  break;
996  case kArm64And32:
997  if (FlagsModeField::decode(opcode) != kFlags_none) {
998  // The ands instruction only sets N and Z, so only the following
999  // conditions make sense.
1000  DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
1001  FlagsConditionField::decode(opcode) == kNotEqual ||
1002  FlagsConditionField::decode(opcode) == kPositiveOrZero ||
1003  FlagsConditionField::decode(opcode) == kNegative);
1004  __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1005  i.InputOperand2_32(1));
1006  } else {
1007  __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1008  i.InputOperand2_32(1));
1009  }
1010  break;
1011  case kArm64Bic:
1012  __ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
1013  i.InputOperand2_64(1));
1014  break;
1015  case kArm64Bic32:
1016  __ Bic(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1017  i.InputOperand2_32(1));
1018  break;
1019  case kArm64Mul:
1020  __ Mul(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1021  break;
1022  case kArm64Mul32:
1023  __ Mul(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1024  break;
1025  case kArm64Smull:
1026  __ Smull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
1027  break;
1028  case kArm64Umull:
1029  __ Umull(i.OutputRegister(), i.InputRegister32(0), i.InputRegister32(1));
1030  break;
1031  case kArm64Madd:
1032  __ Madd(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1033  i.InputRegister(2));
1034  break;
1035  case kArm64Madd32:
1036  __ Madd(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1037  i.InputRegister32(2));
1038  break;
1039  case kArm64Msub:
1040  __ Msub(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1),
1041  i.InputRegister(2));
1042  break;
1043  case kArm64Msub32:
1044  __ Msub(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1),
1045  i.InputRegister32(2));
1046  break;
1047  case kArm64Mneg:
1048  __ Mneg(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1049  break;
1050  case kArm64Mneg32:
1051  __ Mneg(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1052  break;
1053  case kArm64Idiv:
1054  __ Sdiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1055  break;
1056  case kArm64Idiv32:
1057  __ Sdiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1058  break;
1059  case kArm64Udiv:
1060  __ Udiv(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1));
1061  break;
1062  case kArm64Udiv32:
1063  __ Udiv(i.OutputRegister32(), i.InputRegister32(0), i.InputRegister32(1));
1064  break;
1065  case kArm64Imod: {
1066  UseScratchRegisterScope scope(tasm());
1067  Register temp = scope.AcquireX();
1068  __ Sdiv(temp, i.InputRegister(0), i.InputRegister(1));
1069  __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1070  break;
1071  }
1072  case kArm64Imod32: {
1073  UseScratchRegisterScope scope(tasm());
1074  Register temp = scope.AcquireW();
1075  __ Sdiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1076  __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1077  i.InputRegister32(0));
1078  break;
1079  }
1080  case kArm64Umod: {
1081  UseScratchRegisterScope scope(tasm());
1082  Register temp = scope.AcquireX();
1083  __ Udiv(temp, i.InputRegister(0), i.InputRegister(1));
1084  __ Msub(i.OutputRegister(), temp, i.InputRegister(1), i.InputRegister(0));
1085  break;
1086  }
1087  case kArm64Umod32: {
1088  UseScratchRegisterScope scope(tasm());
1089  Register temp = scope.AcquireW();
1090  __ Udiv(temp, i.InputRegister32(0), i.InputRegister32(1));
1091  __ Msub(i.OutputRegister32(), temp, i.InputRegister32(1),
1092  i.InputRegister32(0));
1093  break;
1094  }
1095  case kArm64Not:
1096  __ Mvn(i.OutputRegister(), i.InputOperand(0));
1097  break;
1098  case kArm64Not32:
1099  __ Mvn(i.OutputRegister32(), i.InputOperand32(0));
1100  break;
1101  case kArm64Or:
1102  __ Orr(i.OutputRegister(), i.InputOrZeroRegister64(0),
1103  i.InputOperand2_64(1));
1104  break;
1105  case kArm64Or32:
1106  __ Orr(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1107  i.InputOperand2_32(1));
1108  break;
1109  case kArm64Orn:
1110  __ Orn(i.OutputRegister(), i.InputOrZeroRegister64(0),
1111  i.InputOperand2_64(1));
1112  break;
1113  case kArm64Orn32:
1114  __ Orn(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1115  i.InputOperand2_32(1));
1116  break;
1117  case kArm64Eor:
1118  __ Eor(i.OutputRegister(), i.InputOrZeroRegister64(0),
1119  i.InputOperand2_64(1));
1120  break;
1121  case kArm64Eor32:
1122  __ Eor(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1123  i.InputOperand2_32(1));
1124  break;
1125  case kArm64Eon:
1126  __ Eon(i.OutputRegister(), i.InputOrZeroRegister64(0),
1127  i.InputOperand2_64(1));
1128  break;
1129  case kArm64Eon32:
1130  __ Eon(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1131  i.InputOperand2_32(1));
1132  break;
1133  case kArm64Sub:
1134  if (FlagsModeField::decode(opcode) != kFlags_none) {
1135  __ Subs(i.OutputRegister(), i.InputOrZeroRegister64(0),
1136  i.InputOperand2_64(1));
1137  } else {
1138  __ Sub(i.OutputRegister(), i.InputOrZeroRegister64(0),
1139  i.InputOperand2_64(1));
1140  }
1141  break;
1142  case kArm64Sub32:
1143  if (FlagsModeField::decode(opcode) != kFlags_none) {
1144  __ Subs(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1145  i.InputOperand2_32(1));
1146  } else {
1147  __ Sub(i.OutputRegister32(), i.InputOrZeroRegister32(0),
1148  i.InputOperand2_32(1));
1149  }
1150  break;
1151  case kArm64Lsl:
1152  ASSEMBLE_SHIFT(Lsl, 64);
1153  break;
1154  case kArm64Lsl32:
1155  ASSEMBLE_SHIFT(Lsl, 32);
1156  break;
1157  case kArm64Lsr:
1158  ASSEMBLE_SHIFT(Lsr, 64);
1159  break;
1160  case kArm64Lsr32:
1161  ASSEMBLE_SHIFT(Lsr, 32);
1162  break;
1163  case kArm64Asr:
1164  ASSEMBLE_SHIFT(Asr, 64);
1165  break;
1166  case kArm64Asr32:
1167  ASSEMBLE_SHIFT(Asr, 32);
1168  break;
1169  case kArm64Ror:
1170  ASSEMBLE_SHIFT(Ror, 64);
1171  break;
1172  case kArm64Ror32:
1173  ASSEMBLE_SHIFT(Ror, 32);
1174  break;
1175  case kArm64Mov32:
1176  __ Mov(i.OutputRegister32(), i.InputRegister32(0));
1177  break;
1178  case kArm64Sxtb32:
1179  __ Sxtb(i.OutputRegister32(), i.InputRegister32(0));
1180  break;
1181  case kArm64Sxth32:
1182  __ Sxth(i.OutputRegister32(), i.InputRegister32(0));
1183  break;
1184  case kArm64Sxtb:
1185  __ Sxtb(i.OutputRegister(), i.InputRegister32(0));
1186  break;
1187  case kArm64Sxth:
1188  __ Sxth(i.OutputRegister(), i.InputRegister32(0));
1189  break;
1190  case kArm64Sxtw:
1191  __ Sxtw(i.OutputRegister(), i.InputRegister32(0));
1192  break;
1193  case kArm64Sbfx32:
1194  __ Sbfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1195  i.InputInt5(2));
1196  break;
1197  case kArm64Ubfx:
1198  __ Ubfx(i.OutputRegister(), i.InputRegister(0), i.InputInt6(1),
1199  i.InputInt32(2));
1200  break;
1201  case kArm64Ubfx32:
1202  __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1203  i.InputInt32(2));
1204  break;
1205  case kArm64Ubfiz32:
1206  __ Ubfiz(i.OutputRegister32(), i.InputRegister32(0), i.InputInt5(1),
1207  i.InputInt5(2));
1208  break;
1209  case kArm64Bfi:
1210  __ Bfi(i.OutputRegister(), i.InputRegister(1), i.InputInt6(2),
1211  i.InputInt6(3));
1212  break;
1213  case kArm64TestAndBranch32:
1214  case kArm64TestAndBranch:
1215  // Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
1216  break;
1217  case kArm64CompareAndBranch32:
1218  case kArm64CompareAndBranch:
1219  // Pseudo instruction handled in AssembleArchBranch.
1220  break;
1221  case kArm64Claim: {
1222  int count = i.InputInt32(0);
1223  DCHECK_EQ(count % 2, 0);
1224  __ AssertSpAligned();
1225  if (count > 0) {
1226  __ Claim(count);
1227  frame_access_state()->IncreaseSPDelta(count);
1228  }
1229  break;
1230  }
1231  case kArm64Poke: {
1232  Operand operand(i.InputInt32(1) * kPointerSize);
1233  if (instr->InputAt(0)->IsSimd128Register()) {
1234  __ Poke(i.InputSimd128Register(0), operand);
1235  } else if (instr->InputAt(0)->IsFPRegister()) {
1236  __ Poke(i.InputFloat64Register(0), operand);
1237  } else {
1238  __ Poke(i.InputOrZeroRegister64(0), operand);
1239  }
1240  break;
1241  }
1242  case kArm64PokePair: {
1243  int slot = i.InputInt32(2) - 1;
1244  if (instr->InputAt(0)->IsFPRegister()) {
1245  __ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
1246  slot * kPointerSize);
1247  } else {
1248  __ PokePair(i.InputRegister(1), i.InputRegister(0),
1249  slot * kPointerSize);
1250  }
1251  break;
1252  }
1253  case kArm64Peek: {
1254  int reverse_slot = i.InputInt32(0);
1255  int offset =
1256  FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1257  if (instr->OutputAt(0)->IsFPRegister()) {
1258  LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1259  if (op->representation() == MachineRepresentation::kFloat64) {
1260  __ Ldr(i.OutputDoubleRegister(), MemOperand(fp, offset));
1261  } else {
1262  DCHECK_EQ(MachineRepresentation::kFloat32, op->representation());
1263  __ Ldr(i.OutputFloatRegister(), MemOperand(fp, offset));
1264  }
1265  } else {
1266  __ Ldr(i.OutputRegister(), MemOperand(fp, offset));
1267  }
1268  break;
1269  }
1270  case kArm64Clz:
1271  __ Clz(i.OutputRegister64(), i.InputRegister64(0));
1272  break;
1273  case kArm64Clz32:
1274  __ Clz(i.OutputRegister32(), i.InputRegister32(0));
1275  break;
1276  case kArm64Rbit:
1277  __ Rbit(i.OutputRegister64(), i.InputRegister64(0));
1278  break;
1279  case kArm64Rbit32:
1280  __ Rbit(i.OutputRegister32(), i.InputRegister32(0));
1281  break;
1282  case kArm64Rev:
1283  __ Rev(i.OutputRegister64(), i.InputRegister64(0));
1284  break;
1285  case kArm64Rev32:
1286  __ Rev(i.OutputRegister32(), i.InputRegister32(0));
1287  break;
1288  case kArm64Cmp:
1289  __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1290  break;
1291  case kArm64Cmp32:
1292  __ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1293  break;
1294  case kArm64Cmn:
1295  __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1296  break;
1297  case kArm64Cmn32:
1298  __ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1299  break;
1300  case kArm64Tst:
1301  __ Tst(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
1302  break;
1303  case kArm64Tst32:
1304  __ Tst(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
1305  break;
1306  case kArm64Float32Cmp:
1307  if (instr->InputAt(1)->IsFPRegister()) {
1308  __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
1309  } else {
1310  DCHECK(instr->InputAt(1)->IsImmediate());
1311  // 0.0 is the only immediate supported by fcmp instructions.
1312  DCHECK_EQ(0.0f, i.InputFloat32(1));
1313  __ Fcmp(i.InputFloat32Register(0), i.InputFloat32(1));
1314  }
1315  break;
1316  case kArm64Float32Add:
1317  __ Fadd(i.OutputFloat32Register(), i.InputFloat32Register(0),
1318  i.InputFloat32Register(1));
1319  break;
1320  case kArm64Float32Sub:
1321  __ Fsub(i.OutputFloat32Register(), i.InputFloat32Register(0),
1322  i.InputFloat32Register(1));
1323  break;
1324  case kArm64Float32Mul:
1325  __ Fmul(i.OutputFloat32Register(), i.InputFloat32Register(0),
1326  i.InputFloat32Register(1));
1327  break;
1328  case kArm64Float32Div:
1329  __ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
1330  i.InputFloat32Register(1));
1331  break;
1332  case kArm64Float32Abs:
1333  __ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
1334  break;
1335  case kArm64Float32Neg:
1336  __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
1337  break;
1338  case kArm64Float32Sqrt:
1339  __ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
1340  break;
1341  case kArm64Float64Cmp:
1342  if (instr->InputAt(1)->IsFPRegister()) {
1343  __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1344  } else {
1345  DCHECK(instr->InputAt(1)->IsImmediate());
1346  // 0.0 is the only immediate supported by fcmp instructions.
1347  DCHECK_EQ(0.0, i.InputDouble(1));
1348  __ Fcmp(i.InputDoubleRegister(0), i.InputDouble(1));
1349  }
1350  break;
1351  case kArm64Float64Add:
1352  __ Fadd(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1353  i.InputDoubleRegister(1));
1354  break;
1355  case kArm64Float64Sub:
1356  __ Fsub(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1357  i.InputDoubleRegister(1));
1358  break;
1359  case kArm64Float64Mul:
1360  __ Fmul(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1361  i.InputDoubleRegister(1));
1362  break;
1363  case kArm64Float64Div:
1364  __ Fdiv(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1365  i.InputDoubleRegister(1));
1366  break;
1367  case kArm64Float64Mod: {
1368  // TODO(turbofan): implement directly.
1369  FrameScope scope(tasm(), StackFrame::MANUAL);
1370  DCHECK(d0.is(i.InputDoubleRegister(0)));
1371  DCHECK(d1.is(i.InputDoubleRegister(1)));
1372  DCHECK(d0.is(i.OutputDoubleRegister()));
1373  // TODO(turbofan): make sure this saves all relevant registers.
1374  __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2);
1375  break;
1376  }
1377  case kArm64Float32Max: {
1378  __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
1379  i.InputFloat32Register(1));
1380  break;
1381  }
1382  case kArm64Float64Max: {
1383  __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1384  i.InputDoubleRegister(1));
1385  break;
1386  }
1387  case kArm64Float32Min: {
1388  __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
1389  i.InputFloat32Register(1));
1390  break;
1391  }
1392  case kArm64Float64Min: {
1393  __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1394  i.InputDoubleRegister(1));
1395  break;
1396  }
1397  case kArm64Float64Abs:
1398  __ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1399  break;
1400  case kArm64Float64Neg:
1401  __ Fneg(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1402  break;
1403  case kArm64Float64Sqrt:
1404  __ Fsqrt(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1405  break;
1406  case kArm64Float32ToFloat64:
1407  __ Fcvt(i.OutputDoubleRegister(), i.InputDoubleRegister(0).S());
1408  break;
1409  case kArm64Float64ToFloat32:
1410  __ Fcvt(i.OutputDoubleRegister().S(), i.InputDoubleRegister(0));
1411  break;
1412  case kArm64Float32ToInt32:
1413  __ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
1414  // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
1415  // because INT32_MIN allows easier out-of-bounds detection.
1416  __ Cmn(i.OutputRegister32(), 1);
1417  __ Csinc(i.OutputRegister32(), i.OutputRegister32(), i.OutputRegister32(),
1418  vc);
1419  break;
1420  case kArm64Float64ToInt32:
1421  __ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
1422  break;
1423  case kArm64Float32ToUint32:
1424  __ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
1425  // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
1426  // because 0 allows easier out-of-bounds detection.
1427  __ Cmn(i.OutputRegister32(), 1);
1428  __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
1429  break;
1430  case kArm64Float64ToUint32:
1431  __ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
1432  break;
1433  case kArm64Float32ToInt64:
1434  __ Fcvtzs(i.OutputRegister64(), i.InputFloat32Register(0));
1435  if (i.OutputCount() > 1) {
1436  // Check for inputs below INT64_MIN and NaN.
1437  __ Fcmp(i.InputFloat32Register(0), static_cast<float>(INT64_MIN));
1438  // Check overflow.
1439  // -1 value is used to indicate a possible overflow which will occur
1440  // when subtracting (-1) from the provided INT64_MAX operand.
1441  // OutputRegister(1) is set to 0 if the input was out of range or NaN.
1442  __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1443  __ Cset(i.OutputRegister(1), vc);
1444  }
1445  break;
1446  case kArm64Float64ToInt64:
1447  __ Fcvtzs(i.OutputRegister(0), i.InputDoubleRegister(0));
1448  if (i.OutputCount() > 1) {
1449  // See kArm64Float32ToInt64 for a detailed description.
1450  __ Fcmp(i.InputDoubleRegister(0), static_cast<double>(INT64_MIN));
1451  __ Ccmp(i.OutputRegister(0), -1, VFlag, ge);
1452  __ Cset(i.OutputRegister(1), vc);
1453  }
1454  break;
1455  case kArm64Float32ToUint64:
1456  __ Fcvtzu(i.OutputRegister64(), i.InputFloat32Register(0));
1457  if (i.OutputCount() > 1) {
1458  // See kArm64Float32ToInt64 for a detailed description.
1459  __ Fcmp(i.InputFloat32Register(0), -1.0);
1460  __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1461  __ Cset(i.OutputRegister(1), ne);
1462  }
1463  break;
1464  case kArm64Float64ToUint64:
1465  __ Fcvtzu(i.OutputRegister64(), i.InputDoubleRegister(0));
1466  if (i.OutputCount() > 1) {
1467  // See kArm64Float32ToInt64 for a detailed description.
1468  __ Fcmp(i.InputDoubleRegister(0), -1.0);
1469  __ Ccmp(i.OutputRegister(0), -1, ZFlag, gt);
1470  __ Cset(i.OutputRegister(1), ne);
1471  }
1472  break;
1473  case kArm64Int32ToFloat32:
1474  __ Scvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1475  break;
1476  case kArm64Int32ToFloat64:
1477  __ Scvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1478  break;
1479  case kArm64Int64ToFloat32:
1480  __ Scvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1481  break;
1482  case kArm64Int64ToFloat64:
1483  __ Scvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1484  break;
1485  case kArm64Uint32ToFloat32:
1486  __ Ucvtf(i.OutputFloat32Register(), i.InputRegister32(0));
1487  break;
1488  case kArm64Uint32ToFloat64:
1489  __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister32(0));
1490  break;
1491  case kArm64Uint64ToFloat32:
1492  __ Ucvtf(i.OutputDoubleRegister().S(), i.InputRegister64(0));
1493  break;
1494  case kArm64Uint64ToFloat64:
1495  __ Ucvtf(i.OutputDoubleRegister(), i.InputRegister64(0));
1496  break;
1497  case kArm64Float64ExtractLowWord32:
1498  __ Fmov(i.OutputRegister32(), i.InputFloat32Register(0));
1499  break;
1500  case kArm64Float64ExtractHighWord32:
1501  __ Umov(i.OutputRegister32(), i.InputFloat64Register(0).V2S(), 1);
1502  break;
1503  case kArm64Float64InsertLowWord32:
1504  DCHECK(i.OutputFloat64Register().Is(i.InputFloat64Register(0)));
1505  __ Ins(i.OutputFloat64Register().V2S(), 0, i.InputRegister32(1));
1506  break;
1507  case kArm64Float64InsertHighWord32:
1508  DCHECK(i.OutputFloat64Register().Is(i.InputFloat64Register(0)));
1509  __ Ins(i.OutputFloat64Register().V2S(), 1, i.InputRegister32(1));
1510  break;
1511  case kArm64Float64MoveU64:
1512  __ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
1513  break;
1514  case kArm64Float64SilenceNaN:
1515  __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1516  break;
1517  case kArm64U64MoveFloat64:
1518  __ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
1519  break;
1520  case kArm64Ldrb:
1521  __ Ldrb(i.OutputRegister(), i.MemoryOperand());
1522  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1523  break;
1524  case kArm64Ldrsb:
1525  __ Ldrsb(i.OutputRegister(), i.MemoryOperand());
1526  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1527  break;
1528  case kArm64Strb:
1529  __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1530  break;
1531  case kArm64Ldrh:
1532  __ Ldrh(i.OutputRegister(), i.MemoryOperand());
1533  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1534  break;
1535  case kArm64Ldrsh:
1536  __ Ldrsh(i.OutputRegister(), i.MemoryOperand());
1537  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1538  break;
1539  case kArm64Strh:
1540  __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1541  break;
1542  case kArm64Ldrsw:
1543  __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
1544  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1545  break;
1546  case kArm64LdrW:
1547  __ Ldr(i.OutputRegister32(), i.MemoryOperand());
1548  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1549  break;
1550  case kArm64StrW:
1551  __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
1552  break;
1553  case kArm64Ldr:
1554  __ Ldr(i.OutputRegister(), i.MemoryOperand());
1555  EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
1556  break;
1557  case kArm64Str:
1558  __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
1559  break;
1560  case kArm64LdrS:
1561  __ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
1562  break;
1563  case kArm64StrS:
1564  __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
1565  break;
1566  case kArm64LdrD:
1567  __ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
1568  break;
1569  case kArm64StrD:
1570  __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
1571  break;
1572  case kArm64LdrQ:
1573  __ Ldr(i.OutputSimd128Register(), i.MemoryOperand());
1574  break;
1575  case kArm64StrQ:
1576  __ Str(i.InputSimd128Register(0), i.MemoryOperand(1));
1577  break;
1578  case kArm64DsbIsb:
1579  __ Dsb(FullSystem, BarrierAll);
1580  __ Isb();
1581  break;
1582  case kArchWordPoisonOnSpeculation:
1583  __ And(i.OutputRegister(0), i.InputRegister(0),
1584  Operand(kSpeculationPoisonRegister));
1585  break;
1586  case kWord32AtomicLoadInt8:
1587  ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1588  __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1589  break;
1590  case kWord32AtomicLoadUint8:
1591  case kArm64Word64AtomicLoadUint8:
1592  ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb, Register32);
1593  break;
1594  case kWord32AtomicLoadInt16:
1595  ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1596  __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1597  break;
1598  case kWord32AtomicLoadUint16:
1599  case kArm64Word64AtomicLoadUint16:
1600  ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh, Register32);
1601  break;
1602  case kWord32AtomicLoadWord32:
1603  case kArm64Word64AtomicLoadUint32:
1604  ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register32);
1605  break;
1606  case kArm64Word64AtomicLoadUint64:
1607  ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar, Register);
1608  break;
1609  case kWord32AtomicStoreWord8:
1610  case kArm64Word64AtomicStoreWord8:
1611  ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb, Register32);
1612  break;
1613  case kWord32AtomicStoreWord16:
1614  case kArm64Word64AtomicStoreWord16:
1615  ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh, Register32);
1616  break;
1617  case kWord32AtomicStoreWord32:
1618  case kArm64Word64AtomicStoreWord32:
1619  ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register32);
1620  break;
1621  case kArm64Word64AtomicStoreWord64:
1622  ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr, Register);
1623  break;
1624  case kWord32AtomicExchangeInt8:
1625  ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1626  __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1627  break;
1628  case kWord32AtomicExchangeUint8:
1629  case kArm64Word64AtomicExchangeUint8:
1630  ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb, Register32);
1631  break;
1632  case kWord32AtomicExchangeInt16:
1633  ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
1634  __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1635  break;
1636  case kWord32AtomicExchangeUint16:
1637  case kArm64Word64AtomicExchangeUint16:
1638  ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrh, stlxrh, Register32);
1639  break;
1640  case kWord32AtomicExchangeWord32:
1641  case kArm64Word64AtomicExchangeUint32:
1642  ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register32);
1643  break;
1644  case kArm64Word64AtomicExchangeUint64:
1645  ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxr, stlxr, Register);
1646  break;
1647  case kWord32AtomicCompareExchangeInt8:
1648  ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
1649  Register32);
1650  __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
1651  break;
1652  case kWord32AtomicCompareExchangeUint8:
1653  case kArm64Word64AtomicCompareExchangeUint8:
1654  ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrb, stlxrb, UXTB,
1655  Register32);
1656  break;
1657  case kWord32AtomicCompareExchangeInt16:
1658  ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
1659  Register32);
1660  __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
1661  break;
1662  case kWord32AtomicCompareExchangeUint16:
1663  case kArm64Word64AtomicCompareExchangeUint16:
1664  ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxrh, stlxrh, UXTH,
1665  Register32);
1666  break;
1667  case kWord32AtomicCompareExchangeWord32:
1668  case kArm64Word64AtomicCompareExchangeUint32:
1669  ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTW, Register32);
1670  break;
1671  case kArm64Word64AtomicCompareExchangeUint64:
1672  ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(ldaxr, stlxr, UXTX, Register);
1673  break;
1674 #define ATOMIC_BINOP_CASE(op, inst) \
1675  case kWord32Atomic##op##Int8: \
1676  ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
1677  __ Sxtb(i.OutputRegister(0), i.OutputRegister(0)); \
1678  break; \
1679  case kWord32Atomic##op##Uint8: \
1680  case kArm64Word64Atomic##op##Uint8: \
1681  ASSEMBLE_ATOMIC_BINOP(ldaxrb, stlxrb, inst, Register32); \
1682  break; \
1683  case kWord32Atomic##op##Int16: \
1684  ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
1685  __ Sxth(i.OutputRegister(0), i.OutputRegister(0)); \
1686  break; \
1687  case kWord32Atomic##op##Uint16: \
1688  case kArm64Word64Atomic##op##Uint16: \
1689  ASSEMBLE_ATOMIC_BINOP(ldaxrh, stlxrh, inst, Register32); \
1690  break; \
1691  case kWord32Atomic##op##Word32: \
1692  case kArm64Word64Atomic##op##Uint32: \
1693  ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register32); \
1694  break; \
1695  case kArm64Word64Atomic##op##Uint64: \
1696  ASSEMBLE_ATOMIC_BINOP(ldaxr, stlxr, inst, Register); \
1697  break;
1698  ATOMIC_BINOP_CASE(Add, Add)
1699  ATOMIC_BINOP_CASE(Sub, Sub)
1700  ATOMIC_BINOP_CASE(And, And)
1701  ATOMIC_BINOP_CASE(Or, Orr)
1702  ATOMIC_BINOP_CASE(Xor, Eor)
1703 #undef ATOMIC_BINOP_CASE
1704 #undef ASSEMBLE_SHIFT
1705 #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
1706 #undef ASSEMBLE_ATOMIC_STORE_INTEGER
1707 #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
1708 #undef ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER
1709 #undef ASSEMBLE_ATOMIC_BINOP
1710 #undef ASSEMBLE_IEEE754_BINOP
1711 #undef ASSEMBLE_IEEE754_UNOP
1712 
1713 #define SIMD_UNOP_CASE(Op, Instr, FORMAT) \
1714  case Op: \
1715  __ Instr(i.OutputSimd128Register().V##FORMAT(), \
1716  i.InputSimd128Register(0).V##FORMAT()); \
1717  break;
1718 #define SIMD_WIDENING_UNOP_CASE(Op, Instr, WIDE, NARROW) \
1719  case Op: \
1720  __ Instr(i.OutputSimd128Register().V##WIDE(), \
1721  i.InputSimd128Register(0).V##NARROW()); \
1722  break;
1723 #define SIMD_BINOP_CASE(Op, Instr, FORMAT) \
1724  case Op: \
1725  __ Instr(i.OutputSimd128Register().V##FORMAT(), \
1726  i.InputSimd128Register(0).V##FORMAT(), \
1727  i.InputSimd128Register(1).V##FORMAT()); \
1728  break;
1729 
1730  case kArm64F32x4Splat: {
1731  __ Dup(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).S(), 0);
1732  break;
1733  }
1734  case kArm64F32x4ExtractLane: {
1735  __ Mov(i.OutputSimd128Register().S(), i.InputSimd128Register(0).V4S(),
1736  i.InputInt8(1));
1737  break;
1738  }
1739  case kArm64F32x4ReplaceLane: {
1740  VRegister dst = i.OutputSimd128Register().V4S(),
1741  src1 = i.InputSimd128Register(0).V4S();
1742  if (!dst.is(src1)) {
1743  __ Mov(dst, src1);
1744  }
1745  __ Mov(dst, i.InputInt8(1), i.InputSimd128Register(2).V4S(), 0);
1746  break;
1747  }
1748  SIMD_UNOP_CASE(kArm64F32x4SConvertI32x4, Scvtf, 4S);
1749  SIMD_UNOP_CASE(kArm64F32x4UConvertI32x4, Ucvtf, 4S);
1750  SIMD_UNOP_CASE(kArm64F32x4Abs, Fabs, 4S);
1751  SIMD_UNOP_CASE(kArm64F32x4Neg, Fneg, 4S);
1752  SIMD_UNOP_CASE(kArm64F32x4RecipApprox, Frecpe, 4S);
1753  SIMD_UNOP_CASE(kArm64F32x4RecipSqrtApprox, Frsqrte, 4S);
1754  SIMD_BINOP_CASE(kArm64F32x4Add, Fadd, 4S);
1755  SIMD_BINOP_CASE(kArm64F32x4AddHoriz, Faddp, 4S);
1756  SIMD_BINOP_CASE(kArm64F32x4Sub, Fsub, 4S);
1757  SIMD_BINOP_CASE(kArm64F32x4Mul, Fmul, 4S);
1758  SIMD_BINOP_CASE(kArm64F32x4Min, Fmin, 4S);
1759  SIMD_BINOP_CASE(kArm64F32x4Max, Fmax, 4S);
1760  SIMD_BINOP_CASE(kArm64F32x4Eq, Fcmeq, 4S);
1761  case kArm64F32x4Ne: {
1762  VRegister dst = i.OutputSimd128Register().V4S();
1763  __ Fcmeq(dst, i.InputSimd128Register(0).V4S(),
1764  i.InputSimd128Register(1).V4S());
1765  __ Mvn(dst, dst);
1766  break;
1767  }
1768  case kArm64F32x4Lt: {
1769  __ Fcmgt(i.OutputSimd128Register().V4S(), i.InputSimd128Register(1).V4S(),
1770  i.InputSimd128Register(0).V4S());
1771  break;
1772  }
1773  case kArm64F32x4Le: {
1774  __ Fcmge(i.OutputSimd128Register().V4S(), i.InputSimd128Register(1).V4S(),
1775  i.InputSimd128Register(0).V4S());
1776  break;
1777  }
1778  case kArm64I32x4Splat: {
1779  __ Dup(i.OutputSimd128Register().V4S(), i.InputRegister32(0));
1780  break;
1781  }
1782  case kArm64I32x4ExtractLane: {
1783  __ Mov(i.OutputRegister32(), i.InputSimd128Register(0).V4S(),
1784  i.InputInt8(1));
1785  break;
1786  }
1787  case kArm64I32x4ReplaceLane: {
1788  VRegister dst = i.OutputSimd128Register().V4S(),
1789  src1 = i.InputSimd128Register(0).V4S();
1790  if (!dst.is(src1)) {
1791  __ Mov(dst, src1);
1792  }
1793  __ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
1794  break;
1795  }
1796  SIMD_UNOP_CASE(kArm64I32x4SConvertF32x4, Fcvtzs, 4S);
1797  SIMD_WIDENING_UNOP_CASE(kArm64I32x4SConvertI16x8Low, Sxtl, 4S, 4H);
1798  SIMD_WIDENING_UNOP_CASE(kArm64I32x4SConvertI16x8High, Sxtl2, 4S, 8H);
1799  SIMD_UNOP_CASE(kArm64I32x4Neg, Neg, 4S);
1800  case kArm64I32x4Shl: {
1801  __ Shl(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).V4S(),
1802  i.InputInt5(1));
1803  break;
1804  }
1805  case kArm64I32x4ShrS: {
1806  __ Sshr(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).V4S(),
1807  i.InputInt5(1));
1808  break;
1809  }
1810  SIMD_BINOP_CASE(kArm64I32x4Add, Add, 4S);
1811  SIMD_BINOP_CASE(kArm64I32x4AddHoriz, Addp, 4S);
1812  SIMD_BINOP_CASE(kArm64I32x4Sub, Sub, 4S);
1813  SIMD_BINOP_CASE(kArm64I32x4Mul, Mul, 4S);
1814  SIMD_BINOP_CASE(kArm64I32x4MinS, Smin, 4S);
1815  SIMD_BINOP_CASE(kArm64I32x4MaxS, Smax, 4S);
1816  SIMD_BINOP_CASE(kArm64I32x4Eq, Cmeq, 4S);
1817  case kArm64I32x4Ne: {
1818  VRegister dst = i.OutputSimd128Register().V4S();
1819  __ Cmeq(dst, i.InputSimd128Register(0).V4S(),
1820  i.InputSimd128Register(1).V4S());
1821  __ Mvn(dst, dst);
1822  break;
1823  }
1824  SIMD_BINOP_CASE(kArm64I32x4GtS, Cmgt, 4S);
1825  SIMD_BINOP_CASE(kArm64I32x4GeS, Cmge, 4S);
1826  SIMD_UNOP_CASE(kArm64I32x4UConvertF32x4, Fcvtzu, 4S);
1827  SIMD_WIDENING_UNOP_CASE(kArm64I32x4UConvertI16x8Low, Uxtl, 4S, 4H);
1828  SIMD_WIDENING_UNOP_CASE(kArm64I32x4UConvertI16x8High, Uxtl2, 4S, 8H);
1829  case kArm64I32x4ShrU: {
1830  __ Ushr(i.OutputSimd128Register().V4S(), i.InputSimd128Register(0).V4S(),
1831  i.InputInt5(1));
1832  break;
1833  }
1834  SIMD_BINOP_CASE(kArm64I32x4MinU, Umin, 4S);
1835  SIMD_BINOP_CASE(kArm64I32x4MaxU, Umax, 4S);
1836  SIMD_BINOP_CASE(kArm64I32x4GtU, Cmhi, 4S);
1837  SIMD_BINOP_CASE(kArm64I32x4GeU, Cmhs, 4S);
1838  case kArm64I16x8Splat: {
1839  __ Dup(i.OutputSimd128Register().V8H(), i.InputRegister32(0));
1840  break;
1841  }
1842  case kArm64I16x8ExtractLane: {
1843  __ Smov(i.OutputRegister32(), i.InputSimd128Register(0).V8H(),
1844  i.InputInt8(1));
1845  break;
1846  }
1847  case kArm64I16x8ReplaceLane: {
1848  VRegister dst = i.OutputSimd128Register().V8H(),
1849  src1 = i.InputSimd128Register(0).V8H();
1850  if (!dst.is(src1)) {
1851  __ Mov(dst, src1);
1852  }
1853  __ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
1854  break;
1855  }
1856  SIMD_WIDENING_UNOP_CASE(kArm64I16x8SConvertI8x16Low, Sxtl, 8H, 8B);
1857  SIMD_WIDENING_UNOP_CASE(kArm64I16x8SConvertI8x16High, Sxtl2, 8H, 16B);
1858  SIMD_UNOP_CASE(kArm64I16x8Neg, Neg, 8H);
1859  case kArm64I16x8Shl: {
1860  __ Shl(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8H(),
1861  i.InputInt5(1));
1862  break;
1863  }
1864  case kArm64I16x8ShrS: {
1865  __ Sshr(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8H(),
1866  i.InputInt5(1));
1867  break;
1868  }
1869  case kArm64I16x8SConvertI32x4: {
1870  VRegister dst = i.OutputSimd128Register(),
1871  src0 = i.InputSimd128Register(0),
1872  src1 = i.InputSimd128Register(1);
1873  UseScratchRegisterScope scope(tasm());
1874  VRegister temp = scope.AcquireV(kFormat4S);
1875  if (dst.is(src1)) {
1876  __ Mov(temp, src1.V4S());
1877  src1 = temp;
1878  }
1879  __ Sqxtn(dst.V4H(), src0.V4S());
1880  __ Sqxtn2(dst.V8H(), src1.V4S());
1881  break;
1882  }
1883  SIMD_BINOP_CASE(kArm64I16x8Add, Add, 8H);
1884  SIMD_BINOP_CASE(kArm64I16x8AddSaturateS, Sqadd, 8H);
1885  SIMD_BINOP_CASE(kArm64I16x8AddHoriz, Addp, 8H);
1886  SIMD_BINOP_CASE(kArm64I16x8Sub, Sub, 8H);
1887  SIMD_BINOP_CASE(kArm64I16x8SubSaturateS, Sqsub, 8H);
1888  SIMD_BINOP_CASE(kArm64I16x8Mul, Mul, 8H);
1889  SIMD_BINOP_CASE(kArm64I16x8MinS, Smin, 8H);
1890  SIMD_BINOP_CASE(kArm64I16x8MaxS, Smax, 8H);
1891  SIMD_BINOP_CASE(kArm64I16x8Eq, Cmeq, 8H);
1892  case kArm64I16x8Ne: {
1893  VRegister dst = i.OutputSimd128Register().V8H();
1894  __ Cmeq(dst, i.InputSimd128Register(0).V8H(),
1895  i.InputSimd128Register(1).V8H());
1896  __ Mvn(dst, dst);
1897  break;
1898  }
1899  SIMD_BINOP_CASE(kArm64I16x8GtS, Cmgt, 8H);
1900  SIMD_BINOP_CASE(kArm64I16x8GeS, Cmge, 8H);
1901  case kArm64I16x8UConvertI8x16Low: {
1902  __ Uxtl(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8B());
1903  break;
1904  }
1905  case kArm64I16x8UConvertI8x16High: {
1906  __ Uxtl2(i.OutputSimd128Register().V8H(),
1907  i.InputSimd128Register(0).V16B());
1908  break;
1909  }
1910  case kArm64I16x8ShrU: {
1911  __ Ushr(i.OutputSimd128Register().V8H(), i.InputSimd128Register(0).V8H(),
1912  i.InputInt5(1));
1913  break;
1914  }
1915  case kArm64I16x8UConvertI32x4: {
1916  VRegister dst = i.OutputSimd128Register(),
1917  src0 = i.InputSimd128Register(0),
1918  src1 = i.InputSimd128Register(1);
1919  UseScratchRegisterScope scope(tasm());
1920  VRegister temp = scope.AcquireV(kFormat4S);
1921  if (dst.is(src1)) {
1922  __ Mov(temp, src1.V4S());
1923  src1 = temp;
1924  }
1925  __ Uqxtn(dst.V4H(), src0.V4S());
1926  __ Uqxtn2(dst.V8H(), src1.V4S());
1927  break;
1928  }
1929  SIMD_BINOP_CASE(kArm64I16x8AddSaturateU, Uqadd, 8H);
1930  SIMD_BINOP_CASE(kArm64I16x8SubSaturateU, Uqsub, 8H);
1931  SIMD_BINOP_CASE(kArm64I16x8MinU, Umin, 8H);
1932  SIMD_BINOP_CASE(kArm64I16x8MaxU, Umax, 8H);
1933  SIMD_BINOP_CASE(kArm64I16x8GtU, Cmhi, 8H);
1934  SIMD_BINOP_CASE(kArm64I16x8GeU, Cmhs, 8H);
1935  case kArm64I8x16Splat: {
1936  __ Dup(i.OutputSimd128Register().V16B(), i.InputRegister32(0));
1937  break;
1938  }
1939  case kArm64I8x16ExtractLane: {
1940  __ Smov(i.OutputRegister32(), i.InputSimd128Register(0).V16B(),
1941  i.InputInt8(1));
1942  break;
1943  }
1944  case kArm64I8x16ReplaceLane: {
1945  VRegister dst = i.OutputSimd128Register().V16B(),
1946  src1 = i.InputSimd128Register(0).V16B();
1947  if (!dst.is(src1)) {
1948  __ Mov(dst, src1);
1949  }
1950  __ Mov(dst, i.InputInt8(1), i.InputRegister32(2));
1951  break;
1952  }
1953  SIMD_UNOP_CASE(kArm64I8x16Neg, Neg, 16B);
1954  case kArm64I8x16Shl: {
1955  __ Shl(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
1956  i.InputInt5(1));
1957  break;
1958  }
1959  case kArm64I8x16ShrS: {
1960  __ Sshr(i.OutputSimd128Register().V16B(),
1961  i.InputSimd128Register(0).V16B(), i.InputInt5(1));
1962  break;
1963  }
1964  case kArm64I8x16SConvertI16x8: {
1965  VRegister dst = i.OutputSimd128Register(),
1966  src0 = i.InputSimd128Register(0),
1967  src1 = i.InputSimd128Register(1);
1968  UseScratchRegisterScope scope(tasm());
1969  VRegister temp = scope.AcquireV(kFormat8H);
1970  if (dst.is(src1)) {
1971  __ Mov(temp, src1.V8H());
1972  src1 = temp;
1973  }
1974  __ Sqxtn(dst.V8B(), src0.V8H());
1975  __ Sqxtn2(dst.V16B(), src1.V8H());
1976  break;
1977  }
1978  SIMD_BINOP_CASE(kArm64I8x16Add, Add, 16B);
1979  SIMD_BINOP_CASE(kArm64I8x16AddSaturateS, Sqadd, 16B);
1980  SIMD_BINOP_CASE(kArm64I8x16Sub, Sub, 16B);
1981  SIMD_BINOP_CASE(kArm64I8x16SubSaturateS, Sqsub, 16B);
1982  SIMD_BINOP_CASE(kArm64I8x16Mul, Mul, 16B);
1983  SIMD_BINOP_CASE(kArm64I8x16MinS, Smin, 16B);
1984  SIMD_BINOP_CASE(kArm64I8x16MaxS, Smax, 16B);
1985  SIMD_BINOP_CASE(kArm64I8x16Eq, Cmeq, 16B);
1986  case kArm64I8x16Ne: {
1987  VRegister dst = i.OutputSimd128Register().V16B();
1988  __ Cmeq(dst, i.InputSimd128Register(0).V16B(),
1989  i.InputSimd128Register(1).V16B());
1990  __ Mvn(dst, dst);
1991  break;
1992  }
1993  SIMD_BINOP_CASE(kArm64I8x16GtS, Cmgt, 16B);
1994  SIMD_BINOP_CASE(kArm64I8x16GeS, Cmge, 16B);
1995  case kArm64I8x16ShrU: {
1996  __ Ushr(i.OutputSimd128Register().V16B(),
1997  i.InputSimd128Register(0).V16B(), i.InputInt5(1));
1998  break;
1999  }
2000  case kArm64I8x16UConvertI16x8: {
2001  VRegister dst = i.OutputSimd128Register(),
2002  src0 = i.InputSimd128Register(0),
2003  src1 = i.InputSimd128Register(1);
2004  UseScratchRegisterScope scope(tasm());
2005  VRegister temp = scope.AcquireV(kFormat8H);
2006  if (dst.is(src1)) {
2007  __ Mov(temp, src1.V8H());
2008  src1 = temp;
2009  }
2010  __ Uqxtn(dst.V8B(), src0.V8H());
2011  __ Uqxtn2(dst.V16B(), src1.V8H());
2012  break;
2013  }
2014  SIMD_BINOP_CASE(kArm64I8x16AddSaturateU, Uqadd, 16B);
2015  SIMD_BINOP_CASE(kArm64I8x16SubSaturateU, Uqsub, 16B);
2016  SIMD_BINOP_CASE(kArm64I8x16MinU, Umin, 16B);
2017  SIMD_BINOP_CASE(kArm64I8x16MaxU, Umax, 16B);
2018  SIMD_BINOP_CASE(kArm64I8x16GtU, Cmhi, 16B);
2019  SIMD_BINOP_CASE(kArm64I8x16GeU, Cmhs, 16B);
2020  case kArm64S128Zero: {
2021  __ Movi(i.OutputSimd128Register().V16B(), 0);
2022  break;
2023  }
2024  SIMD_BINOP_CASE(kArm64S128And, And, 16B);
2025  SIMD_BINOP_CASE(kArm64S128Or, Orr, 16B);
2026  SIMD_BINOP_CASE(kArm64S128Xor, Eor, 16B);
2027  SIMD_UNOP_CASE(kArm64S128Not, Mvn, 16B);
2028  case kArm64S128Dup: {
2029  VRegister dst = i.OutputSimd128Register(),
2030  src = i.InputSimd128Register(0);
2031  int lanes = i.InputInt32(1);
2032  int index = i.InputInt32(2);
2033  switch (lanes) {
2034  case 4:
2035  __ Dup(dst.V4S(), src.V4S(), index);
2036  break;
2037  case 8:
2038  __ Dup(dst.V8H(), src.V8H(), index);
2039  break;
2040  case 16:
2041  __ Dup(dst.V16B(), src.V16B(), index);
2042  break;
2043  default:
2044  UNREACHABLE();
2045  break;
2046  }
2047  break;
2048  }
2049  case kArm64S128Select: {
2050  VRegister dst = i.OutputSimd128Register().V16B();
2051  DCHECK(dst.is(i.InputSimd128Register(0).V16B()));
2052  __ Bsl(dst, i.InputSimd128Register(1).V16B(),
2053  i.InputSimd128Register(2).V16B());
2054  break;
2055  }
2056  case kArm64S32x4Shuffle: {
2057  Simd128Register dst = i.OutputSimd128Register().V4S(),
2058  src0 = i.InputSimd128Register(0).V4S(),
2059  src1 = i.InputSimd128Register(1).V4S();
2060  // Check for in-place shuffles.
2061  // If dst == src0 == src1, then the shuffle is unary and we only use src0.
2062  UseScratchRegisterScope scope(tasm());
2063  VRegister temp = scope.AcquireV(kFormat4S);
2064  if (dst.is(src0)) {
2065  __ Mov(temp, src0);
2066  src0 = temp;
2067  } else if (dst.is(src1)) {
2068  __ Mov(temp, src1);
2069  src1 = temp;
2070  }
2071  // Perform shuffle as a vmov per lane.
2072  int32_t shuffle = i.InputInt32(2);
2073  for (int i = 0; i < 4; i++) {
2074  VRegister src = src0;
2075  int lane = shuffle & 0x7;
2076  if (lane >= 4) {
2077  src = src1;
2078  lane &= 0x3;
2079  }
2080  __ Mov(dst, i, src, lane);
2081  shuffle >>= 8;
2082  }
2083  break;
2084  }
2085  SIMD_BINOP_CASE(kArm64S32x4ZipLeft, Zip1, 4S);
2086  SIMD_BINOP_CASE(kArm64S32x4ZipRight, Zip2, 4S);
2087  SIMD_BINOP_CASE(kArm64S32x4UnzipLeft, Uzp1, 4S);
2088  SIMD_BINOP_CASE(kArm64S32x4UnzipRight, Uzp2, 4S);
2089  SIMD_BINOP_CASE(kArm64S32x4TransposeLeft, Trn1, 4S);
2090  SIMD_BINOP_CASE(kArm64S32x4TransposeRight, Trn2, 4S);
2091  SIMD_BINOP_CASE(kArm64S16x8ZipLeft, Zip1, 8H);
2092  SIMD_BINOP_CASE(kArm64S16x8ZipRight, Zip2, 8H);
2093  SIMD_BINOP_CASE(kArm64S16x8UnzipLeft, Uzp1, 8H);
2094  SIMD_BINOP_CASE(kArm64S16x8UnzipRight, Uzp2, 8H);
2095  SIMD_BINOP_CASE(kArm64S16x8TransposeLeft, Trn1, 8H);
2096  SIMD_BINOP_CASE(kArm64S16x8TransposeRight, Trn2, 8H);
2097  SIMD_BINOP_CASE(kArm64S8x16ZipLeft, Zip1, 16B);
2098  SIMD_BINOP_CASE(kArm64S8x16ZipRight, Zip2, 16B);
2099  SIMD_BINOP_CASE(kArm64S8x16UnzipLeft, Uzp1, 16B);
2100  SIMD_BINOP_CASE(kArm64S8x16UnzipRight, Uzp2, 16B);
2101  SIMD_BINOP_CASE(kArm64S8x16TransposeLeft, Trn1, 16B);
2102  SIMD_BINOP_CASE(kArm64S8x16TransposeRight, Trn2, 16B);
2103  case kArm64S8x16Concat: {
2104  __ Ext(i.OutputSimd128Register().V16B(), i.InputSimd128Register(0).V16B(),
2105  i.InputSimd128Register(1).V16B(), i.InputInt4(2));
2106  break;
2107  }
2108  case kArm64S8x16Shuffle: {
2109  Simd128Register dst = i.OutputSimd128Register().V16B(),
2110  src0 = i.InputSimd128Register(0).V16B(),
2111  src1 = i.InputSimd128Register(1).V16B();
2112  // Unary shuffle table is in src0, binary shuffle table is in src0, src1,
2113  // which must be consecutive.
2114  int64_t mask = 0;
2115  if (src0.is(src1)) {
2116  mask = 0x0F0F0F0F;
2117  } else {
2118  mask = 0x1F1F1F1F;
2119  DCHECK(AreConsecutive(src0, src1));
2120  }
2121  int64_t imm1 =
2122  (i.InputInt32(2) & mask) | ((i.InputInt32(3) & mask) << 32);
2123  int64_t imm2 =
2124  (i.InputInt32(4) & mask) | ((i.InputInt32(5) & mask) << 32);
2125  UseScratchRegisterScope scope(tasm());
2126  VRegister temp = scope.AcquireV(kFormat16B);
2127  __ Movi(temp, imm2, imm1);
2128 
2129  if (src0.is(src1)) {
2130  __ Tbl(dst, src0, temp.V16B());
2131  } else {
2132  __ Tbl(dst, src0, src1, temp.V16B());
2133  }
2134  break;
2135  }
2136  SIMD_UNOP_CASE(kArm64S32x2Reverse, Rev64, 4S);
2137  SIMD_UNOP_CASE(kArm64S16x4Reverse, Rev64, 8H);
2138  SIMD_UNOP_CASE(kArm64S16x2Reverse, Rev32, 8H);
2139  SIMD_UNOP_CASE(kArm64S8x8Reverse, Rev64, 16B);
2140  SIMD_UNOP_CASE(kArm64S8x4Reverse, Rev32, 16B);
2141  SIMD_UNOP_CASE(kArm64S8x2Reverse, Rev16, 16B);
2142 
2143 #define SIMD_REDUCE_OP_CASE(Op, Instr, format, FORMAT) \
2144  case Op: { \
2145  UseScratchRegisterScope scope(tasm()); \
2146  VRegister temp = scope.AcquireV(format); \
2147  __ Instr(temp, i.InputSimd128Register(0).V##FORMAT()); \
2148  __ Umov(i.OutputRegister32(), temp, 0); \
2149  break; \
2150  }
2151  SIMD_REDUCE_OP_CASE(kArm64S1x4AnyTrue, Umaxv, kFormatS, 4S);
2152  SIMD_REDUCE_OP_CASE(kArm64S1x4AllTrue, Uminv, kFormatS, 4S);
2153  SIMD_REDUCE_OP_CASE(kArm64S1x8AnyTrue, Umaxv, kFormatH, 8H);
2154  SIMD_REDUCE_OP_CASE(kArm64S1x8AllTrue, Uminv, kFormatH, 8H);
2155  SIMD_REDUCE_OP_CASE(kArm64S1x16AnyTrue, Umaxv, kFormatB, 16B);
2156  SIMD_REDUCE_OP_CASE(kArm64S1x16AllTrue, Uminv, kFormatB, 16B);
2157  }
2158  return kSuccess;
2159 } // NOLINT(readability/fn_size)
2160 
2161 #undef SIMD_UNOP_CASE
2162 #undef SIMD_WIDENING_UNOP_CASE
2163 #undef SIMD_BINOP_CASE
2164 #undef SIMD_REDUCE_OP_CASE
2165 
2166 // Assemble branches after this instruction.
2167 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2168  Arm64OperandConverter i(this, instr);
2169  Label* tlabel = branch->true_label;
2170  Label* flabel = branch->false_label;
2171  FlagsCondition condition = branch->condition;
2172  ArchOpcode opcode = instr->arch_opcode();
2173 
2174  if (opcode == kArm64CompareAndBranch32) {
2175  DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2176  switch (condition) {
2177  case kEqual:
2178  __ Cbz(i.InputRegister32(0), tlabel);
2179  break;
2180  case kNotEqual:
2181  __ Cbnz(i.InputRegister32(0), tlabel);
2182  break;
2183  default:
2184  UNREACHABLE();
2185  }
2186  } else if (opcode == kArm64CompareAndBranch) {
2187  DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2188  switch (condition) {
2189  case kEqual:
2190  __ Cbz(i.InputRegister64(0), tlabel);
2191  break;
2192  case kNotEqual:
2193  __ Cbnz(i.InputRegister64(0), tlabel);
2194  break;
2195  default:
2196  UNREACHABLE();
2197  }
2198  } else if (opcode == kArm64TestAndBranch32) {
2199  DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2200  switch (condition) {
2201  case kEqual:
2202  __ Tbz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2203  break;
2204  case kNotEqual:
2205  __ Tbnz(i.InputRegister32(0), i.InputInt5(1), tlabel);
2206  break;
2207  default:
2208  UNREACHABLE();
2209  }
2210  } else if (opcode == kArm64TestAndBranch) {
2211  DCHECK(FlagsModeField::decode(instr->opcode()) != kFlags_branch_and_poison);
2212  switch (condition) {
2213  case kEqual:
2214  __ Tbz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2215  break;
2216  case kNotEqual:
2217  __ Tbnz(i.InputRegister64(0), i.InputInt6(1), tlabel);
2218  break;
2219  default:
2220  UNREACHABLE();
2221  }
2222  } else {
2223  Condition cc = FlagsConditionToCondition(condition);
2224  __ B(cc, tlabel);
2225  }
2226  if (!branch->fallthru) __ B(flabel); // no fallthru to flabel.
2227 }
2228 
2229 void CodeGenerator::AssembleBranchPoisoning(FlagsCondition condition,
2230  Instruction* instr) {
2231  // TODO(jarin) Handle float comparisons (kUnordered[Not]Equal).
2232  if (condition == kUnorderedEqual || condition == kUnorderedNotEqual) {
2233  return;
2234  }
2235 
2236  condition = NegateFlagsCondition(condition);
2237  __ CmovX(kSpeculationPoisonRegister, xzr,
2238  FlagsConditionToCondition(condition));
2239  __ Csdb();
2240 }
2241 
2242 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2243  BranchInfo* branch) {
2244  AssembleArchBranch(instr, branch);
2245 }
2246 
2247 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2248  if (!IsNextInAssemblyOrder(target)) __ B(GetLabel(target));
2249 }
2250 
2251 void CodeGenerator::AssembleArchTrap(Instruction* instr,
2252  FlagsCondition condition) {
2253  class OutOfLineTrap final : public OutOfLineCode {
2254  public:
2255  OutOfLineTrap(CodeGenerator* gen, Instruction* instr)
2256  : OutOfLineCode(gen), instr_(instr), gen_(gen) {}
2257  void Generate() final {
2258  Arm64OperandConverter i(gen_, instr_);
2259  TrapId trap_id =
2260  static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
2261  GenerateCallToTrap(trap_id);
2262  }
2263 
2264  private:
2265  void GenerateCallToTrap(TrapId trap_id) {
2266  if (trap_id == TrapId::kInvalid) {
2267  // We cannot test calls to the runtime in cctest/test-run-wasm.
2268  // Therefore we emit a call to C here instead of a call to the runtime.
2269  __ CallCFunction(
2270  ExternalReference::wasm_call_trap_callback_for_testing(), 0);
2271  __ LeaveFrame(StackFrame::WASM_COMPILED);
2272  auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
2273  int pop_count =
2274  static_cast<int>(call_descriptor->StackParameterCount());
2275  pop_count += (pop_count & 1); // align
2276  __ Drop(pop_count);
2277  __ Ret();
2278  } else {
2279  gen_->AssembleSourcePosition(instr_);
2280  // A direct call to a wasm runtime stub defined in this module.
2281  // Just encode the stub index. This will be patched when the code
2282  // is added to the native module and copied into wasm code space.
2283  __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
2284  ReferenceMap* reference_map =
2285  new (gen_->zone()) ReferenceMap(gen_->zone());
2286  gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2287  Safepoint::kNoLazyDeopt);
2288  if (FLAG_debug_code) {
2289  // The trap code should never return.
2290  __ Brk(0);
2291  }
2292  }
2293  }
2294  Instruction* instr_;
2295  CodeGenerator* gen_;
2296  };
2297  auto ool = new (zone()) OutOfLineTrap(this, instr);
2298  Label* tlabel = ool->entry();
2299  Condition cc = FlagsConditionToCondition(condition);
2300  __ B(cc, tlabel);
2301 }
2302 
2303 // Assemble boolean materializations after this instruction.
2304 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2305  FlagsCondition condition) {
2306  Arm64OperandConverter i(this, instr);
2307 
2308  // Materialize a full 64-bit 1 or 0 value. The result register is always the
2309  // last output of the instruction.
2310  DCHECK_NE(0u, instr->OutputCount());
2311  Register reg = i.OutputRegister(instr->OutputCount() - 1);
2312  Condition cc = FlagsConditionToCondition(condition);
2313  __ Cset(reg, cc);
2314 }
2315 
2316 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
2317  Arm64OperandConverter i(this, instr);
2318  Register input = i.InputRegister32(0);
2319  std::vector<std::pair<int32_t, Label*>> cases;
2320  for (size_t index = 2; index < instr->InputCount(); index += 2) {
2321  cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
2322  }
2323  AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
2324  cases.data() + cases.size());
2325 }
2326 
2327 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2328  Arm64OperandConverter i(this, instr);
2329  Register input = i.InputRegister32(0);
2330  for (size_t index = 2; index < instr->InputCount(); index += 2) {
2331  __ Cmp(input, i.InputInt32(index + 0));
2332  __ B(eq, GetLabel(i.InputRpo(index + 1)));
2333  }
2334  AssembleArchJump(i.InputRpo(1));
2335 }
2336 
2337 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2338  Arm64OperandConverter i(this, instr);
2339  UseScratchRegisterScope scope(tasm());
2340  Register input = i.InputRegister32(0);
2341  Register temp = scope.AcquireX();
2342  size_t const case_count = instr->InputCount() - 2;
2343  Label table;
2344  __ Cmp(input, case_count);
2345  __ B(hs, GetLabel(i.InputRpo(1)));
2346  __ Adr(temp, &table);
2347  __ Add(temp, temp, Operand(input, UXTW, 2));
2348  __ Br(temp);
2349  __ StartBlockPools();
2350  __ Bind(&table);
2351  for (size_t index = 0; index < case_count; ++index) {
2352  __ B(GetLabel(i.InputRpo(index + 2)));
2353  }
2354  __ EndBlockPools();
2355 }
2356 
2357 void CodeGenerator::FinishFrame(Frame* frame) {
2358  frame->AlignFrame(16);
2359  auto call_descriptor = linkage()->GetIncomingDescriptor();
2360 
2361  // Save FP registers.
2362  CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2363  call_descriptor->CalleeSavedFPRegisters());
2364  int saved_count = saves_fp.Count();
2365  if (saved_count != 0) {
2366  DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedV().list());
2367  DCHECK_EQ(saved_count % 2, 0);
2368  frame->AllocateSavedCalleeRegisterSlots(saved_count *
2369  (kDoubleSize / kPointerSize));
2370  }
2371 
2372  CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2373  call_descriptor->CalleeSavedRegisters());
2374  saved_count = saves.Count();
2375  if (saved_count != 0) {
2376  DCHECK_EQ(saved_count % 2, 0);
2377  frame->AllocateSavedCalleeRegisterSlots(saved_count);
2378  }
2379 }
2380 
2381 void CodeGenerator::AssembleConstructFrame() {
2382  auto call_descriptor = linkage()->GetIncomingDescriptor();
2383  __ AssertSpAligned();
2384 
2385  // The frame has been previously padded in CodeGenerator::FinishFrame().
2386  DCHECK_EQ(frame()->GetTotalFrameSlotCount() % 2, 0);
2387  int shrink_slots = frame()->GetTotalFrameSlotCount() -
2388  call_descriptor->CalculateFixedFrameSize();
2389 
2390  CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2391  call_descriptor->CalleeSavedRegisters());
2392  CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2393  call_descriptor->CalleeSavedFPRegisters());
2394  // The number of slots for returns has to be even to ensure the correct stack
2395  // alignment.
2396  const int returns = RoundUp(frame()->GetReturnSlotCount(), 2);
2397 
2398  if (frame_access_state()->has_frame()) {
2399  // Link the frame
2400  if (call_descriptor->IsJSFunctionCall()) {
2401  __ Prologue();
2402  } else {
2403  __ Push(lr, fp);
2404  __ Mov(fp, sp);
2405  }
2406  unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
2407 
2408  // Create OSR entry if applicable
2409  if (info()->is_osr()) {
2410  // TurboFan OSR-compiled functions cannot be entered directly.
2411  __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
2412 
2413  // Unoptimized code jumps directly to this entrypoint while the
2414  // unoptimized frame is still on the stack. Optimized code uses OSR values
2415  // directly from the unoptimized frame. Thus, all that needs to be done is
2416  // to allocate the remaining stack slots.
2417  if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
2418  osr_pc_offset_ = __ pc_offset();
2419  shrink_slots -= osr_helper()->UnoptimizedFrameSlots();
2420  ResetSpeculationPoison();
2421  }
2422 
2423  if (info()->IsWasm() && shrink_slots > 128) {
2424  // For WebAssembly functions with big frames we have to do the stack
2425  // overflow check before we construct the frame. Otherwise we may not
2426  // have enough space on the stack to call the runtime for the stack
2427  // overflow.
2428  Label done;
2429  // If the frame is bigger than the stack, we throw the stack overflow
2430  // exception unconditionally. Thereby we can avoid the integer overflow
2431  // check in the condition code.
2432  if (shrink_slots * kPointerSize < FLAG_stack_size * 1024) {
2433  UseScratchRegisterScope scope(tasm());
2434  Register scratch = scope.AcquireX();
2435  __ Ldr(scratch, FieldMemOperand(
2436  kWasmInstanceRegister,
2437  WasmInstanceObject::kRealStackLimitAddressOffset));
2438  __ Ldr(scratch, MemOperand(scratch));
2439  __ Add(scratch, scratch, shrink_slots * kPointerSize);
2440  __ Cmp(sp, scratch);
2441  __ B(hs, &done);
2442  }
2443 
2444  {
2445  // Finish the frame that hasn't been fully built yet.
2446  UseScratchRegisterScope temps(tasm());
2447  __ Claim(2); // Claim extra slots for marker + instance.
2448  Register scratch = temps.AcquireX();
2449  __ Mov(scratch,
2450  StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2451  __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2452  __ Str(kWasmInstanceRegister,
2453  MemOperand(fp, WasmCompiledFrameConstants::kWasmInstanceOffset));
2454  }
2455  __ Ldr(x2, FieldMemOperand(kWasmInstanceRegister,
2456  WasmInstanceObject::kCEntryStubOffset));
2457  __ Mov(cp, Smi::zero());
2458  __ CallRuntimeWithCEntry(Runtime::kThrowWasmStackOverflow, x2);
2459  // We come from WebAssembly, there are no references for the GC.
2460  ReferenceMap* reference_map = new (zone()) ReferenceMap(zone());
2461  RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2462  Safepoint::kNoLazyDeopt);
2463  if (FLAG_debug_code) {
2464  __ Brk(0);
2465  }
2466  __ Bind(&done);
2467  }
2468 
2469  // Skip callee-saved slots, which are pushed below.
2470  shrink_slots -= saves.Count();
2471  shrink_slots -= saves_fp.Count();
2472  shrink_slots -= returns;
2473 
2474  // Build remainder of frame, including accounting for and filling-in
2475  // frame-specific header information, i.e. claiming the extra slot that
2476  // other platforms explicitly push for STUB (code object) frames and frames
2477  // recording their argument count.
2478  switch (call_descriptor->kind()) {
2479  case CallDescriptor::kCallJSFunction:
2480  if (call_descriptor->PushArgumentCount()) {
2481  __ Claim(shrink_slots + 1); // Claim extra slot for argc.
2482  __ Str(kJavaScriptCallArgCountRegister,
2483  MemOperand(fp, OptimizedBuiltinFrameConstants::kArgCOffset));
2484  } else {
2485  __ Claim(shrink_slots);
2486  }
2487  break;
2488  case CallDescriptor::kCallCodeObject: {
2489  UseScratchRegisterScope temps(tasm());
2490  __ Claim(shrink_slots + 1); // Claim extra slot for frame type marker.
2491  Register scratch = temps.AcquireX();
2492  __ Mov(scratch,
2493  StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2494  __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2495  } break;
2496  case CallDescriptor::kCallWasmFunction: {
2497  UseScratchRegisterScope temps(tasm());
2498  __ Claim(shrink_slots + 2); // Claim extra slots for marker + instance.
2499  Register scratch = temps.AcquireX();
2500  __ Mov(scratch,
2501  StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2502  __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2503  __ Str(kWasmInstanceRegister,
2504  MemOperand(fp, WasmCompiledFrameConstants::kWasmInstanceOffset));
2505  } break;
2506  case CallDescriptor::kCallWasmImportWrapper: {
2507  UseScratchRegisterScope temps(tasm());
2508  __ ldr(kJSFunctionRegister,
2509  FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue2Offset));
2510  __ ldr(kWasmInstanceRegister,
2511  FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue1Offset));
2512  __ Claim(shrink_slots + 2); // Claim extra slots for marker + instance.
2513  Register scratch = temps.AcquireX();
2514  __ Mov(scratch,
2515  StackFrame::TypeToMarker(info()->GetOutputStackFrameType()));
2516  __ Str(scratch, MemOperand(fp, TypedFrameConstants::kFrameTypeOffset));
2517  __ Str(kWasmInstanceRegister,
2518  MemOperand(fp, WasmCompiledFrameConstants::kWasmInstanceOffset));
2519  } break;
2520  case CallDescriptor::kCallAddress:
2521  __ Claim(shrink_slots);
2522  break;
2523  default:
2524  UNREACHABLE();
2525  }
2526  }
2527 
2528  // Save FP registers.
2529  DCHECK_IMPLIES(saves_fp.Count() != 0,
2530  saves_fp.list() == CPURegList::GetCalleeSavedV().list());
2531  __ PushCPURegList(saves_fp);
2532 
2533  // Save registers.
2534  // TODO(palfia): TF save list is not in sync with
2535  // CPURegList::GetCalleeSaved(): x30 is missing.
2536  // DCHECK(saves.list() == CPURegList::GetCalleeSaved().list());
2537  __ PushCPURegList(saves);
2538 
2539  if (returns != 0) {
2540  __ Claim(returns);
2541  }
2542 }
2543 
2544 void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
2545  auto call_descriptor = linkage()->GetIncomingDescriptor();
2546 
2547  const int returns = RoundUp(frame()->GetReturnSlotCount(), 2);
2548 
2549  if (returns != 0) {
2550  __ Drop(returns);
2551  }
2552 
2553  // Restore registers.
2554  CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
2555  call_descriptor->CalleeSavedRegisters());
2556  __ PopCPURegList(saves);
2557 
2558  // Restore fp registers.
2559  CPURegList saves_fp = CPURegList(CPURegister::kVRegister, kDRegSizeInBits,
2560  call_descriptor->CalleeSavedFPRegisters());
2561  __ PopCPURegList(saves_fp);
2562 
2563  unwinding_info_writer_.MarkBlockWillExit();
2564 
2565  Arm64OperandConverter g(this, nullptr);
2566  int pop_count = static_cast<int>(call_descriptor->StackParameterCount());
2567  if (call_descriptor->IsCFunctionCall()) {
2568  AssembleDeconstructFrame();
2569  } else if (frame_access_state()->has_frame()) {
2570  // Canonicalize JSFunction return sites for now unless they have an variable
2571  // number of stack slot pops.
2572  if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
2573  if (return_label_.is_bound()) {
2574  __ B(&return_label_);
2575  return;
2576  } else {
2577  __ Bind(&return_label_);
2578  AssembleDeconstructFrame();
2579  }
2580  } else {
2581  AssembleDeconstructFrame();
2582  }
2583  }
2584 
2585  if (pop->IsImmediate()) {
2586  pop_count += g.ToConstant(pop).ToInt32();
2587  __ DropArguments(pop_count);
2588  } else {
2589  Register pop_reg = g.ToRegister(pop);
2590  __ Add(pop_reg, pop_reg, pop_count);
2591  __ DropArguments(pop_reg);
2592  }
2593 
2594  __ AssertSpAligned();
2595  __ Ret();
2596 }
2597 
2598 void CodeGenerator::FinishCode() { __ CheckConstPool(true, false); }
2599 
2600 void CodeGenerator::AssembleMove(InstructionOperand* source,
2601  InstructionOperand* destination) {
2602  Arm64OperandConverter g(this, nullptr);
2603  // Helper function to write the given constant to the dst register.
2604  auto MoveConstantToRegister = [&](Register dst, Constant src) {
2605  if (src.type() == Constant::kHeapObject) {
2606  Handle<HeapObject> src_object = src.ToHeapObject();
2607  RootIndex index;
2608  if (IsMaterializableFromRoot(src_object, &index)) {
2609  __ LoadRoot(dst, index);
2610  } else {
2611  __ Mov(dst, src_object);
2612  }
2613  } else {
2614  __ Mov(dst, g.ToImmediate(source));
2615  }
2616  };
2617  switch (MoveType::InferMove(source, destination)) {
2618  case MoveType::kRegisterToRegister:
2619  if (source->IsRegister()) {
2620  __ Mov(g.ToRegister(destination), g.ToRegister(source));
2621  } else if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2622  __ Mov(g.ToDoubleRegister(destination), g.ToDoubleRegister(source));
2623  } else {
2624  DCHECK(source->IsSimd128Register());
2625  __ Mov(g.ToDoubleRegister(destination).Q(),
2626  g.ToDoubleRegister(source).Q());
2627  }
2628  return;
2629  case MoveType::kRegisterToStack: {
2630  MemOperand dst = g.ToMemOperand(destination, tasm());
2631  if (source->IsRegister()) {
2632  __ Str(g.ToRegister(source), dst);
2633  } else {
2634  VRegister src = g.ToDoubleRegister(source);
2635  if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2636  __ Str(src, dst);
2637  } else {
2638  DCHECK(source->IsSimd128Register());
2639  __ Str(src.Q(), dst);
2640  }
2641  }
2642  return;
2643  }
2644  case MoveType::kStackToRegister: {
2645  MemOperand src = g.ToMemOperand(source, tasm());
2646  if (destination->IsRegister()) {
2647  __ Ldr(g.ToRegister(destination), src);
2648  } else {
2649  VRegister dst = g.ToDoubleRegister(destination);
2650  if (destination->IsFloatRegister() || destination->IsDoubleRegister()) {
2651  __ Ldr(dst, src);
2652  } else {
2653  DCHECK(destination->IsSimd128Register());
2654  __ Ldr(dst.Q(), src);
2655  }
2656  }
2657  return;
2658  }
2659  case MoveType::kStackToStack: {
2660  MemOperand src = g.ToMemOperand(source, tasm());
2661  MemOperand dst = g.ToMemOperand(destination, tasm());
2662  if (source->IsSimd128StackSlot()) {
2663  UseScratchRegisterScope scope(tasm());
2664  VRegister temp = scope.AcquireQ();
2665  __ Ldr(temp, src);
2666  __ Str(temp, dst);
2667  } else {
2668  UseScratchRegisterScope scope(tasm());
2669  Register temp = scope.AcquireX();
2670  __ Ldr(temp, src);
2671  __ Str(temp, dst);
2672  }
2673  return;
2674  }
2675  case MoveType::kConstantToRegister: {
2676  Constant src = g.ToConstant(source);
2677  if (destination->IsRegister()) {
2678  MoveConstantToRegister(g.ToRegister(destination), src);
2679  } else {
2680  VRegister dst = g.ToDoubleRegister(destination);
2681  if (destination->IsFloatRegister()) {
2682  __ Fmov(dst.S(), src.ToFloat32());
2683  } else {
2684  DCHECK(destination->IsDoubleRegister());
2685  __ Fmov(dst, src.ToFloat64().value());
2686  }
2687  }
2688  return;
2689  }
2690  case MoveType::kConstantToStack: {
2691  Constant src = g.ToConstant(source);
2692  MemOperand dst = g.ToMemOperand(destination, tasm());
2693  if (destination->IsStackSlot()) {
2694  UseScratchRegisterScope scope(tasm());
2695  Register temp = scope.AcquireX();
2696  MoveConstantToRegister(temp, src);
2697  __ Str(temp, dst);
2698  } else if (destination->IsFloatStackSlot()) {
2699  if (bit_cast<int32_t>(src.ToFloat32()) == 0) {
2700  __ Str(wzr, dst);
2701  } else {
2702  UseScratchRegisterScope scope(tasm());
2703  VRegister temp = scope.AcquireS();
2704  __ Fmov(temp, src.ToFloat32());
2705  __ Str(temp, dst);
2706  }
2707  } else {
2708  DCHECK(destination->IsDoubleStackSlot());
2709  if (src.ToFloat64().AsUint64() == 0) {
2710  __ Str(xzr, dst);
2711  } else {
2712  UseScratchRegisterScope scope(tasm());
2713  VRegister temp = scope.AcquireD();
2714  __ Fmov(temp, src.ToFloat64().value());
2715  __ Str(temp, dst);
2716  }
2717  }
2718  return;
2719  }
2720  }
2721  UNREACHABLE();
2722 }
2723 
2724 void CodeGenerator::AssembleSwap(InstructionOperand* source,
2725  InstructionOperand* destination) {
2726  Arm64OperandConverter g(this, nullptr);
2727  switch (MoveType::InferSwap(source, destination)) {
2728  case MoveType::kRegisterToRegister:
2729  if (source->IsRegister()) {
2730  __ Swap(g.ToRegister(source), g.ToRegister(destination));
2731  } else {
2732  VRegister src = g.ToDoubleRegister(source);
2733  VRegister dst = g.ToDoubleRegister(destination);
2734  if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2735  __ Swap(src, dst);
2736  } else {
2737  DCHECK(source->IsSimd128Register());
2738  __ Swap(src.Q(), dst.Q());
2739  }
2740  }
2741  return;
2742  case MoveType::kRegisterToStack: {
2743  UseScratchRegisterScope scope(tasm());
2744  MemOperand dst = g.ToMemOperand(destination, tasm());
2745  if (source->IsRegister()) {
2746  Register temp = scope.AcquireX();
2747  Register src = g.ToRegister(source);
2748  __ Mov(temp, src);
2749  __ Ldr(src, dst);
2750  __ Str(temp, dst);
2751  } else {
2752  UseScratchRegisterScope scope(tasm());
2753  VRegister src = g.ToDoubleRegister(source);
2754  if (source->IsFloatRegister() || source->IsDoubleRegister()) {
2755  VRegister temp = scope.AcquireD();
2756  __ Mov(temp, src);
2757  __ Ldr(src, dst);
2758  __ Str(temp, dst);
2759  } else {
2760  DCHECK(source->IsSimd128Register());
2761  VRegister temp = scope.AcquireQ();
2762  __ Mov(temp, src.Q());
2763  __ Ldr(src.Q(), dst);
2764  __ Str(temp, dst);
2765  }
2766  }
2767  return;
2768  }
2769  case MoveType::kStackToStack: {
2770  UseScratchRegisterScope scope(tasm());
2771  MemOperand src = g.ToMemOperand(source, tasm());
2772  MemOperand dst = g.ToMemOperand(destination, tasm());
2773  VRegister temp_0 = scope.AcquireD();
2774  VRegister temp_1 = scope.AcquireD();
2775  if (source->IsSimd128StackSlot()) {
2776  __ Ldr(temp_0.Q(), src);
2777  __ Ldr(temp_1.Q(), dst);
2778  __ Str(temp_0.Q(), dst);
2779  __ Str(temp_1.Q(), src);
2780  } else {
2781  __ Ldr(temp_0, src);
2782  __ Ldr(temp_1, dst);
2783  __ Str(temp_0, dst);
2784  __ Str(temp_1, src);
2785  }
2786  return;
2787  }
2788  default:
2789  UNREACHABLE();
2790  break;
2791  }
2792 }
2793 
2794 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2795  // On 64-bit ARM we emit the jump tables inline.
2796  UNREACHABLE();
2797 }
2798 
2799 #undef __
2800 
2801 } // namespace compiler
2802 } // namespace internal
2803 } // namespace v8
int64_t
v8
Definition: libplatform.h:13