V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
liftoff-assembler-arm64.h
1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef V8_WASM_BASELINE_ARM64_LIFTOFF_ASSEMBLER_ARM64_H_
6 #define V8_WASM_BASELINE_ARM64_LIFTOFF_ASSEMBLER_ARM64_H_
7 
8 #include "src/wasm/baseline/liftoff-assembler.h"
9 
10 #define BAILOUT(reason) bailout("arm64 " reason)
11 
12 namespace v8 {
13 namespace internal {
14 namespace wasm {
15 
16 namespace liftoff {
17 
18 // Liftoff Frames.
19 //
20 // slot Frame
21 // +--------------------+---------------------------
22 // n+4 | optional padding slot to keep the stack 16 byte aligned.
23 // n+3 | parameter n |
24 // ... | ... |
25 // 4 | parameter 1 | or parameter 2
26 // 3 | parameter 0 | or parameter 1
27 // 2 | (result address) | or parameter 0
28 // -----+--------------------+---------------------------
29 // 1 | return addr (lr) |
30 // 0 | previous frame (fp)|
31 // -----+--------------------+ <-- frame ptr (fp)
32 // -1 | 0xa: WASM_COMPILED |
33 // -2 | instance |
34 // -----+--------------------+---------------------------
35 // -3 | slot 0 | ^
36 // -4 | slot 1 | |
37 // | | Frame slots
38 // | | |
39 // | | v
40 // | optional padding slot to keep the stack 16 byte aligned.
41 // -----+--------------------+ <-- stack ptr (sp)
42 //
43 
44 constexpr int32_t kInstanceOffset = 2 * kPointerSize;
45 constexpr int32_t kFirstStackSlotOffset = kInstanceOffset + kPointerSize;
46 constexpr int32_t kConstantStackSpace = 0;
47 
48 inline MemOperand GetStackSlot(uint32_t index) {
49  int32_t offset =
50  kFirstStackSlotOffset + index * LiftoffAssembler::kStackSlotSize;
51  return MemOperand(fp, -offset);
52 }
53 
54 inline MemOperand GetInstanceOperand() {
55  return MemOperand(fp, -kInstanceOffset);
56 }
57 
58 inline CPURegister GetRegFromType(const LiftoffRegister& reg, ValueType type) {
59  switch (type) {
60  case kWasmI32:
61  return reg.gp().W();
62  case kWasmI64:
63  return reg.gp().X();
64  case kWasmF32:
65  return reg.fp().S();
66  case kWasmF64:
67  return reg.fp().D();
68  default:
69  UNREACHABLE();
70  }
71 }
72 
73 inline CPURegList PadRegList(RegList list) {
74  if ((base::bits::CountPopulation(list) & 1) != 0) list |= padreg.bit();
75  return CPURegList(CPURegister::kRegister, kXRegSizeInBits, list);
76 }
77 
78 inline CPURegList PadVRegList(RegList list) {
79  if ((base::bits::CountPopulation(list) & 1) != 0) list |= fp_scratch.bit();
80  return CPURegList(CPURegister::kVRegister, kDRegSizeInBits, list);
81 }
82 
83 inline CPURegister AcquireByType(UseScratchRegisterScope* temps,
84  ValueType type) {
85  switch (type) {
86  case kWasmI32:
87  return temps->AcquireW();
88  case kWasmI64:
89  return temps->AcquireX();
90  case kWasmF32:
91  return temps->AcquireS();
92  case kWasmF64:
93  return temps->AcquireD();
94  default:
95  UNREACHABLE();
96  }
97 }
98 
99 inline MemOperand GetMemOp(LiftoffAssembler* assm,
100  UseScratchRegisterScope* temps, Register addr,
101  Register offset, uint32_t offset_imm) {
102  // Wasm memory is limited to a size <2GB, so all offsets can be encoded as
103  // immediate value (in 31 bits, interpreted as signed value).
104  // If the offset is bigger, we always trap and this code is not reached.
105  DCHECK(is_uint31(offset_imm));
106  if (offset.IsValid()) {
107  if (offset_imm == 0) return MemOperand(addr.X(), offset.W(), UXTW);
108  Register tmp = temps->AcquireW();
109  assm->Add(tmp, offset.W(), offset_imm);
110  return MemOperand(addr.X(), tmp, UXTW);
111  }
112  return MemOperand(addr.X(), offset_imm);
113 }
114 
115 } // namespace liftoff
116 
117 int LiftoffAssembler::PrepareStackFrame() {
118  int offset = pc_offset();
119  InstructionAccurateScope scope(this, 1);
120  sub(sp, sp, 0);
121  return offset;
122 }
123 
124 void LiftoffAssembler::PatchPrepareStackFrame(int offset,
125  uint32_t stack_slots) {
126  static_assert(kStackSlotSize == kXRegSize,
127  "kStackSlotSize must equal kXRegSize");
128  uint32_t bytes = liftoff::kConstantStackSpace + kStackSlotSize * stack_slots;
129  // The stack pointer is required to be quadword aligned.
130  // Misalignment will cause a stack alignment fault.
131  bytes = RoundUp(bytes, kQuadWordSizeInBytes);
132  if (!IsImmAddSub(bytes)) {
133  // Round the stack to a page to try to fit a add/sub immediate.
134  bytes = RoundUp(bytes, 0x1000);
135  if (!IsImmAddSub(bytes)) {
136  // Stack greater than 4M! Because this is a quite improbable case, we
137  // just fallback to Turbofan.
138  BAILOUT("Stack too big");
139  return;
140  }
141  }
142 #ifdef USE_SIMULATOR
143  // When using the simulator, deal with Liftoff which allocates the stack
144  // before checking it.
145  // TODO(arm): Remove this when the stack check mechanism will be updated.
146  if (bytes > KB / 2) {
147  BAILOUT("Stack limited to 512 bytes to avoid a bug in StackCheck");
148  return;
149  }
150 #endif
151  PatchingAssembler patching_assembler(AssemblerOptions{}, buffer_ + offset, 1);
152  patching_assembler.PatchSubSp(bytes);
153 }
154 
155 void LiftoffAssembler::FinishCode() { CheckConstPool(true, false); }
156 
157 void LiftoffAssembler::AbortCompilation() { AbortedCodeGeneration(); }
158 
159 void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
160  RelocInfo::Mode rmode) {
161  switch (value.type()) {
162  case kWasmI32:
163  Mov(reg.gp().W(), Immediate(value.to_i32(), rmode));
164  break;
165  case kWasmI64:
166  Mov(reg.gp().X(), Immediate(value.to_i64(), rmode));
167  break;
168  case kWasmF32:
169  Fmov(reg.fp().S(), value.to_f32_boxed().get_scalar());
170  break;
171  case kWasmF64:
172  Fmov(reg.fp().D(), value.to_f64_boxed().get_scalar());
173  break;
174  default:
175  UNREACHABLE();
176  }
177 }
178 
179 void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset,
180  int size) {
181  DCHECK_LE(offset, kMaxInt);
182  Ldr(dst, liftoff::GetInstanceOperand());
183  DCHECK(size == 4 || size == 8);
184  if (size == 4) {
185  Ldr(dst.W(), MemOperand(dst, offset));
186  } else {
187  Ldr(dst, MemOperand(dst, offset));
188  }
189 }
190 
191 void LiftoffAssembler::SpillInstance(Register instance) {
192  Str(instance, liftoff::GetInstanceOperand());
193 }
194 
195 void LiftoffAssembler::FillInstanceInto(Register dst) {
196  Ldr(dst, liftoff::GetInstanceOperand());
197 }
198 
199 void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr,
200  Register offset_reg, uint32_t offset_imm,
201  LoadType type, LiftoffRegList pinned,
202  uint32_t* protected_load_pc, bool is_load_mem) {
203  UseScratchRegisterScope temps(this);
204  MemOperand src_op =
205  liftoff::GetMemOp(this, &temps, src_addr, offset_reg, offset_imm);
206  if (protected_load_pc) *protected_load_pc = pc_offset();
207  switch (type.value()) {
208  case LoadType::kI32Load8U:
209  case LoadType::kI64Load8U:
210  Ldrb(dst.gp().W(), src_op);
211  break;
212  case LoadType::kI32Load8S:
213  Ldrsb(dst.gp().W(), src_op);
214  break;
215  case LoadType::kI64Load8S:
216  Ldrsb(dst.gp().X(), src_op);
217  break;
218  case LoadType::kI32Load16U:
219  case LoadType::kI64Load16U:
220  Ldrh(dst.gp().W(), src_op);
221  break;
222  case LoadType::kI32Load16S:
223  Ldrsh(dst.gp().W(), src_op);
224  break;
225  case LoadType::kI64Load16S:
226  Ldrsh(dst.gp().X(), src_op);
227  break;
228  case LoadType::kI32Load:
229  case LoadType::kI64Load32U:
230  Ldr(dst.gp().W(), src_op);
231  break;
232  case LoadType::kI64Load32S:
233  Ldrsw(dst.gp().X(), src_op);
234  break;
235  case LoadType::kI64Load:
236  Ldr(dst.gp().X(), src_op);
237  break;
238  case LoadType::kF32Load:
239  Ldr(dst.fp().S(), src_op);
240  break;
241  case LoadType::kF64Load:
242  Ldr(dst.fp().D(), src_op);
243  break;
244  default:
245  UNREACHABLE();
246  }
247 }
248 
249 void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
250  uint32_t offset_imm, LiftoffRegister src,
251  StoreType type, LiftoffRegList pinned,
252  uint32_t* protected_store_pc, bool is_store_mem) {
253  UseScratchRegisterScope temps(this);
254  MemOperand dst_op =
255  liftoff::GetMemOp(this, &temps, dst_addr, offset_reg, offset_imm);
256  if (protected_store_pc) *protected_store_pc = pc_offset();
257  switch (type.value()) {
258  case StoreType::kI32Store8:
259  case StoreType::kI64Store8:
260  Strb(src.gp().W(), dst_op);
261  break;
262  case StoreType::kI32Store16:
263  case StoreType::kI64Store16:
264  Strh(src.gp().W(), dst_op);
265  break;
266  case StoreType::kI32Store:
267  case StoreType::kI64Store32:
268  Str(src.gp().W(), dst_op);
269  break;
270  case StoreType::kI64Store:
271  Str(src.gp().X(), dst_op);
272  break;
273  case StoreType::kF32Store:
274  Str(src.fp().S(), dst_op);
275  break;
276  case StoreType::kF64Store:
277  Str(src.fp().D(), dst_op);
278  break;
279  default:
280  UNREACHABLE();
281  }
282 }
283 
284 void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst,
285  uint32_t caller_slot_idx,
286  ValueType type) {
287  int32_t offset = (caller_slot_idx + 1) * LiftoffAssembler::kStackSlotSize;
288  Ldr(liftoff::GetRegFromType(dst, type), MemOperand(fp, offset));
289 }
290 
291 void LiftoffAssembler::MoveStackValue(uint32_t dst_index, uint32_t src_index,
292  ValueType type) {
293  UseScratchRegisterScope temps(this);
294  CPURegister scratch = liftoff::AcquireByType(&temps, type);
295  Ldr(scratch, liftoff::GetStackSlot(src_index));
296  Str(scratch, liftoff::GetStackSlot(dst_index));
297 }
298 
299 void LiftoffAssembler::Move(Register dst, Register src, ValueType type) {
300  if (type == kWasmI32) {
301  Mov(dst.W(), src.W());
302  } else {
303  DCHECK_EQ(kWasmI64, type);
304  Mov(dst.X(), src.X());
305  }
306 }
307 
308 void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src,
309  ValueType type) {
310  if (type == kWasmF32) {
311  Fmov(dst.S(), src.S());
312  } else {
313  DCHECK_EQ(kWasmF64, type);
314  Fmov(dst.D(), src.D());
315  }
316 }
317 
318 void LiftoffAssembler::Spill(uint32_t index, LiftoffRegister reg,
319  ValueType type) {
320  RecordUsedSpillSlot(index);
321  MemOperand dst = liftoff::GetStackSlot(index);
322  Str(liftoff::GetRegFromType(reg, type), dst);
323 }
324 
325 void LiftoffAssembler::Spill(uint32_t index, WasmValue value) {
326  RecordUsedSpillSlot(index);
327  MemOperand dst = liftoff::GetStackSlot(index);
328  UseScratchRegisterScope temps(this);
329  CPURegister src = CPURegister::no_reg();
330  switch (value.type()) {
331  case kWasmI32:
332  src = temps.AcquireW();
333  Mov(src.W(), value.to_i32());
334  break;
335  case kWasmI64:
336  src = temps.AcquireX();
337  Mov(src.X(), value.to_i64());
338  break;
339  default:
340  // We do not track f32 and f64 constants, hence they are unreachable.
341  UNREACHABLE();
342  }
343  Str(src, dst);
344 }
345 
346 void LiftoffAssembler::Fill(LiftoffRegister reg, uint32_t index,
347  ValueType type) {
348  MemOperand src = liftoff::GetStackSlot(index);
349  Ldr(liftoff::GetRegFromType(reg, type), src);
350 }
351 
352 void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) {
353  UNREACHABLE();
354 }
355 
356 #define I32_BINOP(name, instruction) \
357  void LiftoffAssembler::emit_##name(Register dst, Register lhs, \
358  Register rhs) { \
359  instruction(dst.W(), lhs.W(), rhs.W()); \
360  }
361 #define I64_BINOP(name, instruction) \
362  void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister lhs, \
363  LiftoffRegister rhs) { \
364  instruction(dst.gp().X(), lhs.gp().X(), rhs.gp().X()); \
365  }
366 #define FP32_BINOP(name, instruction) \
367  void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister lhs, \
368  DoubleRegister rhs) { \
369  instruction(dst.S(), lhs.S(), rhs.S()); \
370  }
371 #define FP32_UNOP(name, instruction) \
372  void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
373  instruction(dst.S(), src.S()); \
374  }
375 #define FP32_UNOP_RETURN_TRUE(name, instruction) \
376  bool LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
377  instruction(dst.S(), src.S()); \
378  return true; \
379  }
380 #define FP64_BINOP(name, instruction) \
381  void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister lhs, \
382  DoubleRegister rhs) { \
383  instruction(dst.D(), lhs.D(), rhs.D()); \
384  }
385 #define FP64_UNOP(name, instruction) \
386  void LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
387  instruction(dst.D(), src.D()); \
388  }
389 #define FP64_UNOP_RETURN_TRUE(name, instruction) \
390  bool LiftoffAssembler::emit_##name(DoubleRegister dst, DoubleRegister src) { \
391  instruction(dst.D(), src.D()); \
392  return true; \
393  }
394 #define I32_SHIFTOP(name, instruction) \
395  void LiftoffAssembler::emit_##name(Register dst, Register src, \
396  Register amount, LiftoffRegList pinned) { \
397  instruction(dst.W(), src.W(), amount.W()); \
398  }
399 #define I32_SHIFTOP_I(name, instruction) \
400  I32_SHIFTOP(name, instruction) \
401  void LiftoffAssembler::emit_##name(Register dst, Register src, int amount) { \
402  DCHECK(is_uint5(amount)); \
403  instruction(dst.W(), src.W(), amount); \
404  }
405 #define I64_SHIFTOP(name, instruction) \
406  void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister src, \
407  Register amount, LiftoffRegList pinned) { \
408  instruction(dst.gp().X(), src.gp().X(), amount.X()); \
409  }
410 #define I64_SHIFTOP_I(name, instruction) \
411  I64_SHIFTOP(name, instruction) \
412  void LiftoffAssembler::emit_##name(LiftoffRegister dst, LiftoffRegister src, \
413  int amount) { \
414  DCHECK(is_uint6(amount)); \
415  instruction(dst.gp().X(), src.gp().X(), amount); \
416  }
417 
418 I32_BINOP(i32_add, Add)
419 I32_BINOP(i32_sub, Sub)
420 I32_BINOP(i32_mul, Mul)
421 I32_BINOP(i32_and, And)
422 I32_BINOP(i32_or, Orr)
423 I32_BINOP(i32_xor, Eor)
424 I32_SHIFTOP(i32_shl, Lsl)
425 I32_SHIFTOP(i32_sar, Asr)
426 I32_SHIFTOP_I(i32_shr, Lsr)
427 I64_BINOP(i64_add, Add)
428 I64_BINOP(i64_sub, Sub)
429 I64_BINOP(i64_mul, Mul)
430 I64_BINOP(i64_and, And)
431 I64_BINOP(i64_or, Orr)
432 I64_BINOP(i64_xor, Eor)
433 I64_SHIFTOP(i64_shl, Lsl)
434 I64_SHIFTOP(i64_sar, Asr)
435 I64_SHIFTOP_I(i64_shr, Lsr)
436 FP32_BINOP(f32_add, Fadd)
437 FP32_BINOP(f32_sub, Fsub)
438 FP32_BINOP(f32_mul, Fmul)
439 FP32_BINOP(f32_div, Fdiv)
440 FP32_BINOP(f32_min, Fmin)
441 FP32_BINOP(f32_max, Fmax)
442 FP32_UNOP(f32_abs, Fabs)
443 FP32_UNOP(f32_neg, Fneg)
444 FP32_UNOP_RETURN_TRUE(f32_ceil, Frintp)
445 FP32_UNOP_RETURN_TRUE(f32_floor, Frintm)
446 FP32_UNOP_RETURN_TRUE(f32_trunc, Frintz)
447 FP32_UNOP_RETURN_TRUE(f32_nearest_int, Frintn)
448 FP32_UNOP(f32_sqrt, Fsqrt)
449 FP64_BINOP(f64_add, Fadd)
450 FP64_BINOP(f64_sub, Fsub)
451 FP64_BINOP(f64_mul, Fmul)
452 FP64_BINOP(f64_div, Fdiv)
453 FP64_BINOP(f64_min, Fmin)
454 FP64_BINOP(f64_max, Fmax)
455 FP64_UNOP(f64_abs, Fabs)
456 FP64_UNOP(f64_neg, Fneg)
457 FP64_UNOP_RETURN_TRUE(f64_ceil, Frintp)
458 FP64_UNOP_RETURN_TRUE(f64_floor, Frintm)
459 FP64_UNOP_RETURN_TRUE(f64_trunc, Frintz)
460 FP64_UNOP_RETURN_TRUE(f64_nearest_int, Frintn)
461 FP64_UNOP(f64_sqrt, Fsqrt)
462 
463 #undef I32_BINOP
464 #undef I64_BINOP
465 #undef FP32_BINOP
466 #undef FP32_UNOP
467 #undef FP64_BINOP
468 #undef FP64_UNOP
469 #undef FP64_UNOP_RETURN_TRUE
470 #undef I32_SHIFTOP
471 #undef I32_SHIFTOP_I
472 #undef I64_SHIFTOP
473 #undef I64_SHIFTOP_I
474 
475 bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) {
476  Clz(dst.W(), src.W());
477  return true;
478 }
479 
480 bool LiftoffAssembler::emit_i32_ctz(Register dst, Register src) {
481  Rbit(dst.W(), src.W());
482  Clz(dst.W(), dst.W());
483  return true;
484 }
485 
486 bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) {
487  UseScratchRegisterScope temps(this);
488  VRegister scratch = temps.AcquireV(kFormat8B);
489  Fmov(scratch.S(), src.W());
490  Cnt(scratch, scratch);
491  Addv(scratch.B(), scratch);
492  Fmov(dst.W(), scratch.S());
493  return true;
494 }
495 
496 void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs,
497  Label* trap_div_by_zero,
498  Label* trap_div_unrepresentable) {
499  Register dst_w = dst.W();
500  Register lhs_w = lhs.W();
501  Register rhs_w = rhs.W();
502  bool can_use_dst = !dst_w.Aliases(lhs_w) && !dst_w.Aliases(rhs_w);
503  if (can_use_dst) {
504  // Do div early.
505  Sdiv(dst_w, lhs_w, rhs_w);
506  }
507  // Check for division by zero.
508  Cbz(rhs_w, trap_div_by_zero);
509  // Check for kMinInt / -1. This is unrepresentable.
510  Cmp(rhs_w, -1);
511  Ccmp(lhs_w, 1, NoFlag, eq);
512  B(trap_div_unrepresentable, vs);
513  if (!can_use_dst) {
514  // Do div.
515  Sdiv(dst_w, lhs_w, rhs_w);
516  }
517 }
518 
519 void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs,
520  Label* trap_div_by_zero) {
521  // Check for division by zero.
522  Cbz(rhs.W(), trap_div_by_zero);
523  // Do div.
524  Udiv(dst.W(), lhs.W(), rhs.W());
525 }
526 
527 void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs,
528  Label* trap_div_by_zero) {
529  Register dst_w = dst.W();
530  Register lhs_w = lhs.W();
531  Register rhs_w = rhs.W();
532  // Do early div.
533  // No need to check kMinInt / -1 because the result is kMinInt and then
534  // kMinInt * -1 -> kMinInt. In this case, the Msub result is therefore 0.
535  UseScratchRegisterScope temps(this);
536  Register scratch = temps.AcquireW();
537  Sdiv(scratch, lhs_w, rhs_w);
538  // Check for division by zero.
539  Cbz(rhs_w, trap_div_by_zero);
540  // Compute remainder.
541  Msub(dst_w, scratch, rhs_w, lhs_w);
542 }
543 
544 void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs,
545  Label* trap_div_by_zero) {
546  Register dst_w = dst.W();
547  Register lhs_w = lhs.W();
548  Register rhs_w = rhs.W();
549  // Do early div.
550  UseScratchRegisterScope temps(this);
551  Register scratch = temps.AcquireW();
552  Udiv(scratch, lhs_w, rhs_w);
553  // Check for division by zero.
554  Cbz(rhs_w, trap_div_by_zero);
555  // Compute remainder.
556  Msub(dst_w, scratch, rhs_w, lhs_w);
557 }
558 
559 bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs,
560  LiftoffRegister rhs,
561  Label* trap_div_by_zero,
562  Label* trap_div_unrepresentable) {
563  Register dst_x = dst.gp().X();
564  Register lhs_x = lhs.gp().X();
565  Register rhs_x = rhs.gp().X();
566  bool can_use_dst = !dst_x.Aliases(lhs_x) && !dst_x.Aliases(rhs_x);
567  if (can_use_dst) {
568  // Do div early.
569  Sdiv(dst_x, lhs_x, rhs_x);
570  }
571  // Check for division by zero.
572  Cbz(rhs_x, trap_div_by_zero);
573  // Check for kMinInt / -1. This is unrepresentable.
574  Cmp(rhs_x, -1);
575  Ccmp(lhs_x, 1, NoFlag, eq);
576  B(trap_div_unrepresentable, vs);
577  if (!can_use_dst) {
578  // Do div.
579  Sdiv(dst_x, lhs_x, rhs_x);
580  }
581  return true;
582 }
583 
584 bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs,
585  LiftoffRegister rhs,
586  Label* trap_div_by_zero) {
587  // Check for division by zero.
588  Cbz(rhs.gp().X(), trap_div_by_zero);
589  // Do div.
590  Udiv(dst.gp().X(), lhs.gp().X(), rhs.gp().X());
591  return true;
592 }
593 
594 bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs,
595  LiftoffRegister rhs,
596  Label* trap_div_by_zero) {
597  Register dst_x = dst.gp().X();
598  Register lhs_x = lhs.gp().X();
599  Register rhs_x = rhs.gp().X();
600  // Do early div.
601  // No need to check kMinInt / -1 because the result is kMinInt and then
602  // kMinInt * -1 -> kMinInt. In this case, the Msub result is therefore 0.
603  UseScratchRegisterScope temps(this);
604  Register scratch = temps.AcquireX();
605  Sdiv(scratch, lhs_x, rhs_x);
606  // Check for division by zero.
607  Cbz(rhs_x, trap_div_by_zero);
608  // Compute remainder.
609  Msub(dst_x, scratch, rhs_x, lhs_x);
610  return true;
611 }
612 
613 bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs,
614  LiftoffRegister rhs,
615  Label* trap_div_by_zero) {
616  Register dst_x = dst.gp().X();
617  Register lhs_x = lhs.gp().X();
618  Register rhs_x = rhs.gp().X();
619  // Do early div.
620  UseScratchRegisterScope temps(this);
621  Register scratch = temps.AcquireX();
622  Udiv(scratch, lhs_x, rhs_x);
623  // Check for division by zero.
624  Cbz(rhs_x, trap_div_by_zero);
625  // Compute remainder.
626  Msub(dst_x, scratch, rhs_x, lhs_x);
627  return true;
628 }
629 
630 void LiftoffAssembler::emit_i32_to_intptr(Register dst, Register src) {
631  Sxtw(dst, src);
632 }
633 
634 void LiftoffAssembler::emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs,
635  DoubleRegister rhs) {
636  UseScratchRegisterScope temps(this);
637  DoubleRegister scratch = temps.AcquireD();
638  Ushr(scratch.V2S(), rhs.V2S(), 31);
639  if (dst != lhs) {
640  Fmov(dst.S(), lhs.S());
641  }
642  Sli(dst.V2S(), scratch.V2S(), 31);
643 }
644 
645 void LiftoffAssembler::emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs,
646  DoubleRegister rhs) {
647  UseScratchRegisterScope temps(this);
648  DoubleRegister scratch = temps.AcquireD();
649  Ushr(scratch.V1D(), rhs.V1D(), 63);
650  if (dst != lhs) {
651  Fmov(dst.D(), lhs.D());
652  }
653  Sli(dst.V1D(), scratch.V1D(), 63);
654 }
655 
656 bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
657  LiftoffRegister dst,
658  LiftoffRegister src, Label* trap) {
659  switch (opcode) {
660  case kExprI32ConvertI64:
661  if (src != dst) Mov(dst.gp().W(), src.gp().W());
662  return true;
663  case kExprI32SConvertF32:
664  Fcvtzs(dst.gp().W(), src.fp().S()); // f32 -> i32 round to zero.
665  // Check underflow and NaN.
666  Fcmp(src.fp().S(), static_cast<float>(INT32_MIN));
667  // Check overflow.
668  Ccmp(dst.gp().W(), -1, VFlag, ge);
669  B(trap, vs);
670  return true;
671  case kExprI32UConvertF32:
672  Fcvtzu(dst.gp().W(), src.fp().S()); // f32 -> i32 round to zero.
673  // Check underflow and NaN.
674  Fcmp(src.fp().S(), -1.0);
675  // Check overflow.
676  Ccmp(dst.gp().W(), -1, ZFlag, gt);
677  B(trap, eq);
678  return true;
679  case kExprI32SConvertF64: {
680  // INT32_MIN and INT32_MAX are valid results, we cannot test the result
681  // to detect the overflows. We could have done two immediate floating
682  // point comparisons but it would have generated two conditional branches.
683  UseScratchRegisterScope temps(this);
684  VRegister fp_ref = temps.AcquireD();
685  VRegister fp_cmp = temps.AcquireD();
686  Fcvtzs(dst.gp().W(), src.fp().D()); // f64 -> i32 round to zero.
687  Frintz(fp_ref, src.fp().D()); // f64 -> f64 round to zero.
688  Scvtf(fp_cmp, dst.gp().W()); // i32 -> f64.
689  // If comparison fails, we have an overflow or a NaN.
690  Fcmp(fp_cmp, fp_ref);
691  B(trap, ne);
692  return true;
693  }
694  case kExprI32UConvertF64: {
695  // INT32_MAX is a valid result, we cannot test the result to detect the
696  // overflows. We could have done two immediate floating point comparisons
697  // but it would have generated two conditional branches.
698  UseScratchRegisterScope temps(this);
699  VRegister fp_ref = temps.AcquireD();
700  VRegister fp_cmp = temps.AcquireD();
701  Fcvtzu(dst.gp().W(), src.fp().D()); // f64 -> i32 round to zero.
702  Frintz(fp_ref, src.fp().D()); // f64 -> f64 round to zero.
703  Ucvtf(fp_cmp, dst.gp().W()); // i32 -> f64.
704  // If comparison fails, we have an overflow or a NaN.
705  Fcmp(fp_cmp, fp_ref);
706  B(trap, ne);
707  return true;
708  }
709  case kExprI32ReinterpretF32:
710  Fmov(dst.gp().W(), src.fp().S());
711  return true;
712  case kExprI64SConvertI32:
713  Sxtw(dst.gp().X(), src.gp().W());
714  return true;
715  case kExprI64SConvertF32:
716  Fcvtzs(dst.gp().X(), src.fp().S()); // f32 -> i64 round to zero.
717  // Check underflow and NaN.
718  Fcmp(src.fp().S(), static_cast<float>(INT64_MIN));
719  // Check overflow.
720  Ccmp(dst.gp().X(), -1, VFlag, ge);
721  B(trap, vs);
722  return true;
723  case kExprI64UConvertF32:
724  Fcvtzu(dst.gp().X(), src.fp().S()); // f32 -> i64 round to zero.
725  // Check underflow and NaN.
726  Fcmp(src.fp().S(), -1.0);
727  // Check overflow.
728  Ccmp(dst.gp().X(), -1, ZFlag, gt);
729  B(trap, eq);
730  return true;
731  case kExprI64SConvertF64:
732  Fcvtzs(dst.gp().X(), src.fp().D()); // f64 -> i64 round to zero.
733  // Check underflow and NaN.
734  Fcmp(src.fp().D(), static_cast<float>(INT64_MIN));
735  // Check overflow.
736  Ccmp(dst.gp().X(), -1, VFlag, ge);
737  B(trap, vs);
738  return true;
739  case kExprI64UConvertF64:
740  Fcvtzu(dst.gp().X(), src.fp().D()); // f64 -> i64 round to zero.
741  // Check underflow and NaN.
742  Fcmp(src.fp().D(), -1.0);
743  // Check overflow.
744  Ccmp(dst.gp().X(), -1, ZFlag, gt);
745  B(trap, eq);
746  return true;
747  case kExprI64UConvertI32:
748  Mov(dst.gp().W(), src.gp().W());
749  return true;
750  case kExprI64ReinterpretF64:
751  Fmov(dst.gp().X(), src.fp().D());
752  return true;
753  case kExprF32SConvertI32:
754  Scvtf(dst.fp().S(), src.gp().W());
755  return true;
756  case kExprF32UConvertI32:
757  Ucvtf(dst.fp().S(), src.gp().W());
758  return true;
759  case kExprF32SConvertI64:
760  Scvtf(dst.fp().S(), src.gp().X());
761  return true;
762  case kExprF32UConvertI64:
763  Ucvtf(dst.fp().S(), src.gp().X());
764  return true;
765  case kExprF32ConvertF64:
766  Fcvt(dst.fp().S(), src.fp().D());
767  return true;
768  case kExprF32ReinterpretI32:
769  Fmov(dst.fp().S(), src.gp().W());
770  return true;
771  case kExprF64SConvertI32:
772  Scvtf(dst.fp().D(), src.gp().W());
773  return true;
774  case kExprF64UConvertI32:
775  Ucvtf(dst.fp().D(), src.gp().W());
776  return true;
777  case kExprF64SConvertI64:
778  Scvtf(dst.fp().D(), src.gp().X());
779  return true;
780  case kExprF64UConvertI64:
781  Ucvtf(dst.fp().D(), src.gp().X());
782  return true;
783  case kExprF64ConvertF32:
784  Fcvt(dst.fp().D(), src.fp().S());
785  return true;
786  case kExprF64ReinterpretI64:
787  Fmov(dst.fp().D(), src.gp().X());
788  return true;
789  default:
790  UNREACHABLE();
791  }
792 }
793 
794 void LiftoffAssembler::emit_i32_signextend_i8(Register dst, Register src) {
795  sxtb(dst, src);
796 }
797 
798 void LiftoffAssembler::emit_i32_signextend_i16(Register dst, Register src) {
799  sxth(dst, src);
800 }
801 
802 void LiftoffAssembler::emit_i64_signextend_i8(LiftoffRegister dst,
803  LiftoffRegister src) {
804  sxtb(dst.gp(), src.gp());
805 }
806 
807 void LiftoffAssembler::emit_i64_signextend_i16(LiftoffRegister dst,
808  LiftoffRegister src) {
809  sxth(dst.gp(), src.gp());
810 }
811 
812 void LiftoffAssembler::emit_i64_signextend_i32(LiftoffRegister dst,
813  LiftoffRegister src) {
814  sxtw(dst.gp(), src.gp());
815 }
816 
817 void LiftoffAssembler::emit_jump(Label* label) { B(label); }
818 
819 void LiftoffAssembler::emit_jump(Register target) { Br(target); }
820 
821 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
822  ValueType type, Register lhs,
823  Register rhs) {
824  switch (type) {
825  case kWasmI32:
826  if (rhs.IsValid()) {
827  Cmp(lhs.W(), rhs.W());
828  } else {
829  Cmp(lhs.W(), wzr);
830  }
831  break;
832  case kWasmI64:
833  if (rhs.IsValid()) {
834  Cmp(lhs.X(), rhs.X());
835  } else {
836  Cmp(lhs.X(), xzr);
837  }
838  break;
839  default:
840  UNREACHABLE();
841  }
842  B(label, cond);
843 }
844 
845 void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) {
846  Cmp(src.W(), wzr);
847  Cset(dst.W(), eq);
848 }
849 
850 void LiftoffAssembler::emit_i32_set_cond(Condition cond, Register dst,
851  Register lhs, Register rhs) {
852  Cmp(lhs.W(), rhs.W());
853  Cset(dst.W(), cond);
854 }
855 
856 void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) {
857  Cmp(src.gp().X(), xzr);
858  Cset(dst.W(), eq);
859 }
860 
861 void LiftoffAssembler::emit_i64_set_cond(Condition cond, Register dst,
862  LiftoffRegister lhs,
863  LiftoffRegister rhs) {
864  Cmp(lhs.gp().X(), rhs.gp().X());
865  Cset(dst.W(), cond);
866 }
867 
868 void LiftoffAssembler::emit_f32_set_cond(Condition cond, Register dst,
869  DoubleRegister lhs,
870  DoubleRegister rhs) {
871  Fcmp(lhs.S(), rhs.S());
872  Cset(dst.W(), cond);
873  if (cond != ne) {
874  // If V flag set, at least one of the arguments was a Nan -> false.
875  Csel(dst.W(), wzr, dst.W(), vs);
876  }
877 }
878 
879 void LiftoffAssembler::emit_f64_set_cond(Condition cond, Register dst,
880  DoubleRegister lhs,
881  DoubleRegister rhs) {
882  Fcmp(lhs.D(), rhs.D());
883  Cset(dst.W(), cond);
884  if (cond != ne) {
885  // If V flag set, at least one of the arguments was a Nan -> false.
886  Csel(dst.W(), wzr, dst.W(), vs);
887  }
888 }
889 
890 void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) {
891  Ldr(limit_address, MemOperand(limit_address));
892  Cmp(sp, limit_address);
893  B(ool_code, ls);
894 }
895 
896 void LiftoffAssembler::CallTrapCallbackForTesting() {
897  CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0);
898 }
899 
900 void LiftoffAssembler::AssertUnreachable(AbortReason reason) {
901  TurboAssembler::AssertUnreachable(reason);
902 }
903 
904 void LiftoffAssembler::PushRegisters(LiftoffRegList regs) {
905  PushCPURegList(liftoff::PadRegList(regs.GetGpList()));
906  PushCPURegList(liftoff::PadVRegList(regs.GetFpList()));
907 }
908 
909 void LiftoffAssembler::PopRegisters(LiftoffRegList regs) {
910  PopCPURegList(liftoff::PadVRegList(regs.GetFpList()));
911  PopCPURegList(liftoff::PadRegList(regs.GetGpList()));
912 }
913 
914 void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) {
915  DropSlots(num_stack_slots);
916  Ret();
917 }
918 
919 void LiftoffAssembler::CallC(wasm::FunctionSig* sig,
920  const LiftoffRegister* args,
921  const LiftoffRegister* rets,
922  ValueType out_argument_type, int stack_bytes,
923  ExternalReference ext_ref) {
924  // The stack pointer is required to be quadword aligned.
925  int total_size = RoundUp(stack_bytes, kQuadWordSizeInBytes);
926  // Reserve space in the stack.
927  Claim(total_size, 1);
928 
929  int arg_bytes = 0;
930  for (ValueType param_type : sig->parameters()) {
931  Poke(liftoff::GetRegFromType(*args++, param_type), arg_bytes);
932  arg_bytes += ValueTypes::MemSize(param_type);
933  }
934  DCHECK_LE(arg_bytes, stack_bytes);
935 
936  // Pass a pointer to the buffer with the arguments to the C function.
937  Mov(x0, sp);
938 
939  // Now call the C function.
940  constexpr int kNumCCallArgs = 1;
941  CallCFunction(ext_ref, kNumCCallArgs);
942 
943  // Move return value to the right register.
944  const LiftoffRegister* next_result_reg = rets;
945  if (sig->return_count() > 0) {
946  DCHECK_EQ(1, sig->return_count());
947  constexpr Register kReturnReg = x0;
948  if (kReturnReg != next_result_reg->gp()) {
949  Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0));
950  }
951  ++next_result_reg;
952  }
953 
954  // Load potential output value from the buffer on the stack.
955  if (out_argument_type != kWasmStmt) {
956  Peek(liftoff::GetRegFromType(*next_result_reg, out_argument_type), 0);
957  }
958 
959  Drop(total_size, 1);
960 }
961 
962 void LiftoffAssembler::CallNativeWasmCode(Address addr) {
963  Call(addr, RelocInfo::WASM_CALL);
964 }
965 
966 void LiftoffAssembler::CallIndirect(wasm::FunctionSig* sig,
967  compiler::CallDescriptor* call_descriptor,
968  Register target) {
969  // For Arm64, we have more cache registers than wasm parameters. That means
970  // that target will always be in a register.
971  DCHECK(target.IsValid());
972  Call(target);
973 }
974 
975 void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) {
976  // A direct call to a wasm runtime stub defined in this module.
977  // Just encode the stub index. This will be patched at relocation.
978  Call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL);
979 }
980 
981 void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) {
982  // The stack pointer is required to be quadword aligned.
983  size = RoundUp(size, kQuadWordSizeInBytes);
984  Claim(size, 1);
985  Mov(addr, sp);
986 }
987 
988 void LiftoffAssembler::DeallocateStackSlot(uint32_t size) {
989  // The stack pointer is required to be quadword aligned.
990  size = RoundUp(size, kQuadWordSizeInBytes);
991  Drop(size, 1);
992 }
993 
994 void LiftoffStackSlots::Construct() {
995  size_t slot_count = slots_.size();
996  // The stack pointer is required to be quadword aligned.
997  asm_->Claim(RoundUp(slot_count, 2));
998  size_t slot_index = 0;
999  for (auto& slot : slots_) {
1000  size_t poke_offset = (slot_count - slot_index - 1) * kXRegSize;
1001  switch (slot.src_.loc()) {
1002  case LiftoffAssembler::VarState::kStack: {
1003  UseScratchRegisterScope temps(asm_);
1004  CPURegister scratch = liftoff::AcquireByType(&temps, slot.src_.type());
1005  asm_->Ldr(scratch, liftoff::GetStackSlot(slot.src_index_));
1006  asm_->Poke(scratch, poke_offset);
1007  break;
1008  }
1009  case LiftoffAssembler::VarState::kRegister:
1010  asm_->Poke(liftoff::GetRegFromType(slot.src_.reg(), slot.src_.type()),
1011  poke_offset);
1012  break;
1013  case LiftoffAssembler::VarState::KIntConst:
1014  DCHECK(slot.src_.type() == kWasmI32 || slot.src_.type() == kWasmI64);
1015  if (slot.src_.i32_const() == 0) {
1016  Register zero_reg = slot.src_.type() == kWasmI32 ? wzr : xzr;
1017  asm_->Poke(zero_reg, poke_offset);
1018  } else {
1019  UseScratchRegisterScope temps(asm_);
1020  Register scratch = slot.src_.type() == kWasmI32 ? temps.AcquireW()
1021  : temps.AcquireX();
1022  asm_->Mov(scratch, int64_t{slot.src_.i32_const()});
1023  asm_->Poke(scratch, poke_offset);
1024  }
1025  break;
1026  }
1027  slot_index++;
1028  }
1029 }
1030 
1031 } // namespace wasm
1032 } // namespace internal
1033 } // namespace v8
1034 
1035 #undef BAILOUT
1036 
1037 #endif // V8_WASM_BASELINE_ARM64_LIFTOFF_ASSEMBLER_ARM64_H_
int64_t
v8
Definition: libplatform.h:13
uint32_t