V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
liftoff-assembler-x64.h
1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
6 #define V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
7 
8 #include "src/wasm/baseline/liftoff-assembler.h"
9 
10 #include "src/assembler.h"
11 #include "src/wasm/value-type.h"
12 
13 namespace v8 {
14 namespace internal {
15 namespace wasm {
16 
17 #define REQUIRE_CPU_FEATURE(name, ...) \
18  if (!CpuFeatures::IsSupported(name)) { \
19  bailout("no " #name); \
20  return __VA_ARGS__; \
21  } \
22  CpuFeatureScope feature(this, name);
23 
24 namespace liftoff {
25 
26 constexpr Register kScratchRegister2 = r11;
27 static_assert(kScratchRegister != kScratchRegister2, "collision");
28 static_assert((kLiftoffAssemblerGpCacheRegs &
29  Register::ListOf<kScratchRegister, kScratchRegister2>()) == 0,
30  "scratch registers must not be used as cache registers");
31 
32 constexpr DoubleRegister kScratchDoubleReg2 = xmm14;
33 static_assert(kScratchDoubleReg != kScratchDoubleReg2, "collision");
34 static_assert(
35  (kLiftoffAssemblerFpCacheRegs &
36  DoubleRegister::ListOf<kScratchDoubleReg, kScratchDoubleReg2>()) == 0,
37  "scratch registers must not be used as cache registers");
38 
39 // rbp-8 holds the stack marker, rbp-16 is the instance parameter, first stack
40 // slot is located at rbp-24.
41 constexpr int32_t kConstantStackSpace = 16;
42 constexpr int32_t kFirstStackSlotOffset =
43  kConstantStackSpace + LiftoffAssembler::kStackSlotSize;
44 
45 inline Operand GetStackSlot(uint32_t index) {
46  int32_t offset = index * LiftoffAssembler::kStackSlotSize;
47  return Operand(rbp, -kFirstStackSlotOffset - offset);
48 }
49 
50 // TODO(clemensh): Make this a constexpr variable once Operand is constexpr.
51 inline Operand GetInstanceOperand() { return Operand(rbp, -16); }
52 
53 inline Operand GetMemOp(LiftoffAssembler* assm, Register addr, Register offset,
54  uint32_t offset_imm) {
55  if (is_uint31(offset_imm)) {
56  if (offset == no_reg) return Operand(addr, offset_imm);
57  return Operand(addr, offset, times_1, offset_imm);
58  }
59  // Offset immediate does not fit in 31 bits.
60  Register scratch = kScratchRegister;
61  assm->movl(scratch, Immediate(offset_imm));
62  if (offset != no_reg) {
63  assm->addq(scratch, offset);
64  }
65  return Operand(addr, scratch, times_1, 0);
66 }
67 
68 inline void Load(LiftoffAssembler* assm, LiftoffRegister dst, Operand src,
69  ValueType type) {
70  switch (type) {
71  case kWasmI32:
72  assm->movl(dst.gp(), src);
73  break;
74  case kWasmI64:
75  assm->movq(dst.gp(), src);
76  break;
77  case kWasmF32:
78  assm->Movss(dst.fp(), src);
79  break;
80  case kWasmF64:
81  assm->Movsd(dst.fp(), src);
82  break;
83  default:
84  UNREACHABLE();
85  }
86 }
87 
88 inline void Store(LiftoffAssembler* assm, Operand dst, LiftoffRegister src,
89  ValueType type) {
90  switch (type) {
91  case kWasmI32:
92  assm->movl(dst, src.gp());
93  break;
94  case kWasmI64:
95  assm->movq(dst, src.gp());
96  break;
97  case kWasmF32:
98  assm->Movss(dst, src.fp());
99  break;
100  case kWasmF64:
101  assm->Movsd(dst, src.fp());
102  break;
103  default:
104  UNREACHABLE();
105  }
106 }
107 
108 inline void push(LiftoffAssembler* assm, LiftoffRegister reg, ValueType type) {
109  switch (type) {
110  case kWasmI32:
111  case kWasmI64:
112  assm->pushq(reg.gp());
113  break;
114  case kWasmF32:
115  assm->subp(rsp, Immediate(kPointerSize));
116  assm->Movss(Operand(rsp, 0), reg.fp());
117  break;
118  case kWasmF64:
119  assm->subp(rsp, Immediate(kPointerSize));
120  assm->Movsd(Operand(rsp, 0), reg.fp());
121  break;
122  default:
123  UNREACHABLE();
124  }
125 }
126 
127 template <typename... Regs>
128 inline void SpillRegisters(LiftoffAssembler* assm, Regs... regs) {
129  for (LiftoffRegister r : {LiftoffRegister(regs)...}) {
130  if (assm->cache_state()->is_used(r)) assm->SpillRegister(r);
131  }
132 }
133 
134 } // namespace liftoff
135 
136 int LiftoffAssembler::PrepareStackFrame() {
137  int offset = pc_offset();
138  sub_sp_32(0);
139  return offset;
140 }
141 
142 void LiftoffAssembler::PatchPrepareStackFrame(int offset,
143  uint32_t stack_slots) {
144  uint32_t bytes = liftoff::kConstantStackSpace + kStackSlotSize * stack_slots;
145  DCHECK_LE(bytes, kMaxInt);
146  // We can't run out of space, just pass anything big enough to not cause the
147  // assembler to try to grow the buffer.
148  constexpr int kAvailableSpace = 64;
149  Assembler patching_assembler(AssemblerOptions{}, buffer_ + offset,
150  kAvailableSpace);
151  patching_assembler.sub_sp_32(bytes);
152 }
153 
154 void LiftoffAssembler::FinishCode() {}
155 
156 void LiftoffAssembler::AbortCompilation() {}
157 
158 void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
159  RelocInfo::Mode rmode) {
160  switch (value.type()) {
161  case kWasmI32:
162  if (value.to_i32() == 0 && RelocInfo::IsNone(rmode)) {
163  xorl(reg.gp(), reg.gp());
164  } else {
165  movl(reg.gp(), Immediate(value.to_i32(), rmode));
166  }
167  break;
168  case kWasmI64:
169  if (RelocInfo::IsNone(rmode)) {
170  TurboAssembler::Set(reg.gp(), value.to_i64());
171  } else {
172  movq(reg.gp(), value.to_i64(), rmode);
173  }
174  break;
175  case kWasmF32:
176  TurboAssembler::Move(reg.fp(), value.to_f32_boxed().get_bits());
177  break;
178  case kWasmF64:
179  TurboAssembler::Move(reg.fp(), value.to_f64_boxed().get_bits());
180  break;
181  default:
182  UNREACHABLE();
183  }
184 }
185 
186 void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset,
187  int size) {
188  DCHECK_LE(offset, kMaxInt);
189  movp(dst, liftoff::GetInstanceOperand());
190  DCHECK(size == 4 || size == 8);
191  if (size == 4) {
192  movl(dst, Operand(dst, offset));
193  } else {
194  movq(dst, Operand(dst, offset));
195  }
196 }
197 
198 void LiftoffAssembler::SpillInstance(Register instance) {
199  movp(liftoff::GetInstanceOperand(), instance);
200 }
201 
202 void LiftoffAssembler::FillInstanceInto(Register dst) {
203  movp(dst, liftoff::GetInstanceOperand());
204 }
205 
206 void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr,
207  Register offset_reg, uint32_t offset_imm,
208  LoadType type, LiftoffRegList pinned,
209  uint32_t* protected_load_pc, bool is_load_mem) {
210  if (emit_debug_code() && offset_reg != no_reg) {
211  AssertZeroExtended(offset_reg);
212  }
213  Operand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
214  if (protected_load_pc) *protected_load_pc = pc_offset();
215  switch (type.value()) {
216  case LoadType::kI32Load8U:
217  case LoadType::kI64Load8U:
218  movzxbl(dst.gp(), src_op);
219  break;
220  case LoadType::kI32Load8S:
221  movsxbl(dst.gp(), src_op);
222  break;
223  case LoadType::kI64Load8S:
224  movsxbq(dst.gp(), src_op);
225  break;
226  case LoadType::kI32Load16U:
227  case LoadType::kI64Load16U:
228  movzxwl(dst.gp(), src_op);
229  break;
230  case LoadType::kI32Load16S:
231  movsxwl(dst.gp(), src_op);
232  break;
233  case LoadType::kI64Load16S:
234  movsxwq(dst.gp(), src_op);
235  break;
236  case LoadType::kI32Load:
237  case LoadType::kI64Load32U:
238  movl(dst.gp(), src_op);
239  break;
240  case LoadType::kI64Load32S:
241  movsxlq(dst.gp(), src_op);
242  break;
243  case LoadType::kI64Load:
244  movq(dst.gp(), src_op);
245  break;
246  case LoadType::kF32Load:
247  Movss(dst.fp(), src_op);
248  break;
249  case LoadType::kF64Load:
250  Movsd(dst.fp(), src_op);
251  break;
252  default:
253  UNREACHABLE();
254  }
255 }
256 
257 void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
258  uint32_t offset_imm, LiftoffRegister src,
259  StoreType type, LiftoffRegList /* pinned */,
260  uint32_t* protected_store_pc, bool is_store_mem) {
261  if (emit_debug_code() && offset_reg != no_reg) {
262  AssertZeroExtended(offset_reg);
263  }
264  Operand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
265  if (protected_store_pc) *protected_store_pc = pc_offset();
266  switch (type.value()) {
267  case StoreType::kI32Store8:
268  case StoreType::kI64Store8:
269  movb(dst_op, src.gp());
270  break;
271  case StoreType::kI32Store16:
272  case StoreType::kI64Store16:
273  movw(dst_op, src.gp());
274  break;
275  case StoreType::kI32Store:
276  case StoreType::kI64Store32:
277  movl(dst_op, src.gp());
278  break;
279  case StoreType::kI64Store:
280  movq(dst_op, src.gp());
281  break;
282  case StoreType::kF32Store:
283  Movss(dst_op, src.fp());
284  break;
285  case StoreType::kF64Store:
286  Movsd(dst_op, src.fp());
287  break;
288  default:
289  UNREACHABLE();
290  }
291 }
292 
293 void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst,
294  uint32_t caller_slot_idx,
295  ValueType type) {
296  Operand src(rbp, kPointerSize * (caller_slot_idx + 1));
297  liftoff::Load(this, dst, src, type);
298 }
299 
300 void LiftoffAssembler::MoveStackValue(uint32_t dst_index, uint32_t src_index,
301  ValueType type) {
302  DCHECK_NE(dst_index, src_index);
303  if (cache_state_.has_unused_register(kGpReg)) {
304  Fill(LiftoffRegister{kScratchRegister}, src_index, type);
305  Spill(dst_index, LiftoffRegister{kScratchRegister}, type);
306  } else {
307  pushq(liftoff::GetStackSlot(src_index));
308  popq(liftoff::GetStackSlot(dst_index));
309  }
310 }
311 
312 void LiftoffAssembler::Move(Register dst, Register src, ValueType type) {
313  DCHECK_NE(dst, src);
314  if (type == kWasmI32) {
315  movl(dst, src);
316  } else {
317  DCHECK_EQ(kWasmI64, type);
318  movq(dst, src);
319  }
320 }
321 
322 void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src,
323  ValueType type) {
324  DCHECK_NE(dst, src);
325  if (type == kWasmF32) {
326  Movss(dst, src);
327  } else {
328  DCHECK_EQ(kWasmF64, type);
329  Movsd(dst, src);
330  }
331 }
332 
333 void LiftoffAssembler::Spill(uint32_t index, LiftoffRegister reg,
334  ValueType type) {
335  RecordUsedSpillSlot(index);
336  Operand dst = liftoff::GetStackSlot(index);
337  switch (type) {
338  case kWasmI32:
339  movl(dst, reg.gp());
340  break;
341  case kWasmI64:
342  movq(dst, reg.gp());
343  break;
344  case kWasmF32:
345  Movss(dst, reg.fp());
346  break;
347  case kWasmF64:
348  Movsd(dst, reg.fp());
349  break;
350  default:
351  UNREACHABLE();
352  }
353 }
354 
355 void LiftoffAssembler::Spill(uint32_t index, WasmValue value) {
356  RecordUsedSpillSlot(index);
357  Operand dst = liftoff::GetStackSlot(index);
358  switch (value.type()) {
359  case kWasmI32:
360  movl(dst, Immediate(value.to_i32()));
361  break;
362  case kWasmI64: {
363  if (is_int32(value.to_i64())) {
364  // Sign extend low word.
365  movq(dst, Immediate(static_cast<int32_t>(value.to_i64())));
366  } else if (is_uint32(value.to_i64())) {
367  // Zero extend low word.
368  movl(kScratchRegister, Immediate(static_cast<int32_t>(value.to_i64())));
369  movq(dst, kScratchRegister);
370  } else {
371  movq(kScratchRegister, value.to_i64());
372  movq(dst, kScratchRegister);
373  }
374  break;
375  }
376  default:
377  // We do not track f32 and f64 constants, hence they are unreachable.
378  UNREACHABLE();
379  }
380 }
381 
382 void LiftoffAssembler::Fill(LiftoffRegister reg, uint32_t index,
383  ValueType type) {
384  Operand src = liftoff::GetStackSlot(index);
385  switch (type) {
386  case kWasmI32:
387  movl(reg.gp(), src);
388  break;
389  case kWasmI64:
390  movq(reg.gp(), src);
391  break;
392  case kWasmF32:
393  Movss(reg.fp(), src);
394  break;
395  case kWasmF64:
396  Movsd(reg.fp(), src);
397  break;
398  default:
399  UNREACHABLE();
400  }
401 }
402 
403 void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) {
404  UNREACHABLE();
405 }
406 
407 void LiftoffAssembler::emit_i32_add(Register dst, Register lhs, Register rhs) {
408  if (lhs != dst) {
409  leal(dst, Operand(lhs, rhs, times_1, 0));
410  } else {
411  addl(dst, rhs);
412  }
413 }
414 
415 void LiftoffAssembler::emit_i32_sub(Register dst, Register lhs, Register rhs) {
416  if (dst == rhs) {
417  negl(dst);
418  addl(dst, lhs);
419  } else {
420  if (dst != lhs) movl(dst, lhs);
421  subl(dst, rhs);
422  }
423 }
424 
425 namespace liftoff {
426 template <void (Assembler::*op)(Register, Register),
427  void (Assembler::*mov)(Register, Register)>
428 void EmitCommutativeBinOp(LiftoffAssembler* assm, Register dst, Register lhs,
429  Register rhs) {
430  if (dst == rhs) {
431  (assm->*op)(dst, lhs);
432  } else {
433  if (dst != lhs) (assm->*mov)(dst, lhs);
434  (assm->*op)(dst, rhs);
435  }
436 }
437 } // namespace liftoff
438 
439 void LiftoffAssembler::emit_i32_mul(Register dst, Register lhs, Register rhs) {
440  liftoff::EmitCommutativeBinOp<&Assembler::imull, &Assembler::movl>(this, dst,
441  lhs, rhs);
442 }
443 
444 namespace liftoff {
445 enum class DivOrRem : uint8_t { kDiv, kRem };
446 template <typename type, DivOrRem div_or_rem>
447 void EmitIntDivOrRem(LiftoffAssembler* assm, Register dst, Register lhs,
448  Register rhs, Label* trap_div_by_zero,
449  Label* trap_div_unrepresentable) {
450  constexpr bool needs_unrepresentable_check =
451  std::is_signed<type>::value && div_or_rem == DivOrRem::kDiv;
452  constexpr bool special_case_minus_1 =
453  std::is_signed<type>::value && div_or_rem == DivOrRem::kRem;
454  DCHECK_EQ(needs_unrepresentable_check, trap_div_unrepresentable != nullptr);
455 
456 #define iop(name, ...) \
457  do { \
458  if (sizeof(type) == 4) { \
459  assm->name##l(__VA_ARGS__); \
460  } else { \
461  assm->name##q(__VA_ARGS__); \
462  } \
463  } while (false)
464 
465  // For division, the lhs is always taken from {edx:eax}. Thus, make sure that
466  // these registers are unused. If {rhs} is stored in one of them, move it to
467  // another temporary register.
468  // Do all this before any branch, such that the code is executed
469  // unconditionally, as the cache state will also be modified unconditionally.
470  liftoff::SpillRegisters(assm, rdx, rax);
471  if (rhs == rax || rhs == rdx) {
472  iop(mov, kScratchRegister, rhs);
473  rhs = kScratchRegister;
474  }
475 
476  // Check for division by zero.
477  iop(test, rhs, rhs);
478  assm->j(zero, trap_div_by_zero);
479 
480  Label done;
481  if (needs_unrepresentable_check) {
482  // Check for {kMinInt / -1}. This is unrepresentable.
483  Label do_div;
484  iop(cmp, rhs, Immediate(-1));
485  assm->j(not_equal, &do_div);
486  // {lhs} is min int if {lhs - 1} overflows.
487  iop(cmp, lhs, Immediate(1));
488  assm->j(overflow, trap_div_unrepresentable);
489  assm->bind(&do_div);
490  } else if (special_case_minus_1) {
491  // {lhs % -1} is always 0 (needs to be special cased because {kMinInt / -1}
492  // cannot be computed).
493  Label do_rem;
494  iop(cmp, rhs, Immediate(-1));
495  assm->j(not_equal, &do_rem);
496  // clang-format off
497  // (conflicts with presubmit checks because it is confused about "xor")
498  iop(xor, dst, dst);
499  // clang-format on
500  assm->jmp(&done);
501  assm->bind(&do_rem);
502  }
503 
504  // Now move {lhs} into {eax}, then zero-extend or sign-extend into {edx}, then
505  // do the division.
506  if (lhs != rax) iop(mov, rax, lhs);
507  if (std::is_same<int32_t, type>::value) { // i32
508  assm->cdq();
509  assm->idivl(rhs);
510  } else if (std::is_same<uint32_t, type>::value) { // u32
511  assm->xorl(rdx, rdx);
512  assm->divl(rhs);
513  } else if (std::is_same<int64_t, type>::value) { // i64
514  assm->cqo();
515  assm->idivq(rhs);
516  } else { // u64
517  assm->xorq(rdx, rdx);
518  assm->divq(rhs);
519  }
520 
521  // Move back the result (in {eax} or {edx}) into the {dst} register.
522  constexpr Register kResultReg = div_or_rem == DivOrRem::kDiv ? rax : rdx;
523  if (dst != kResultReg) {
524  iop(mov, dst, kResultReg);
525  }
526  if (special_case_minus_1) assm->bind(&done);
527 }
528 } // namespace liftoff
529 
530 void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs,
531  Label* trap_div_by_zero,
532  Label* trap_div_unrepresentable) {
533  liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kDiv>(
534  this, dst, lhs, rhs, trap_div_by_zero, trap_div_unrepresentable);
535 }
536 
537 void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs,
538  Label* trap_div_by_zero) {
539  liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kDiv>(
540  this, dst, lhs, rhs, trap_div_by_zero, nullptr);
541 }
542 
543 void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs,
544  Label* trap_div_by_zero) {
545  liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kRem>(
546  this, dst, lhs, rhs, trap_div_by_zero, nullptr);
547 }
548 
549 void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs,
550  Label* trap_div_by_zero) {
551  liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kRem>(
552  this, dst, lhs, rhs, trap_div_by_zero, nullptr);
553 }
554 
555 void LiftoffAssembler::emit_i32_and(Register dst, Register lhs, Register rhs) {
556  liftoff::EmitCommutativeBinOp<&Assembler::andl, &Assembler::movl>(this, dst,
557  lhs, rhs);
558 }
559 
560 void LiftoffAssembler::emit_i32_or(Register dst, Register lhs, Register rhs) {
561  liftoff::EmitCommutativeBinOp<&Assembler::orl, &Assembler::movl>(this, dst,
562  lhs, rhs);
563 }
564 
565 void LiftoffAssembler::emit_i32_xor(Register dst, Register lhs, Register rhs) {
566  liftoff::EmitCommutativeBinOp<&Assembler::xorl, &Assembler::movl>(this, dst,
567  lhs, rhs);
568 }
569 
570 namespace liftoff {
571 template <ValueType type>
572 inline void EmitShiftOperation(LiftoffAssembler* assm, Register dst,
573  Register src, Register amount,
574  void (Assembler::*emit_shift)(Register),
575  LiftoffRegList pinned) {
576  // If dst is rcx, compute into the scratch register first, then move to rcx.
577  if (dst == rcx) {
578  assm->Move(kScratchRegister, src, type);
579  if (amount != rcx) assm->Move(rcx, amount, type);
580  (assm->*emit_shift)(kScratchRegister);
581  assm->Move(rcx, kScratchRegister, type);
582  return;
583  }
584 
585  // Move amount into rcx. If rcx is in use, move its content into the scratch
586  // register. If src is rcx, src is now the scratch register.
587  bool use_scratch = false;
588  if (amount != rcx) {
589  use_scratch = src == rcx ||
590  assm->cache_state()->is_used(LiftoffRegister(rcx)) ||
591  pinned.has(LiftoffRegister(rcx));
592  if (use_scratch) assm->movq(kScratchRegister, rcx);
593  if (src == rcx) src = kScratchRegister;
594  assm->Move(rcx, amount, type);
595  }
596 
597  // Do the actual shift.
598  if (dst != src) assm->Move(dst, src, type);
599  (assm->*emit_shift)(dst);
600 
601  // Restore rcx if needed.
602  if (use_scratch) assm->movq(rcx, kScratchRegister);
603 }
604 } // namespace liftoff
605 
606 void LiftoffAssembler::emit_i32_shl(Register dst, Register src, Register amount,
607  LiftoffRegList pinned) {
608  liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
609  &Assembler::shll_cl, pinned);
610 }
611 
612 void LiftoffAssembler::emit_i32_sar(Register dst, Register src, Register amount,
613  LiftoffRegList pinned) {
614  liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
615  &Assembler::sarl_cl, pinned);
616 }
617 
618 void LiftoffAssembler::emit_i32_shr(Register dst, Register src, Register amount,
619  LiftoffRegList pinned) {
620  liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
621  &Assembler::shrl_cl, pinned);
622 }
623 
624 void LiftoffAssembler::emit_i32_shr(Register dst, Register src, int amount) {
625  if (dst != src) movl(dst, src);
626  DCHECK(is_uint5(amount));
627  shrl(dst, Immediate(amount));
628 }
629 
630 bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) {
631  Label nonzero_input;
632  Label continuation;
633  testl(src, src);
634  j(not_zero, &nonzero_input, Label::kNear);
635  movl(dst, Immediate(32));
636  jmp(&continuation, Label::kNear);
637 
638  bind(&nonzero_input);
639  // Get most significant bit set (MSBS).
640  bsrl(dst, src);
641  // CLZ = 31 - MSBS = MSBS ^ 31.
642  xorl(dst, Immediate(31));
643 
644  bind(&continuation);
645  return true;
646 }
647 
648 bool LiftoffAssembler::emit_i32_ctz(Register dst, Register src) {
649  Label nonzero_input;
650  Label continuation;
651  testl(src, src);
652  j(not_zero, &nonzero_input, Label::kNear);
653  movl(dst, Immediate(32));
654  jmp(&continuation, Label::kNear);
655 
656  bind(&nonzero_input);
657  // Get least significant bit set, which equals number of trailing zeros.
658  bsfl(dst, src);
659 
660  bind(&continuation);
661  return true;
662 }
663 
664 bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) {
665  if (!CpuFeatures::IsSupported(POPCNT)) return false;
666  CpuFeatureScope scope(this, POPCNT);
667  popcntl(dst, src);
668  return true;
669 }
670 
671 void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs,
672  LiftoffRegister rhs) {
673  if (lhs.gp() != dst.gp()) {
674  leap(dst.gp(), Operand(lhs.gp(), rhs.gp(), times_1, 0));
675  } else {
676  addp(dst.gp(), rhs.gp());
677  }
678 }
679 
680 void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs,
681  LiftoffRegister rhs) {
682  if (dst.gp() == rhs.gp()) {
683  negq(dst.gp());
684  addq(dst.gp(), lhs.gp());
685  } else {
686  if (dst.gp() != lhs.gp()) movq(dst.gp(), lhs.gp());
687  subq(dst.gp(), rhs.gp());
688  }
689 }
690 
691 void LiftoffAssembler::emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs,
692  LiftoffRegister rhs) {
693  liftoff::EmitCommutativeBinOp<&Assembler::imulq, &Assembler::movq>(
694  this, dst.gp(), lhs.gp(), rhs.gp());
695 }
696 
697 bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs,
698  LiftoffRegister rhs,
699  Label* trap_div_by_zero,
700  Label* trap_div_unrepresentable) {
701  liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kDiv>(
702  this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero,
703  trap_div_unrepresentable);
704  return true;
705 }
706 
707 bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs,
708  LiftoffRegister rhs,
709  Label* trap_div_by_zero) {
710  liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kDiv>(
711  this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
712  return true;
713 }
714 
715 bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs,
716  LiftoffRegister rhs,
717  Label* trap_div_by_zero) {
718  liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kRem>(
719  this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
720  return true;
721 }
722 
723 bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs,
724  LiftoffRegister rhs,
725  Label* trap_div_by_zero) {
726  liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kRem>(
727  this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
728  return true;
729 }
730 
731 void LiftoffAssembler::emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs,
732  LiftoffRegister rhs) {
733  liftoff::EmitCommutativeBinOp<&Assembler::andq, &Assembler::movq>(
734  this, dst.gp(), lhs.gp(), rhs.gp());
735 }
736 
737 void LiftoffAssembler::emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs,
738  LiftoffRegister rhs) {
739  liftoff::EmitCommutativeBinOp<&Assembler::orq, &Assembler::movq>(
740  this, dst.gp(), lhs.gp(), rhs.gp());
741 }
742 
743 void LiftoffAssembler::emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs,
744  LiftoffRegister rhs) {
745  liftoff::EmitCommutativeBinOp<&Assembler::xorq, &Assembler::movq>(
746  this, dst.gp(), lhs.gp(), rhs.gp());
747 }
748 
749 void LiftoffAssembler::emit_i64_shl(LiftoffRegister dst, LiftoffRegister src,
750  Register amount, LiftoffRegList pinned) {
751  liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
752  &Assembler::shlq_cl, pinned);
753 }
754 
755 void LiftoffAssembler::emit_i64_sar(LiftoffRegister dst, LiftoffRegister src,
756  Register amount, LiftoffRegList pinned) {
757  liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
758  &Assembler::sarq_cl, pinned);
759 }
760 
761 void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src,
762  Register amount, LiftoffRegList pinned) {
763  liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
764  &Assembler::shrq_cl, pinned);
765 }
766 
767 void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src,
768  int amount) {
769  if (dst.gp() != src.gp()) movl(dst.gp(), src.gp());
770  DCHECK(is_uint6(amount));
771  shrq(dst.gp(), Immediate(amount));
772 }
773 
774 void LiftoffAssembler::emit_i32_to_intptr(Register dst, Register src) {
775  movsxlq(dst, src);
776 }
777 
778 void LiftoffAssembler::emit_f32_add(DoubleRegister dst, DoubleRegister lhs,
779  DoubleRegister rhs) {
780  if (CpuFeatures::IsSupported(AVX)) {
781  CpuFeatureScope scope(this, AVX);
782  vaddss(dst, lhs, rhs);
783  } else if (dst == rhs) {
784  addss(dst, lhs);
785  } else {
786  if (dst != lhs) movss(dst, lhs);
787  addss(dst, rhs);
788  }
789 }
790 
791 void LiftoffAssembler::emit_f32_sub(DoubleRegister dst, DoubleRegister lhs,
792  DoubleRegister rhs) {
793  if (CpuFeatures::IsSupported(AVX)) {
794  CpuFeatureScope scope(this, AVX);
795  vsubss(dst, lhs, rhs);
796  } else if (dst == rhs) {
797  movss(kScratchDoubleReg, rhs);
798  movss(dst, lhs);
799  subss(dst, kScratchDoubleReg);
800  } else {
801  if (dst != lhs) movss(dst, lhs);
802  subss(dst, rhs);
803  }
804 }
805 
806 void LiftoffAssembler::emit_f32_mul(DoubleRegister dst, DoubleRegister lhs,
807  DoubleRegister rhs) {
808  if (CpuFeatures::IsSupported(AVX)) {
809  CpuFeatureScope scope(this, AVX);
810  vmulss(dst, lhs, rhs);
811  } else if (dst == rhs) {
812  mulss(dst, lhs);
813  } else {
814  if (dst != lhs) movss(dst, lhs);
815  mulss(dst, rhs);
816  }
817 }
818 
819 void LiftoffAssembler::emit_f32_div(DoubleRegister dst, DoubleRegister lhs,
820  DoubleRegister rhs) {
821  if (CpuFeatures::IsSupported(AVX)) {
822  CpuFeatureScope scope(this, AVX);
823  vdivss(dst, lhs, rhs);
824  } else if (dst == rhs) {
825  movss(kScratchDoubleReg, rhs);
826  movss(dst, lhs);
827  divss(dst, kScratchDoubleReg);
828  } else {
829  if (dst != lhs) movss(dst, lhs);
830  divss(dst, rhs);
831  }
832 }
833 
834 namespace liftoff {
835 enum class MinOrMax : uint8_t { kMin, kMax };
836 template <typename type>
837 inline void EmitFloatMinOrMax(LiftoffAssembler* assm, DoubleRegister dst,
838  DoubleRegister lhs, DoubleRegister rhs,
839  MinOrMax min_or_max) {
840  Label is_nan;
841  Label lhs_below_rhs;
842  Label lhs_above_rhs;
843  Label done;
844 
845 #define dop(name, ...) \
846  do { \
847  if (sizeof(type) == 4) { \
848  assm->name##s(__VA_ARGS__); \
849  } else { \
850  assm->name##d(__VA_ARGS__); \
851  } \
852  } while (false)
853 
854  // Check the easy cases first: nan (e.g. unordered), smaller and greater.
855  // NaN has to be checked first, because PF=1 implies CF=1.
856  dop(Ucomis, lhs, rhs);
857  assm->j(parity_even, &is_nan, Label::kNear); // PF=1
858  assm->j(below, &lhs_below_rhs, Label::kNear); // CF=1
859  assm->j(above, &lhs_above_rhs, Label::kNear); // CF=0 && ZF=0
860 
861  // If we get here, then either
862  // a) {lhs == rhs},
863  // b) {lhs == -0.0} and {rhs == 0.0}, or
864  // c) {lhs == 0.0} and {rhs == -0.0}.
865  // For a), it does not matter whether we return {lhs} or {rhs}. Check the sign
866  // bit of {rhs} to differentiate b) and c).
867  dop(Movmskp, kScratchRegister, rhs);
868  assm->testl(kScratchRegister, Immediate(1));
869  assm->j(zero, &lhs_below_rhs, Label::kNear);
870  assm->jmp(&lhs_above_rhs, Label::kNear);
871 
872  assm->bind(&is_nan);
873  // Create a NaN output.
874  dop(Xorp, dst, dst);
875  dop(Divs, dst, dst);
876  assm->jmp(&done, Label::kNear);
877 
878  assm->bind(&lhs_below_rhs);
879  DoubleRegister lhs_below_rhs_src = min_or_max == MinOrMax::kMin ? lhs : rhs;
880  if (dst != lhs_below_rhs_src) dop(Movs, dst, lhs_below_rhs_src);
881  assm->jmp(&done, Label::kNear);
882 
883  assm->bind(&lhs_above_rhs);
884  DoubleRegister lhs_above_rhs_src = min_or_max == MinOrMax::kMin ? rhs : lhs;
885  if (dst != lhs_above_rhs_src) dop(Movs, dst, lhs_above_rhs_src);
886 
887  assm->bind(&done);
888 }
889 } // namespace liftoff
890 
891 void LiftoffAssembler::emit_f32_min(DoubleRegister dst, DoubleRegister lhs,
892  DoubleRegister rhs) {
893  liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs,
894  liftoff::MinOrMax::kMin);
895 }
896 
897 void LiftoffAssembler::emit_f32_max(DoubleRegister dst, DoubleRegister lhs,
898  DoubleRegister rhs) {
899  liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs,
900  liftoff::MinOrMax::kMax);
901 }
902 
903 void LiftoffAssembler::emit_f32_copysign(DoubleRegister dst, DoubleRegister lhs,
904  DoubleRegister rhs) {
905  static constexpr int kF32SignBit = 1 << 31;
906  Movd(kScratchRegister, lhs);
907  andl(kScratchRegister, Immediate(~kF32SignBit));
908  Movd(liftoff::kScratchRegister2, rhs);
909  andl(liftoff::kScratchRegister2, Immediate(kF32SignBit));
910  orl(kScratchRegister, liftoff::kScratchRegister2);
911  Movd(dst, kScratchRegister);
912 }
913 
914 void LiftoffAssembler::emit_f32_abs(DoubleRegister dst, DoubleRegister src) {
915  static constexpr uint32_t kSignBit = uint32_t{1} << 31;
916  if (dst == src) {
917  TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1);
918  Andps(dst, kScratchDoubleReg);
919  } else {
920  TurboAssembler::Move(dst, kSignBit - 1);
921  Andps(dst, src);
922  }
923 }
924 
925 void LiftoffAssembler::emit_f32_neg(DoubleRegister dst, DoubleRegister src) {
926  static constexpr uint32_t kSignBit = uint32_t{1} << 31;
927  if (dst == src) {
928  TurboAssembler::Move(kScratchDoubleReg, kSignBit);
929  Xorps(dst, kScratchDoubleReg);
930  } else {
931  TurboAssembler::Move(dst, kSignBit);
932  Xorps(dst, src);
933  }
934 }
935 
936 bool LiftoffAssembler::emit_f32_ceil(DoubleRegister dst, DoubleRegister src) {
937  if (CpuFeatures::IsSupported(SSE4_1)) {
938  CpuFeatureScope feature(this, SSE4_1);
939  Roundss(dst, src, kRoundUp);
940  return true;
941  }
942  return false;
943 }
944 
945 bool LiftoffAssembler::emit_f32_floor(DoubleRegister dst, DoubleRegister src) {
946  if (CpuFeatures::IsSupported(SSE4_1)) {
947  CpuFeatureScope feature(this, SSE4_1);
948  Roundss(dst, src, kRoundDown);
949  return true;
950  }
951  return false;
952 }
953 
954 bool LiftoffAssembler::emit_f32_trunc(DoubleRegister dst, DoubleRegister src) {
955  if (CpuFeatures::IsSupported(SSE4_1)) {
956  CpuFeatureScope feature(this, SSE4_1);
957  Roundss(dst, src, kRoundToZero);
958  return true;
959  }
960  return false;
961 }
962 
963 bool LiftoffAssembler::emit_f32_nearest_int(DoubleRegister dst,
964  DoubleRegister src) {
965  if (CpuFeatures::IsSupported(SSE4_1)) {
966  CpuFeatureScope feature(this, SSE4_1);
967  Roundss(dst, src, kRoundToNearest);
968  return true;
969  }
970  return false;
971 }
972 
973 void LiftoffAssembler::emit_f32_sqrt(DoubleRegister dst, DoubleRegister src) {
974  Sqrtss(dst, src);
975 }
976 
977 void LiftoffAssembler::emit_f64_add(DoubleRegister dst, DoubleRegister lhs,
978  DoubleRegister rhs) {
979  if (CpuFeatures::IsSupported(AVX)) {
980  CpuFeatureScope scope(this, AVX);
981  vaddsd(dst, lhs, rhs);
982  } else if (dst == rhs) {
983  addsd(dst, lhs);
984  } else {
985  if (dst != lhs) movsd(dst, lhs);
986  addsd(dst, rhs);
987  }
988 }
989 
990 void LiftoffAssembler::emit_f64_sub(DoubleRegister dst, DoubleRegister lhs,
991  DoubleRegister rhs) {
992  if (CpuFeatures::IsSupported(AVX)) {
993  CpuFeatureScope scope(this, AVX);
994  vsubsd(dst, lhs, rhs);
995  } else if (dst == rhs) {
996  movsd(kScratchDoubleReg, rhs);
997  movsd(dst, lhs);
998  subsd(dst, kScratchDoubleReg);
999  } else {
1000  if (dst != lhs) movsd(dst, lhs);
1001  subsd(dst, rhs);
1002  }
1003 }
1004 
1005 void LiftoffAssembler::emit_f64_mul(DoubleRegister dst, DoubleRegister lhs,
1006  DoubleRegister rhs) {
1007  if (CpuFeatures::IsSupported(AVX)) {
1008  CpuFeatureScope scope(this, AVX);
1009  vmulsd(dst, lhs, rhs);
1010  } else if (dst == rhs) {
1011  mulsd(dst, lhs);
1012  } else {
1013  if (dst != lhs) movsd(dst, lhs);
1014  mulsd(dst, rhs);
1015  }
1016 }
1017 
1018 void LiftoffAssembler::emit_f64_div(DoubleRegister dst, DoubleRegister lhs,
1019  DoubleRegister rhs) {
1020  if (CpuFeatures::IsSupported(AVX)) {
1021  CpuFeatureScope scope(this, AVX);
1022  vdivsd(dst, lhs, rhs);
1023  } else if (dst == rhs) {
1024  movsd(kScratchDoubleReg, rhs);
1025  movsd(dst, lhs);
1026  divsd(dst, kScratchDoubleReg);
1027  } else {
1028  if (dst != lhs) movsd(dst, lhs);
1029  divsd(dst, rhs);
1030  }
1031 }
1032 
1033 void LiftoffAssembler::emit_f64_min(DoubleRegister dst, DoubleRegister lhs,
1034  DoubleRegister rhs) {
1035  liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs,
1036  liftoff::MinOrMax::kMin);
1037 }
1038 
1039 void LiftoffAssembler::emit_f64_copysign(DoubleRegister dst, DoubleRegister lhs,
1040  DoubleRegister rhs) {
1041  // Extract sign bit from {rhs} into {kScratchRegister2}.
1042  Movq(liftoff::kScratchRegister2, rhs);
1043  shrq(liftoff::kScratchRegister2, Immediate(63));
1044  shlq(liftoff::kScratchRegister2, Immediate(63));
1045  // Reset sign bit of {lhs} (in {kScratchRegister}).
1046  Movq(kScratchRegister, lhs);
1047  btrq(kScratchRegister, Immediate(63));
1048  // Combine both values into {kScratchRegister} and move into {dst}.
1049  orq(kScratchRegister, liftoff::kScratchRegister2);
1050  Movq(dst, kScratchRegister);
1051 }
1052 
1053 void LiftoffAssembler::emit_f64_max(DoubleRegister dst, DoubleRegister lhs,
1054  DoubleRegister rhs) {
1055  liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs,
1056  liftoff::MinOrMax::kMax);
1057 }
1058 
1059 void LiftoffAssembler::emit_f64_abs(DoubleRegister dst, DoubleRegister src) {
1060  static constexpr uint64_t kSignBit = uint64_t{1} << 63;
1061  if (dst == src) {
1062  TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1);
1063  Andpd(dst, kScratchDoubleReg);
1064  } else {
1065  TurboAssembler::Move(dst, kSignBit - 1);
1066  Andpd(dst, src);
1067  }
1068 }
1069 
1070 void LiftoffAssembler::emit_f64_neg(DoubleRegister dst, DoubleRegister src) {
1071  static constexpr uint64_t kSignBit = uint64_t{1} << 63;
1072  if (dst == src) {
1073  TurboAssembler::Move(kScratchDoubleReg, kSignBit);
1074  Xorpd(dst, kScratchDoubleReg);
1075  } else {
1076  TurboAssembler::Move(dst, kSignBit);
1077  Xorpd(dst, src);
1078  }
1079 }
1080 
1081 bool LiftoffAssembler::emit_f64_ceil(DoubleRegister dst, DoubleRegister src) {
1082  REQUIRE_CPU_FEATURE(SSE4_1, true);
1083  Roundsd(dst, src, kRoundUp);
1084  return true;
1085 }
1086 
1087 bool LiftoffAssembler::emit_f64_floor(DoubleRegister dst, DoubleRegister src) {
1088  REQUIRE_CPU_FEATURE(SSE4_1, true);
1089  Roundsd(dst, src, kRoundDown);
1090  return true;
1091 }
1092 
1093 bool LiftoffAssembler::emit_f64_trunc(DoubleRegister dst, DoubleRegister src) {
1094  REQUIRE_CPU_FEATURE(SSE4_1, true);
1095  Roundsd(dst, src, kRoundToZero);
1096  return true;
1097 }
1098 
1099 bool LiftoffAssembler::emit_f64_nearest_int(DoubleRegister dst,
1100  DoubleRegister src) {
1101  REQUIRE_CPU_FEATURE(SSE4_1, true);
1102  Roundsd(dst, src, kRoundToNearest);
1103  return true;
1104 }
1105 
1106 void LiftoffAssembler::emit_f64_sqrt(DoubleRegister dst, DoubleRegister src) {
1107  Sqrtsd(dst, src);
1108 }
1109 
1110 namespace liftoff {
1111 // Used for float to int conversions. If the value in {converted_back} equals
1112 // {src} afterwards, the conversion succeeded.
1113 template <typename dst_type, typename src_type>
1114 inline void ConvertFloatToIntAndBack(LiftoffAssembler* assm, Register dst,
1115  DoubleRegister src,
1116  DoubleRegister converted_back) {
1117  if (std::is_same<double, src_type>::value) { // f64
1118  if (std::is_same<int32_t, dst_type>::value) { // f64 -> i32
1119  assm->Cvttsd2si(dst, src);
1120  assm->Cvtlsi2sd(converted_back, dst);
1121  } else if (std::is_same<uint32_t, dst_type>::value) { // f64 -> u32
1122  assm->Cvttsd2siq(dst, src);
1123  assm->movl(dst, dst);
1124  assm->Cvtqsi2sd(converted_back, dst);
1125  } else if (std::is_same<int64_t, dst_type>::value) { // f64 -> i64
1126  assm->Cvttsd2siq(dst, src);
1127  assm->Cvtqsi2sd(converted_back, dst);
1128  } else {
1129  UNREACHABLE();
1130  }
1131  } else { // f32
1132  if (std::is_same<int32_t, dst_type>::value) { // f32 -> i32
1133  assm->Cvttss2si(dst, src);
1134  assm->Cvtlsi2ss(converted_back, dst);
1135  } else if (std::is_same<uint32_t, dst_type>::value) { // f32 -> u32
1136  assm->Cvttss2siq(dst, src);
1137  assm->movl(dst, dst);
1138  assm->Cvtqsi2ss(converted_back, dst);
1139  } else if (std::is_same<int64_t, dst_type>::value) { // f32 -> i64
1140  assm->Cvttss2siq(dst, src);
1141  assm->Cvtqsi2ss(converted_back, dst);
1142  } else {
1143  UNREACHABLE();
1144  }
1145  }
1146 }
1147 
1148 template <typename dst_type, typename src_type>
1149 inline bool EmitTruncateFloatToInt(LiftoffAssembler* assm, Register dst,
1150  DoubleRegister src, Label* trap) {
1151  if (!CpuFeatures::IsSupported(SSE4_1)) {
1152  assm->bailout("no SSE4.1");
1153  return true;
1154  }
1155  CpuFeatureScope feature(assm, SSE4_1);
1156 
1157  DoubleRegister rounded = kScratchDoubleReg;
1158  DoubleRegister converted_back = kScratchDoubleReg2;
1159 
1160  if (std::is_same<double, src_type>::value) { // f64
1161  assm->Roundsd(rounded, src, kRoundToZero);
1162  } else { // f32
1163  assm->Roundss(rounded, src, kRoundToZero);
1164  }
1165  ConvertFloatToIntAndBack<dst_type, src_type>(assm, dst, rounded,
1166  converted_back);
1167  if (std::is_same<double, src_type>::value) { // f64
1168  assm->Ucomisd(converted_back, rounded);
1169  } else { // f32
1170  assm->Ucomiss(converted_back, rounded);
1171  }
1172 
1173  // Jump to trap if PF is 0 (one of the operands was NaN) or they are not
1174  // equal.
1175  assm->j(parity_even, trap);
1176  assm->j(not_equal, trap);
1177  return true;
1178 }
1179 } // namespace liftoff
1180 
1181 bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
1182  LiftoffRegister dst,
1183  LiftoffRegister src, Label* trap) {
1184  switch (opcode) {
1185  case kExprI32ConvertI64:
1186  movl(dst.gp(), src.gp());
1187  return true;
1188  case kExprI32SConvertF32:
1189  return liftoff::EmitTruncateFloatToInt<int32_t, float>(this, dst.gp(),
1190  src.fp(), trap);
1191  case kExprI32UConvertF32:
1192  return liftoff::EmitTruncateFloatToInt<uint32_t, float>(this, dst.gp(),
1193  src.fp(), trap);
1194  case kExprI32SConvertF64:
1195  return liftoff::EmitTruncateFloatToInt<int32_t, double>(this, dst.gp(),
1196  src.fp(), trap);
1197  case kExprI32UConvertF64:
1198  return liftoff::EmitTruncateFloatToInt<uint32_t, double>(this, dst.gp(),
1199  src.fp(), trap);
1200  case kExprI32ReinterpretF32:
1201  Movd(dst.gp(), src.fp());
1202  return true;
1203  case kExprI64SConvertI32:
1204  movsxlq(dst.gp(), src.gp());
1205  return true;
1206  case kExprI64SConvertF32:
1207  return liftoff::EmitTruncateFloatToInt<int64_t, float>(this, dst.gp(),
1208  src.fp(), trap);
1209  case kExprI64UConvertF32: {
1210  REQUIRE_CPU_FEATURE(SSE4_1, true);
1211  Cvttss2uiq(dst.gp(), src.fp(), trap);
1212  return true;
1213  }
1214  case kExprI64SConvertF64:
1215  return liftoff::EmitTruncateFloatToInt<int64_t, double>(this, dst.gp(),
1216  src.fp(), trap);
1217  case kExprI64UConvertF64: {
1218  REQUIRE_CPU_FEATURE(SSE4_1, true);
1219  Cvttsd2uiq(dst.gp(), src.fp(), trap);
1220  return true;
1221  }
1222  case kExprI64UConvertI32:
1223  AssertZeroExtended(src.gp());
1224  if (dst.gp() != src.gp()) movl(dst.gp(), src.gp());
1225  return true;
1226  case kExprI64ReinterpretF64:
1227  Movq(dst.gp(), src.fp());
1228  return true;
1229  case kExprF32SConvertI32:
1230  Cvtlsi2ss(dst.fp(), src.gp());
1231  return true;
1232  case kExprF32UConvertI32:
1233  movl(kScratchRegister, src.gp());
1234  Cvtqsi2ss(dst.fp(), kScratchRegister);
1235  return true;
1236  case kExprF32SConvertI64:
1237  Cvtqsi2ss(dst.fp(), src.gp());
1238  return true;
1239  case kExprF32UConvertI64:
1240  Cvtqui2ss(dst.fp(), src.gp());
1241  return true;
1242  case kExprF32ConvertF64:
1243  Cvtsd2ss(dst.fp(), src.fp());
1244  return true;
1245  case kExprF32ReinterpretI32:
1246  Movd(dst.fp(), src.gp());
1247  return true;
1248  case kExprF64SConvertI32:
1249  Cvtlsi2sd(dst.fp(), src.gp());
1250  return true;
1251  case kExprF64UConvertI32:
1252  movl(kScratchRegister, src.gp());
1253  Cvtqsi2sd(dst.fp(), kScratchRegister);
1254  return true;
1255  case kExprF64SConvertI64:
1256  Cvtqsi2sd(dst.fp(), src.gp());
1257  return true;
1258  case kExprF64UConvertI64:
1259  Cvtqui2sd(dst.fp(), src.gp());
1260  return true;
1261  case kExprF64ConvertF32:
1262  Cvtss2sd(dst.fp(), src.fp());
1263  return true;
1264  case kExprF64ReinterpretI64:
1265  Movq(dst.fp(), src.gp());
1266  return true;
1267  default:
1268  UNREACHABLE();
1269  }
1270 }
1271 
1272 void LiftoffAssembler::emit_i32_signextend_i8(Register dst, Register src) {
1273  movsxbl(dst, src);
1274 }
1275 
1276 void LiftoffAssembler::emit_i32_signextend_i16(Register dst, Register src) {
1277  movsxwl(dst, src);
1278 }
1279 
1280 void LiftoffAssembler::emit_i64_signextend_i8(LiftoffRegister dst,
1281  LiftoffRegister src) {
1282  movsxbq(dst.gp(), src.gp());
1283 }
1284 
1285 void LiftoffAssembler::emit_i64_signextend_i16(LiftoffRegister dst,
1286  LiftoffRegister src) {
1287  movsxwq(dst.gp(), src.gp());
1288 }
1289 
1290 void LiftoffAssembler::emit_i64_signextend_i32(LiftoffRegister dst,
1291  LiftoffRegister src) {
1292  movsxlq(dst.gp(), src.gp());
1293 }
1294 
1295 void LiftoffAssembler::emit_jump(Label* label) { jmp(label); }
1296 
1297 void LiftoffAssembler::emit_jump(Register target) { jmp(target); }
1298 
1299 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
1300  ValueType type, Register lhs,
1301  Register rhs) {
1302  if (rhs != no_reg) {
1303  switch (type) {
1304  case kWasmI32:
1305  cmpl(lhs, rhs);
1306  break;
1307  case kWasmI64:
1308  cmpq(lhs, rhs);
1309  break;
1310  default:
1311  UNREACHABLE();
1312  }
1313  } else {
1314  DCHECK_EQ(type, kWasmI32);
1315  testl(lhs, lhs);
1316  }
1317 
1318  j(cond, label);
1319 }
1320 
1321 void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) {
1322  testl(src, src);
1323  setcc(equal, dst);
1324  movzxbl(dst, dst);
1325 }
1326 
1327 void LiftoffAssembler::emit_i32_set_cond(Condition cond, Register dst,
1328  Register lhs, Register rhs) {
1329  cmpl(lhs, rhs);
1330  setcc(cond, dst);
1331  movzxbl(dst, dst);
1332 }
1333 
1334 void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) {
1335  testq(src.gp(), src.gp());
1336  setcc(equal, dst);
1337  movzxbl(dst, dst);
1338 }
1339 
1340 void LiftoffAssembler::emit_i64_set_cond(Condition cond, Register dst,
1341  LiftoffRegister lhs,
1342  LiftoffRegister rhs) {
1343  cmpq(lhs.gp(), rhs.gp());
1344  setcc(cond, dst);
1345  movzxbl(dst, dst);
1346 }
1347 
1348 namespace liftoff {
1349 template <void (TurboAssembler::*cmp_op)(DoubleRegister, DoubleRegister)>
1350 void EmitFloatSetCond(LiftoffAssembler* assm, Condition cond, Register dst,
1351  DoubleRegister lhs, DoubleRegister rhs) {
1352  Label cont;
1353  Label not_nan;
1354 
1355  (assm->*cmp_op)(lhs, rhs);
1356  // If PF is one, one of the operands was NaN. This needs special handling.
1357  assm->j(parity_odd, &not_nan, Label::kNear);
1358  // Return 1 for f32.ne, 0 for all other cases.
1359  if (cond == not_equal) {
1360  assm->movl(dst, Immediate(1));
1361  } else {
1362  assm->xorl(dst, dst);
1363  }
1364  assm->jmp(&cont, Label::kNear);
1365  assm->bind(&not_nan);
1366 
1367  assm->setcc(cond, dst);
1368  assm->movzxbl(dst, dst);
1369  assm->bind(&cont);
1370 }
1371 } // namespace liftoff
1372 
1373 void LiftoffAssembler::emit_f32_set_cond(Condition cond, Register dst,
1374  DoubleRegister lhs,
1375  DoubleRegister rhs) {
1376  liftoff::EmitFloatSetCond<&TurboAssembler::Ucomiss>(this, cond, dst, lhs,
1377  rhs);
1378 }
1379 
1380 void LiftoffAssembler::emit_f64_set_cond(Condition cond, Register dst,
1381  DoubleRegister lhs,
1382  DoubleRegister rhs) {
1383  liftoff::EmitFloatSetCond<&TurboAssembler::Ucomisd>(this, cond, dst, lhs,
1384  rhs);
1385 }
1386 
1387 void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) {
1388  cmpp(rsp, Operand(limit_address, 0));
1389  j(below_equal, ool_code);
1390 }
1391 
1392 void LiftoffAssembler::CallTrapCallbackForTesting() {
1393  PrepareCallCFunction(0);
1394  CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0);
1395 }
1396 
1397 void LiftoffAssembler::AssertUnreachable(AbortReason reason) {
1398  TurboAssembler::AssertUnreachable(reason);
1399 }
1400 
1401 void LiftoffAssembler::PushRegisters(LiftoffRegList regs) {
1402  LiftoffRegList gp_regs = regs & kGpCacheRegList;
1403  while (!gp_regs.is_empty()) {
1404  LiftoffRegister reg = gp_regs.GetFirstRegSet();
1405  pushq(reg.gp());
1406  gp_regs.clear(reg);
1407  }
1408  LiftoffRegList fp_regs = regs & kFpCacheRegList;
1409  unsigned num_fp_regs = fp_regs.GetNumRegsSet();
1410  if (num_fp_regs) {
1411  subp(rsp, Immediate(num_fp_regs * kStackSlotSize));
1412  unsigned offset = 0;
1413  while (!fp_regs.is_empty()) {
1414  LiftoffRegister reg = fp_regs.GetFirstRegSet();
1415  Movsd(Operand(rsp, offset), reg.fp());
1416  fp_regs.clear(reg);
1417  offset += sizeof(double);
1418  }
1419  DCHECK_EQ(offset, num_fp_regs * sizeof(double));
1420  }
1421 }
1422 
1423 void LiftoffAssembler::PopRegisters(LiftoffRegList regs) {
1424  LiftoffRegList fp_regs = regs & kFpCacheRegList;
1425  unsigned fp_offset = 0;
1426  while (!fp_regs.is_empty()) {
1427  LiftoffRegister reg = fp_regs.GetFirstRegSet();
1428  Movsd(reg.fp(), Operand(rsp, fp_offset));
1429  fp_regs.clear(reg);
1430  fp_offset += sizeof(double);
1431  }
1432  if (fp_offset) addp(rsp, Immediate(fp_offset));
1433  LiftoffRegList gp_regs = regs & kGpCacheRegList;
1434  while (!gp_regs.is_empty()) {
1435  LiftoffRegister reg = gp_regs.GetLastRegSet();
1436  popq(reg.gp());
1437  gp_regs.clear(reg);
1438  }
1439 }
1440 
1441 void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) {
1442  DCHECK_LT(num_stack_slots, (1 << 16) / kPointerSize); // 16 bit immediate
1443  ret(static_cast<int>(num_stack_slots * kPointerSize));
1444 }
1445 
1446 void LiftoffAssembler::CallC(wasm::FunctionSig* sig,
1447  const LiftoffRegister* args,
1448  const LiftoffRegister* rets,
1449  ValueType out_argument_type, int stack_bytes,
1450  ExternalReference ext_ref) {
1451  subp(rsp, Immediate(stack_bytes));
1452 
1453  int arg_bytes = 0;
1454  for (ValueType param_type : sig->parameters()) {
1455  liftoff::Store(this, Operand(rsp, arg_bytes), *args++, param_type);
1456  arg_bytes += ValueTypes::MemSize(param_type);
1457  }
1458  DCHECK_LE(arg_bytes, stack_bytes);
1459 
1460  // Pass a pointer to the buffer with the arguments to the C function.
1461  movp(arg_reg_1, rsp);
1462 
1463  constexpr int kNumCCallArgs = 1;
1464 
1465  // Now call the C function.
1466  PrepareCallCFunction(kNumCCallArgs);
1467  CallCFunction(ext_ref, kNumCCallArgs);
1468 
1469  // Move return value to the right register.
1470  const LiftoffRegister* next_result_reg = rets;
1471  if (sig->return_count() > 0) {
1472  DCHECK_EQ(1, sig->return_count());
1473  constexpr Register kReturnReg = rax;
1474  if (kReturnReg != next_result_reg->gp()) {
1475  Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0));
1476  }
1477  ++next_result_reg;
1478  }
1479 
1480  // Load potential output value from the buffer on the stack.
1481  if (out_argument_type != kWasmStmt) {
1482  liftoff::Load(this, *next_result_reg, Operand(rsp, 0), out_argument_type);
1483  }
1484 
1485  addp(rsp, Immediate(stack_bytes));
1486 }
1487 
1488 void LiftoffAssembler::CallNativeWasmCode(Address addr) {
1489  near_call(addr, RelocInfo::WASM_CALL);
1490 }
1491 
1492 void LiftoffAssembler::CallIndirect(wasm::FunctionSig* sig,
1493  compiler::CallDescriptor* call_descriptor,
1494  Register target) {
1495  if (target == no_reg) {
1496  popq(kScratchRegister);
1497  target = kScratchRegister;
1498  }
1499  if (FLAG_untrusted_code_mitigations) {
1500  RetpolineCall(target);
1501  } else {
1502  call(target);
1503  }
1504 }
1505 
1506 void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) {
1507  // A direct call to a wasm runtime stub defined in this module.
1508  // Just encode the stub index. This will be patched at relocation.
1509  near_call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL);
1510 }
1511 
1512 void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) {
1513  subp(rsp, Immediate(size));
1514  movp(addr, rsp);
1515 }
1516 
1517 void LiftoffAssembler::DeallocateStackSlot(uint32_t size) {
1518  addp(rsp, Immediate(size));
1519 }
1520 
1521 void LiftoffStackSlots::Construct() {
1522  for (auto& slot : slots_) {
1523  const LiftoffAssembler::VarState& src = slot.src_;
1524  switch (src.loc()) {
1525  case LiftoffAssembler::VarState::kStack:
1526  if (src.type() == kWasmI32) {
1527  // Load i32 values to a register first to ensure they are zero
1528  // extended.
1529  asm_->movl(kScratchRegister, liftoff::GetStackSlot(slot.src_index_));
1530  asm_->pushq(kScratchRegister);
1531  } else {
1532  // For all other types, just push the whole (8-byte) stack slot.
1533  // This is also ok for f32 values (even though we copy 4 uninitialized
1534  // bytes), because f32 and f64 values are clearly distinguished in
1535  // Turbofan, so the uninitialized bytes are never accessed.
1536  asm_->pushq(liftoff::GetStackSlot(slot.src_index_));
1537  }
1538  break;
1539  case LiftoffAssembler::VarState::kRegister:
1540  liftoff::push(asm_, src.reg(), src.type());
1541  break;
1542  case LiftoffAssembler::VarState::KIntConst:
1543  asm_->pushq(Immediate(src.i32_const()));
1544  break;
1545  }
1546  }
1547 }
1548 
1549 #undef REQUIRE_CPU_FEATURE
1550 
1551 } // namespace wasm
1552 } // namespace internal
1553 } // namespace v8
1554 
1555 #endif // V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
v8
Definition: libplatform.h:13
uint32_t