V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
assembler-x64.cc
1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/x64/assembler-x64.h"
6 
7 #include <cstring>
8 
9 #if V8_TARGET_ARCH_X64
10 
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_MACOSX
15 #include <sys/sysctl.h>
16 #endif
17 
18 #include "src/assembler-inl.h"
19 #include "src/base/bits.h"
20 #include "src/base/cpu.h"
21 #include "src/code-stubs.h"
22 #include "src/deoptimizer.h"
23 #include "src/macro-assembler.h"
24 #include "src/string-constants.h"
25 #include "src/v8.h"
26 
27 namespace v8 {
28 namespace internal {
29 
30 // -----------------------------------------------------------------------------
31 // Implementation of CpuFeatures
32 
33 namespace {
34 
35 #if !V8_LIBC_MSVCRT
36 
37 V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
38  unsigned eax, edx;
39  // Check xgetbv; this uses a .byte sequence instead of the instruction
40  // directly because older assemblers do not include support for xgetbv and
41  // there is no easy way to conditionally compile based on the assembler
42  // used.
43  __asm__ volatile(".byte 0x0F, 0x01, 0xD0" : "=a"(eax), "=d"(edx) : "c"(xcr));
44  return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
45 }
46 
47 #define _XCR_XFEATURE_ENABLED_MASK 0
48 
49 #endif // !V8_LIBC_MSVCRT
50 
51 
52 bool OSHasAVXSupport() {
53 #if V8_OS_MACOSX
54  // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
55  // caused by ISRs, so we detect that here and disable AVX in that case.
56  char buffer[128];
57  size_t buffer_size = arraysize(buffer);
58  int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
59  if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
60  FATAL("V8 failed to get kernel version");
61  }
62  // The buffer now contains a string of the form XX.YY.ZZ, where
63  // XX is the major kernel version component.
64  char* period_pos = strchr(buffer, '.');
65  DCHECK_NOT_NULL(period_pos);
66  *period_pos = '\0';
67  long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
68  if (kernel_version_major <= 13) return false;
69 #endif // V8_OS_MACOSX
70  // Check whether OS claims to support AVX.
71  uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
72  return (feature_mask & 0x6) == 0x6;
73 }
74 
75 } // namespace
76 
77 
78 void CpuFeatures::ProbeImpl(bool cross_compile) {
79  base::CPU cpu;
80  CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
81  CHECK(cpu.has_cmov()); // CMOV support is mandatory.
82 
83  // Only use statically determined features for cross compile (snapshot).
84  if (cross_compile) return;
85 
86  if (cpu.has_sse41() && FLAG_enable_sse4_1) {
87  supported_ |= 1u << SSE4_1;
88  supported_ |= 1u << SSSE3;
89  }
90  if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
91  if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
92  // SAHF is not generally available in long mode.
93  if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
94  if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
95  OSHasAVXSupport()) {
96  supported_ |= 1u << AVX;
97  }
98  if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
99  OSHasAVXSupport()) {
100  supported_ |= 1u << FMA3;
101  }
102  if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
103  if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
104  if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
105  if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
106  if (strcmp(FLAG_mcpu, "auto") == 0) {
107  if (cpu.is_atom()) supported_ |= 1u << ATOM;
108  } else if (strcmp(FLAG_mcpu, "atom") == 0) {
109  supported_ |= 1u << ATOM;
110  }
111 }
112 
113 
114 void CpuFeatures::PrintTarget() { }
115 void CpuFeatures::PrintFeatures() {
116  printf(
117  "SSE3=%d SSSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d "
118  "LZCNT=%d "
119  "POPCNT=%d ATOM=%d\n",
120  CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
121  CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SAHF),
122  CpuFeatures::IsSupported(AVX), CpuFeatures::IsSupported(FMA3),
123  CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2),
124  CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT),
125  CpuFeatures::IsSupported(ATOM));
126 }
127 
128 // -----------------------------------------------------------------------------
129 // Implementation of RelocInfo
130 
131 uint32_t RelocInfo::wasm_call_tag() const {
132  DCHECK(rmode_ == WASM_CALL || rmode_ == WASM_STUB_CALL);
133  return Memory<uint32_t>(pc_);
134 }
135 
136 // -----------------------------------------------------------------------------
137 // Implementation of Operand
138 
139 namespace {
140 class OperandBuilder {
141  public:
142  OperandBuilder(Register base, int32_t disp) {
143  if (base == rsp || base == r12) {
144  // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
145  set_sib(times_1, rsp, base);
146  }
147 
148  if (disp == 0 && base != rbp && base != r13) {
149  set_modrm(0, base);
150  } else if (is_int8(disp)) {
151  set_modrm(1, base);
152  set_disp8(disp);
153  } else {
154  set_modrm(2, base);
155  set_disp32(disp);
156  }
157  }
158 
159  OperandBuilder(Register base, Register index, ScaleFactor scale,
160  int32_t disp) {
161  DCHECK(index != rsp);
162  set_sib(scale, index, base);
163  if (disp == 0 && base != rbp && base != r13) {
164  // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
165  // possibly set by set_sib.
166  set_modrm(0, rsp);
167  } else if (is_int8(disp)) {
168  set_modrm(1, rsp);
169  set_disp8(disp);
170  } else {
171  set_modrm(2, rsp);
172  set_disp32(disp);
173  }
174  }
175 
176  OperandBuilder(Register index, ScaleFactor scale, int32_t disp) {
177  DCHECK(index != rsp);
178  set_modrm(0, rsp);
179  set_sib(scale, index, rbp);
180  set_disp32(disp);
181  }
182 
183  OperandBuilder(Label* label, int addend) {
184  data_.addend = addend;
185  DCHECK_NOT_NULL(label);
186  DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
187  set_modrm(0, rbp);
188  set_disp64(reinterpret_cast<intptr_t>(label));
189  }
190 
191  OperandBuilder(Operand operand, int32_t offset) {
192  DCHECK_GE(operand.data().len, 1);
193  // Operand encodes REX ModR/M [SIB] [Disp].
194  byte modrm = operand.data().buf[0];
195  DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
196  bool has_sib = ((modrm & 0x07) == 0x04);
197  byte mode = modrm & 0xC0;
198  int disp_offset = has_sib ? 2 : 1;
199  int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
200  // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
201  // displacement.
202  bool is_baseless =
203  (mode == 0) && (base_reg == 0x05); // No base or RIP base.
204  int32_t disp_value = 0;
205  if (mode == 0x80 || is_baseless) {
206  // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
207  disp_value = *bit_cast<const int32_t*>(&operand.data().buf[disp_offset]);
208  } else if (mode == 0x40) {
209  // Mode 1: Byte displacement.
210  disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
211  }
212 
213  // Write new operand with same registers, but with modified displacement.
214  DCHECK(offset >= 0 ? disp_value + offset > disp_value
215  : disp_value + offset < disp_value); // No overflow.
216  disp_value += offset;
217  data_.rex = operand.data().rex;
218  if (!is_int8(disp_value) || is_baseless) {
219  // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
220  data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
221  data_.len = disp_offset + 4;
222  Memory<int32_t>(reinterpret_cast<Address>(&data_.buf[disp_offset])) =
223  disp_value;
224  } else if (disp_value != 0 || (base_reg == 0x05)) {
225  // Need 8 bits of displacement.
226  data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
227  data_.len = disp_offset + 1;
228  data_.buf[disp_offset] = static_cast<byte>(disp_value);
229  } else {
230  // Need no displacement.
231  data_.buf[0] = (modrm & 0x3F); // Mode 0.
232  data_.len = disp_offset;
233  }
234  if (has_sib) {
235  data_.buf[1] = operand.data().buf[1];
236  }
237  }
238 
239  void set_modrm(int mod, Register rm_reg) {
240  DCHECK(is_uint2(mod));
241  data_.buf[0] = mod << 6 | rm_reg.low_bits();
242  // Set REX.B to the high bit of rm.code().
243  data_.rex |= rm_reg.high_bit();
244  }
245 
246  void set_sib(ScaleFactor scale, Register index, Register base) {
247  DCHECK_EQ(data_.len, 1);
248  DCHECK(is_uint2(scale));
249  // Use SIB with no index register only for base rsp or r12. Otherwise we
250  // would skip the SIB byte entirely.
251  DCHECK(index != rsp || base == rsp || base == r12);
252  data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
253  data_.rex |= index.high_bit() << 1 | base.high_bit();
254  data_.len = 2;
255  }
256 
257  void set_disp8(int disp) {
258  DCHECK(is_int8(disp));
259  DCHECK(data_.len == 1 || data_.len == 2);
260  int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
261  *p = disp;
262  data_.len += sizeof(int8_t);
263  }
264 
265  void set_disp32(int disp) {
266  DCHECK(data_.len == 1 || data_.len == 2);
267  int32_t* p = reinterpret_cast<int32_t*>(&data_.buf[data_.len]);
268  *p = disp;
269  data_.len += sizeof(int32_t);
270  }
271 
272  void set_disp64(int64_t disp) {
273  DCHECK_EQ(1, data_.len);
274  int64_t* p = reinterpret_cast<int64_t*>(&data_.buf[data_.len]);
275  *p = disp;
276  data_.len += sizeof(disp);
277  }
278 
279  const Operand::Data& data() const { return data_; }
280 
281  private:
282  Operand::Data data_;
283 };
284 } // namespace
285 
286 Operand::Operand(Register base, int32_t disp)
287  : data_(OperandBuilder(base, disp).data()) {}
288 
289 Operand::Operand(Register base, Register index, ScaleFactor scale, int32_t disp)
290  : data_(OperandBuilder(base, index, scale, disp).data()) {}
291 
292 Operand::Operand(Register index, ScaleFactor scale, int32_t disp)
293  : data_(OperandBuilder(index, scale, disp).data()) {}
294 
295 Operand::Operand(Label* label, int addend)
296  : data_(OperandBuilder(label, addend).data()) {}
297 
298 Operand::Operand(Operand operand, int32_t offset)
299  : data_(OperandBuilder(operand, offset).data()) {}
300 
301 bool Operand::AddressUsesRegister(Register reg) const {
302  int code = reg.code();
303  DCHECK_NE(data_.buf[0] & 0xC0, 0xC0); // Always a memory operand.
304  // Start with only low three bits of base register. Initial decoding
305  // doesn't distinguish on the REX.B bit.
306  int base_code = data_.buf[0] & 0x07;
307  if (base_code == rsp.code()) {
308  // SIB byte present in buf_[1].
309  // Check the index register from the SIB byte + REX.X prefix.
310  int index_code = ((data_.buf[1] >> 3) & 0x07) | ((data_.rex & 0x02) << 2);
311  // Index code (including REX.X) of 0x04 (rsp) means no index register.
312  if (index_code != rsp.code() && index_code == code) return true;
313  // Add REX.B to get the full base register code.
314  base_code = (data_.buf[1] & 0x07) | ((data_.rex & 0x01) << 3);
315  // A base register of 0x05 (rbp) with mod = 0 means no base register.
316  if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
317  return code == base_code;
318  } else {
319  // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
320  // no base register.
321  if (base_code == rbp.code() && ((data_.buf[0] & 0xC0) == 0)) return false;
322  base_code |= ((data_.rex & 0x01) << 3);
323  return code == base_code;
324  }
325 }
326 
327 void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
328  DCHECK_IMPLIES(isolate == nullptr, heap_object_requests_.empty());
329  for (auto& request : heap_object_requests_) {
330  Address pc = reinterpret_cast<Address>(buffer_) + request.offset();
331  switch (request.kind()) {
332  case HeapObjectRequest::kHeapNumber: {
333  Handle<HeapNumber> object =
334  isolate->factory()->NewHeapNumber(request.heap_number(), TENURED);
335  Memory<Handle<Object>>(pc) = object;
336  break;
337  }
338  case HeapObjectRequest::kCodeStub: {
339  request.code_stub()->set_isolate(isolate);
340  UpdateCodeTarget(Memory<int32_t>(pc), request.code_stub()->GetCode());
341  break;
342  }
343  case HeapObjectRequest::kStringConstant: {
344  const StringConstantBase* str = request.string();
345  CHECK_NOT_NULL(str);
346  Handle<String> allocated = str->AllocateStringConstant(isolate);
347  Memory<Handle<Object>>(pc) = allocated;
348  break;
349  }
350  }
351  }
352 }
353 
354 // Partial Constant Pool.
355 bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
356  auto existing = entries_.find(data);
357  if (existing == entries_.end()) {
358  entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
359  return false;
360  }
361 
362  // Make sure this is called with strictly ascending offsets.
363  DCHECK_GT(offset + kMoveImm64Offset, existing->second);
364 
365  entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
366  return true;
367 }
368 
369 bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
370  if (!FLAG_partial_constant_pool) return false;
371  if (!RelocInfo::IsShareableRelocMode(mode)) return false;
372 
373  // Currently, partial constant pool only handles the following kinds of
374  // RelocInfo.
375  if (mode != RelocInfo::NONE && mode != RelocInfo::EXTERNAL_REFERENCE &&
376  mode != RelocInfo::OFF_HEAP_TARGET)
377  return false;
378 
379  uint64_t raw_data = static_cast<uint64_t>(data);
380  int offset = assm_->pc_offset();
381  return AddSharedEntry(raw_data, offset);
382 }
383 
384 bool ConstPool::IsMoveRipRelative(byte* instr) {
385  if ((*reinterpret_cast<uint32_t*>(instr) & kMoveRipRelativeMask) ==
386  kMoveRipRelativeInstr)
387  return true;
388  return false;
389 }
390 
391 void ConstPool::Clear() { entries_.clear(); }
392 
393 void ConstPool::PatchEntries() {
394  for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
395  iter = entries_.upper_bound(iter->first)) {
396  std::pair<EntryMap::iterator, EntryMap::iterator> range =
397  entries_.equal_range(iter->first);
398  int constant_entry_offset = 0;
399  for (EntryMap::iterator it = range.first; it != range.second; it++) {
400  if (it == range.first) {
401  constant_entry_offset = it->second;
402  continue;
403  }
404 
405  DCHECK_GT(constant_entry_offset, 0);
406  DCHECK_LT(constant_entry_offset, it->second);
407  int32_t disp32 =
408  constant_entry_offset - (it->second + kRipRelativeDispSize);
409  byte* disp_addr = assm_->addr_at(it->second);
410 
411  // Check if the instruction is actually a rip-relative move.
412  DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset));
413  // The displacement of the rip-relative move should be 0 before patching.
414  DCHECK(*reinterpret_cast<uint32_t*>(disp_addr) == 0);
415  *reinterpret_cast<int32_t*>(disp_addr) = disp32;
416  }
417  }
418  Clear();
419 }
420 
421 void Assembler::PatchConstPool() {
422  // There is nothing to do if there are no pending entries.
423  if (constpool_.IsEmpty()) {
424  return;
425  }
426  constpool_.PatchEntries();
427 }
428 
429 bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
430  if (!FLAG_partial_constant_pool) return false;
431  return (rmode == RelocInfo::NONE || rmode == RelocInfo::EXTERNAL_REFERENCE ||
432  rmode == RelocInfo::OFF_HEAP_TARGET);
433 }
434 
435 // -----------------------------------------------------------------------------
436 // Implementation of Assembler.
437 
438 Assembler::Assembler(const AssemblerOptions& options, void* buffer,
439  int buffer_size)
440  : AssemblerBase(options, buffer, buffer_size), constpool_(this) {
441 // Clear the buffer in debug mode unless it was provided by the
442 // caller in which case we can't be sure it's okay to overwrite
443 // existing code in it.
444 #ifdef DEBUG
445  if (own_buffer_) ZapCode(reinterpret_cast<Address>(buffer_), buffer_size_);
446 #endif
447 
448  ReserveCodeTargetSpace(100);
449  reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
450  if (CpuFeatures::IsSupported(SSE4_1)) {
451  EnableCpuFeature(SSSE3);
452  }
453 }
454 
455 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc) {
456  PatchConstPool();
457  DCHECK(constpool_.IsEmpty());
458 
459  // At this point overflow() may be true, but the gap ensures
460  // that we are still not overlapping instructions and relocation info.
461  DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
462 
463  AllocateAndInstallRequestedHeapObjects(isolate);
464 
465  // Set up code descriptor.
466  desc->buffer = buffer_;
467  desc->buffer_size = buffer_size_;
468  desc->instr_size = pc_offset();
469  DCHECK_GT(desc->instr_size, 0); // Zero-size code objects upset the system.
470  desc->reloc_size =
471  static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
472  desc->origin = this;
473  desc->constant_pool_size = 0;
474  desc->unwinding_info_size = 0;
475  desc->unwinding_info = nullptr;
476 
477  // Collection stage
478  auto jump_opt = jump_optimization_info();
479  if (jump_opt && jump_opt->is_collecting()) {
480  auto& bitmap = jump_opt->farjmp_bitmap();
481  int num = static_cast<int>(farjmp_positions_.size());
482  if (num && bitmap.empty()) {
483  bool can_opt = false;
484 
485  bitmap.resize((num + 31) / 32, 0);
486  for (int i = 0; i < num; i++) {
487  int disp_pos = farjmp_positions_[i];
488  int disp = long_at(disp_pos);
489  if (is_int8(disp)) {
490  bitmap[i / 32] |= 1 << (i & 31);
491  can_opt = true;
492  }
493  }
494  if (can_opt) {
495  jump_opt->set_optimizable();
496  }
497  }
498  }
499 }
500 
501 
502 void Assembler::Align(int m) {
503  DCHECK(base::bits::IsPowerOfTwo(m));
504  int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
505  Nop(delta);
506 }
507 
508 
509 void Assembler::CodeTargetAlign() {
510  Align(16); // Preferred alignment of jump targets on x64.
511 }
512 
513 
514 bool Assembler::IsNop(Address addr) {
515  byte* a = reinterpret_cast<byte*>(addr);
516  while (*a == 0x66) a++;
517  if (*a == 0x90) return true;
518  if (a[0] == 0xF && a[1] == 0x1F) return true;
519  return false;
520 }
521 
522 
523 void Assembler::bind_to(Label* L, int pos) {
524  DCHECK(!L->is_bound()); // Label may only be bound once.
525  DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
526  if (L->is_linked()) {
527  int current = L->pos();
528  int next = long_at(current);
529  while (next != current) {
530  if (current >= 4 && long_at(current - 4) == 0) {
531  // Absolute address.
532  intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
533  *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
534  internal_reference_positions_.push_back(current - 4);
535  } else {
536  // Relative address, relative to point after address.
537  int imm32 = pos - (current + sizeof(int32_t));
538  long_at_put(current, imm32);
539  }
540  current = next;
541  next = long_at(next);
542  }
543  // Fix up last fixup on linked list.
544  if (current >= 4 && long_at(current - 4) == 0) {
545  // Absolute address.
546  intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
547  *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
548  internal_reference_positions_.push_back(current - 4);
549  } else {
550  // Relative address, relative to point after address.
551  int imm32 = pos - (current + sizeof(int32_t));
552  long_at_put(current, imm32);
553  }
554  }
555  while (L->is_near_linked()) {
556  int fixup_pos = L->near_link_pos();
557  int offset_to_next =
558  static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
559  DCHECK_LE(offset_to_next, 0);
560  int disp = pos - (fixup_pos + sizeof(int8_t));
561  CHECK(is_int8(disp));
562  set_byte_at(fixup_pos, disp);
563  if (offset_to_next < 0) {
564  L->link_to(fixup_pos + offset_to_next, Label::kNear);
565  } else {
566  L->UnuseNear();
567  }
568  }
569 
570  // Optimization stage
571  auto jump_opt = jump_optimization_info();
572  if (jump_opt && jump_opt->is_optimizing()) {
573  auto it = label_farjmp_maps_.find(L);
574  if (it != label_farjmp_maps_.end()) {
575  auto& pos_vector = it->second;
576  for (auto fixup_pos : pos_vector) {
577  int disp = pos - (fixup_pos + sizeof(int8_t));
578  CHECK(is_int8(disp));
579  set_byte_at(fixup_pos, disp);
580  }
581  label_farjmp_maps_.erase(it);
582  }
583  }
584  L->bind_to(pos);
585 }
586 
587 
588 void Assembler::bind(Label* L) {
589  bind_to(L, pc_offset());
590 }
591 
592 void Assembler::record_farjmp_position(Label* L, int pos) {
593  auto& pos_vector = label_farjmp_maps_[L];
594  pos_vector.push_back(pos);
595 }
596 
597 bool Assembler::is_optimizable_farjmp(int idx) {
598  if (predictable_code_size()) return false;
599 
600  auto jump_opt = jump_optimization_info();
601  CHECK(jump_opt->is_optimizing());
602 
603  auto& bitmap = jump_opt->farjmp_bitmap();
604  CHECK(idx < static_cast<int>(bitmap.size() * 32));
605  return !!(bitmap[idx / 32] & (1 << (idx & 31)));
606 }
607 
608 void Assembler::GrowBuffer() {
609  DCHECK(buffer_overflow());
610  if (!own_buffer_) FATAL("external code buffer is too small");
611 
612  // Compute new buffer size.
613  CodeDesc desc; // the new buffer
614  desc.buffer_size = 2 * buffer_size_;
615 
616  // Some internal data structures overflow for very large buffers,
617  // they must ensure that kMaximalBufferSize is not too large.
618  if (desc.buffer_size > kMaximalBufferSize) {
619  V8::FatalProcessOutOfMemory(nullptr, "Assembler::GrowBuffer");
620  }
621 
622  // Set up new buffer.
623  desc.buffer = NewArray<byte>(desc.buffer_size);
624  desc.origin = this;
625  desc.instr_size = pc_offset();
626  desc.reloc_size =
627  static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
628 
629  // Clear the buffer in debug mode. Use 'int3' instructions to make
630  // sure to get into problems if we ever run uninitialized code.
631 #ifdef DEBUG
632  ZapCode(reinterpret_cast<Address>(desc.buffer), desc.buffer_size);
633 #endif
634 
635  // Copy the data.
636  intptr_t pc_delta = desc.buffer - buffer_;
637  intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
638  (buffer_ + buffer_size_);
639  MemMove(desc.buffer, buffer_, desc.instr_size);
640  MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
641  desc.reloc_size);
642 
643  // Switch buffers.
644  DeleteArray(buffer_);
645  buffer_ = desc.buffer;
646  buffer_size_ = desc.buffer_size;
647  pc_ += pc_delta;
648  reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
649  reloc_info_writer.last_pc() + pc_delta);
650 
651  // Relocate internal references.
652  for (auto pos : internal_reference_positions_) {
653  intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
654  *p += pc_delta;
655  }
656 
657  DCHECK(!buffer_overflow());
658 }
659 
660 void Assembler::emit_operand(int code, Operand adr) {
661  DCHECK(is_uint3(code));
662  const unsigned length = adr.data().len;
663  DCHECK_GT(length, 0);
664 
665  // Emit updated ModR/M byte containing the given register.
666  DCHECK_EQ(adr.data().buf[0] & 0x38, 0);
667  *pc_++ = adr.data().buf[0] | code << 3;
668 
669  // Recognize RIP relative addressing.
670  if (adr.data().buf[0] == 5) {
671  DCHECK_EQ(9u, length);
672  Label* label = *bit_cast<Label* const*>(&adr.data().buf[1]);
673  if (label->is_bound()) {
674  int offset =
675  label->pos() - pc_offset() - sizeof(int32_t) + adr.data().addend;
676  DCHECK_GE(0, offset);
677  emitl(offset);
678  } else if (label->is_linked()) {
679  emitl(label->pos());
680  label->link_to(pc_offset() - sizeof(int32_t));
681  } else {
682  DCHECK(label->is_unused());
683  int32_t current = pc_offset();
684  emitl(current);
685  label->link_to(current);
686  }
687  } else {
688  // Emit the rest of the encoded operand.
689  for (unsigned i = 1; i < length; i++) *pc_++ = adr.data().buf[i];
690  }
691 }
692 
693 
694 // Assembler Instruction implementations.
695 
696 void Assembler::arithmetic_op(byte opcode, Register reg, Operand op, int size) {
697  EnsureSpace ensure_space(this);
698  emit_rex(reg, op, size);
699  emit(opcode);
700  emit_operand(reg, op);
701 }
702 
703 
704 void Assembler::arithmetic_op(byte opcode,
705  Register reg,
706  Register rm_reg,
707  int size) {
708  EnsureSpace ensure_space(this);
709  DCHECK_EQ(opcode & 0xC6, 2);
710  if (rm_reg.low_bits() == 4) { // Forces SIB byte.
711  // Swap reg and rm_reg and change opcode operand order.
712  emit_rex(rm_reg, reg, size);
713  emit(opcode ^ 0x02);
714  emit_modrm(rm_reg, reg);
715  } else {
716  emit_rex(reg, rm_reg, size);
717  emit(opcode);
718  emit_modrm(reg, rm_reg);
719  }
720 }
721 
722 
723 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
724  EnsureSpace ensure_space(this);
725  DCHECK_EQ(opcode & 0xC6, 2);
726  if (rm_reg.low_bits() == 4) { // Forces SIB byte.
727  // Swap reg and rm_reg and change opcode operand order.
728  emit(0x66);
729  emit_optional_rex_32(rm_reg, reg);
730  emit(opcode ^ 0x02);
731  emit_modrm(rm_reg, reg);
732  } else {
733  emit(0x66);
734  emit_optional_rex_32(reg, rm_reg);
735  emit(opcode);
736  emit_modrm(reg, rm_reg);
737  }
738 }
739 
740 void Assembler::arithmetic_op_16(byte opcode, Register reg, Operand rm_reg) {
741  EnsureSpace ensure_space(this);
742  emit(0x66);
743  emit_optional_rex_32(reg, rm_reg);
744  emit(opcode);
745  emit_operand(reg, rm_reg);
746 }
747 
748 void Assembler::arithmetic_op_8(byte opcode, Register reg, Operand op) {
749  EnsureSpace ensure_space(this);
750  if (!reg.is_byte_register()) {
751  emit_rex_32(reg, op);
752  } else {
753  emit_optional_rex_32(reg, op);
754  }
755  emit(opcode);
756  emit_operand(reg, op);
757 }
758 
759 
760 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
761  EnsureSpace ensure_space(this);
762  DCHECK_EQ(opcode & 0xC6, 2);
763  if (rm_reg.low_bits() == 4) { // Forces SIB byte.
764  // Swap reg and rm_reg and change opcode operand order.
765  if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
766  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
767  emit_rex_32(rm_reg, reg);
768  }
769  emit(opcode ^ 0x02);
770  emit_modrm(rm_reg, reg);
771  } else {
772  if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
773  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
774  emit_rex_32(reg, rm_reg);
775  }
776  emit(opcode);
777  emit_modrm(reg, rm_reg);
778  }
779 }
780 
781 
782 void Assembler::immediate_arithmetic_op(byte subcode,
783  Register dst,
784  Immediate src,
785  int size) {
786  EnsureSpace ensure_space(this);
787  emit_rex(dst, size);
788  if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
789  emit(0x83);
790  emit_modrm(subcode, dst);
791  emit(src.value_);
792  } else if (dst == rax) {
793  emit(0x05 | (subcode << 3));
794  emit(src);
795  } else {
796  emit(0x81);
797  emit_modrm(subcode, dst);
798  emit(src);
799  }
800 }
801 
802 void Assembler::immediate_arithmetic_op(byte subcode, Operand dst,
803  Immediate src, int size) {
804  EnsureSpace ensure_space(this);
805  emit_rex(dst, size);
806  if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
807  emit(0x83);
808  emit_operand(subcode, dst);
809  emit(src.value_);
810  } else {
811  emit(0x81);
812  emit_operand(subcode, dst);
813  emit(src);
814  }
815 }
816 
817 
818 void Assembler::immediate_arithmetic_op_16(byte subcode,
819  Register dst,
820  Immediate src) {
821  EnsureSpace ensure_space(this);
822  emit(0x66); // Operand size override prefix.
823  emit_optional_rex_32(dst);
824  if (is_int8(src.value_)) {
825  emit(0x83);
826  emit_modrm(subcode, dst);
827  emit(src.value_);
828  } else if (dst == rax) {
829  emit(0x05 | (subcode << 3));
830  emitw(src.value_);
831  } else {
832  emit(0x81);
833  emit_modrm(subcode, dst);
834  emitw(src.value_);
835  }
836 }
837 
838 void Assembler::immediate_arithmetic_op_16(byte subcode, Operand dst,
839  Immediate src) {
840  EnsureSpace ensure_space(this);
841  emit(0x66); // Operand size override prefix.
842  emit_optional_rex_32(dst);
843  if (is_int8(src.value_)) {
844  emit(0x83);
845  emit_operand(subcode, dst);
846  emit(src.value_);
847  } else {
848  emit(0x81);
849  emit_operand(subcode, dst);
850  emitw(src.value_);
851  }
852 }
853 
854 void Assembler::immediate_arithmetic_op_8(byte subcode, Operand dst,
855  Immediate src) {
856  EnsureSpace ensure_space(this);
857  emit_optional_rex_32(dst);
858  DCHECK(is_int8(src.value_) || is_uint8(src.value_));
859  emit(0x80);
860  emit_operand(subcode, dst);
861  emit(src.value_);
862 }
863 
864 
865 void Assembler::immediate_arithmetic_op_8(byte subcode,
866  Register dst,
867  Immediate src) {
868  EnsureSpace ensure_space(this);
869  if (!dst.is_byte_register()) {
870  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
871  emit_rex_32(dst);
872  }
873  DCHECK(is_int8(src.value_) || is_uint8(src.value_));
874  emit(0x80);
875  emit_modrm(subcode, dst);
876  emit(src.value_);
877 }
878 
879 
880 void Assembler::shift(Register dst,
881  Immediate shift_amount,
882  int subcode,
883  int size) {
884  EnsureSpace ensure_space(this);
885  DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
886  : is_uint5(shift_amount.value_));
887  if (shift_amount.value_ == 1) {
888  emit_rex(dst, size);
889  emit(0xD1);
890  emit_modrm(subcode, dst);
891  } else {
892  emit_rex(dst, size);
893  emit(0xC1);
894  emit_modrm(subcode, dst);
895  emit(shift_amount.value_);
896  }
897 }
898 
899 
900 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
901  int size) {
902  EnsureSpace ensure_space(this);
903  DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
904  : is_uint5(shift_amount.value_));
905  if (shift_amount.value_ == 1) {
906  emit_rex(dst, size);
907  emit(0xD1);
908  emit_operand(subcode, dst);
909  } else {
910  emit_rex(dst, size);
911  emit(0xC1);
912  emit_operand(subcode, dst);
913  emit(shift_amount.value_);
914  }
915 }
916 
917 
918 void Assembler::shift(Register dst, int subcode, int size) {
919  EnsureSpace ensure_space(this);
920  emit_rex(dst, size);
921  emit(0xD3);
922  emit_modrm(subcode, dst);
923 }
924 
925 
926 void Assembler::shift(Operand dst, int subcode, int size) {
927  EnsureSpace ensure_space(this);
928  emit_rex(dst, size);
929  emit(0xD3);
930  emit_operand(subcode, dst);
931 }
932 
933 void Assembler::bswapl(Register dst) {
934  EnsureSpace ensure_space(this);
935  emit_rex_32(dst);
936  emit(0x0F);
937  emit(0xC8 + dst.low_bits());
938 }
939 
940 void Assembler::bswapq(Register dst) {
941  EnsureSpace ensure_space(this);
942  emit_rex_64(dst);
943  emit(0x0F);
944  emit(0xC8 + dst.low_bits());
945 }
946 
947 void Assembler::btq(Operand dst, Register src) {
948  EnsureSpace ensure_space(this);
949  emit_rex_64(src, dst);
950  emit(0x0F);
951  emit(0xA3);
952  emit_operand(src, dst);
953 }
954 
955 void Assembler::btsq(Operand dst, Register src) {
956  EnsureSpace ensure_space(this);
957  emit_rex_64(src, dst);
958  emit(0x0F);
959  emit(0xAB);
960  emit_operand(src, dst);
961 }
962 
963 void Assembler::btsq(Register dst, Immediate imm8) {
964  EnsureSpace ensure_space(this);
965  emit_rex_64(dst);
966  emit(0x0F);
967  emit(0xBA);
968  emit_modrm(0x5, dst);
969  emit(imm8.value_);
970 }
971 
972 void Assembler::btrq(Register dst, Immediate imm8) {
973  EnsureSpace ensure_space(this);
974  emit_rex_64(dst);
975  emit(0x0F);
976  emit(0xBA);
977  emit_modrm(0x6, dst);
978  emit(imm8.value_);
979 }
980 
981 void Assembler::bsrl(Register dst, Register src) {
982  EnsureSpace ensure_space(this);
983  emit_optional_rex_32(dst, src);
984  emit(0x0F);
985  emit(0xBD);
986  emit_modrm(dst, src);
987 }
988 
989 void Assembler::bsrl(Register dst, Operand src) {
990  EnsureSpace ensure_space(this);
991  emit_optional_rex_32(dst, src);
992  emit(0x0F);
993  emit(0xBD);
994  emit_operand(dst, src);
995 }
996 
997 
998 void Assembler::bsrq(Register dst, Register src) {
999  EnsureSpace ensure_space(this);
1000  emit_rex_64(dst, src);
1001  emit(0x0F);
1002  emit(0xBD);
1003  emit_modrm(dst, src);
1004 }
1005 
1006 void Assembler::bsrq(Register dst, Operand src) {
1007  EnsureSpace ensure_space(this);
1008  emit_rex_64(dst, src);
1009  emit(0x0F);
1010  emit(0xBD);
1011  emit_operand(dst, src);
1012 }
1013 
1014 
1015 void Assembler::bsfl(Register dst, Register src) {
1016  EnsureSpace ensure_space(this);
1017  emit_optional_rex_32(dst, src);
1018  emit(0x0F);
1019  emit(0xBC);
1020  emit_modrm(dst, src);
1021 }
1022 
1023 void Assembler::bsfl(Register dst, Operand src) {
1024  EnsureSpace ensure_space(this);
1025  emit_optional_rex_32(dst, src);
1026  emit(0x0F);
1027  emit(0xBC);
1028  emit_operand(dst, src);
1029 }
1030 
1031 
1032 void Assembler::bsfq(Register dst, Register src) {
1033  EnsureSpace ensure_space(this);
1034  emit_rex_64(dst, src);
1035  emit(0x0F);
1036  emit(0xBC);
1037  emit_modrm(dst, src);
1038 }
1039 
1040 void Assembler::bsfq(Register dst, Operand src) {
1041  EnsureSpace ensure_space(this);
1042  emit_rex_64(dst, src);
1043  emit(0x0F);
1044  emit(0xBC);
1045  emit_operand(dst, src);
1046 }
1047 
1048 void Assembler::pshufw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
1049  EnsureSpace ensure_space(this);
1050  emit_optional_rex_32(dst, src);
1051  emit(0x0F);
1052  emit(0x70);
1053  emit_sse_operand(dst, src);
1054  emit(shuffle);
1055 }
1056 
1057 void Assembler::pshufw(XMMRegister dst, Operand src, uint8_t shuffle) {
1058  EnsureSpace ensure_space(this);
1059  emit_optional_rex_32(dst, src);
1060  emit(0x0F);
1061  emit(0x70);
1062  emit_operand(dst.code(), src);
1063  emit(shuffle);
1064 }
1065 
1066 void Assembler::pblendw(XMMRegister dst, Operand src, uint8_t mask) {
1067  sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
1068  emit(mask);
1069 }
1070 
1071 void Assembler::pblendw(XMMRegister dst, XMMRegister src, uint8_t mask) {
1072  sse4_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0E);
1073  emit(mask);
1074 }
1075 
1076 void Assembler::palignr(XMMRegister dst, Operand src, uint8_t mask) {
1077  ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
1078  emit(mask);
1079 }
1080 
1081 void Assembler::palignr(XMMRegister dst, XMMRegister src, uint8_t mask) {
1082  ssse3_instr(dst, src, 0x66, 0x0F, 0x3A, 0x0F);
1083  emit(mask);
1084 }
1085 
1086 void Assembler::call(Label* L) {
1087  EnsureSpace ensure_space(this);
1088  // 1110 1000 #32-bit disp.
1089  emit(0xE8);
1090  if (L->is_bound()) {
1091  int offset = L->pos() - pc_offset() - sizeof(int32_t);
1092  DCHECK_LE(offset, 0);
1093  emitl(offset);
1094  } else if (L->is_linked()) {
1095  emitl(L->pos());
1096  L->link_to(pc_offset() - sizeof(int32_t));
1097  } else {
1098  DCHECK(L->is_unused());
1099  int32_t current = pc_offset();
1100  emitl(current);
1101  L->link_to(current);
1102  }
1103 }
1104 
1105 
1106 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
1107  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1108  EnsureSpace ensure_space(this);
1109  // 1110 1000 #32-bit disp.
1110  emit(0xE8);
1111  emit_runtime_entry(entry, rmode);
1112 }
1113 
1114 void Assembler::call(CodeStub* stub) {
1115  EnsureSpace ensure_space(this);
1116  // 1110 1000 #32-bit disp.
1117  emit(0xE8);
1118  RequestHeapObject(HeapObjectRequest(stub));
1119  RecordRelocInfo(RelocInfo::CODE_TARGET);
1120  int code_target_index = AddCodeTarget(Handle<Code>());
1121  emitl(code_target_index);
1122 }
1123 
1124 void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
1125  DCHECK(RelocInfo::IsCodeTarget(rmode));
1126  EnsureSpace ensure_space(this);
1127  // 1110 1000 #32-bit disp.
1128  emit(0xE8);
1129  RecordRelocInfo(rmode);
1130  int code_target_index = AddCodeTarget(target);
1131  emitl(code_target_index);
1132 }
1133 
1134 void Assembler::near_call(Address addr, RelocInfo::Mode rmode) {
1135  EnsureSpace ensure_space(this);
1136  emit(0xE8);
1137  intptr_t value = static_cast<intptr_t>(addr);
1138  DCHECK(is_int32(value));
1139  RecordRelocInfo(rmode);
1140  emitl(static_cast<int32_t>(value));
1141 }
1142 
1143 void Assembler::near_jmp(Address addr, RelocInfo::Mode rmode) {
1144  EnsureSpace ensure_space(this);
1145  emit(0xE9);
1146  intptr_t value = static_cast<intptr_t>(addr);
1147  DCHECK(is_int32(value));
1148  RecordRelocInfo(rmode);
1149  emitl(static_cast<int32_t>(value));
1150 }
1151 
1152 void Assembler::call(Register adr) {
1153  EnsureSpace ensure_space(this);
1154  // Opcode: FF /2 r64.
1155  emit_optional_rex_32(adr);
1156  emit(0xFF);
1157  emit_modrm(0x2, adr);
1158 }
1159 
1160 void Assembler::call(Operand op) {
1161  EnsureSpace ensure_space(this);
1162  // Opcode: FF /2 m64.
1163  emit_optional_rex_32(op);
1164  emit(0xFF);
1165  emit_operand(0x2, op);
1166 }
1167 
1168 
1169 // Calls directly to the given address using a relative offset.
1170 // Should only ever be used in Code objects for calls within the
1171 // same Code object. Should not be used when generating new code (use labels),
1172 // but only when patching existing code.
1173 void Assembler::call(Address target) {
1174  EnsureSpace ensure_space(this);
1175  // 1110 1000 #32-bit disp.
1176  emit(0xE8);
1177  Address source = reinterpret_cast<Address>(pc_) + 4;
1178  intptr_t displacement = target - source;
1179  DCHECK(is_int32(displacement));
1180  emitl(static_cast<int32_t>(displacement));
1181 }
1182 
1183 
1184 void Assembler::clc() {
1185  EnsureSpace ensure_space(this);
1186  emit(0xF8);
1187 }
1188 
1189 
1190 void Assembler::cld() {
1191  EnsureSpace ensure_space(this);
1192  emit(0xFC);
1193 }
1194 
1195 void Assembler::cdq() {
1196  EnsureSpace ensure_space(this);
1197  emit(0x99);
1198 }
1199 
1200 
1201 void Assembler::cmovq(Condition cc, Register dst, Register src) {
1202  if (cc == always) {
1203  movq(dst, src);
1204  } else if (cc == never) {
1205  return;
1206  }
1207  // No need to check CpuInfo for CMOV support, it's a required part of the
1208  // 64-bit architecture.
1209  DCHECK_GE(cc, 0); // Use mov for unconditional moves.
1210  EnsureSpace ensure_space(this);
1211  // Opcode: REX.W 0f 40 + cc /r.
1212  emit_rex_64(dst, src);
1213  emit(0x0F);
1214  emit(0x40 + cc);
1215  emit_modrm(dst, src);
1216 }
1217 
1218 void Assembler::cmovq(Condition cc, Register dst, Operand src) {
1219  if (cc == always) {
1220  movq(dst, src);
1221  } else if (cc == never) {
1222  return;
1223  }
1224  DCHECK_GE(cc, 0);
1225  EnsureSpace ensure_space(this);
1226  // Opcode: REX.W 0f 40 + cc /r.
1227  emit_rex_64(dst, src);
1228  emit(0x0F);
1229  emit(0x40 + cc);
1230  emit_operand(dst, src);
1231 }
1232 
1233 
1234 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1235  if (cc == always) {
1236  movl(dst, src);
1237  } else if (cc == never) {
1238  return;
1239  }
1240  DCHECK_GE(cc, 0);
1241  EnsureSpace ensure_space(this);
1242  // Opcode: 0f 40 + cc /r.
1243  emit_optional_rex_32(dst, src);
1244  emit(0x0F);
1245  emit(0x40 + cc);
1246  emit_modrm(dst, src);
1247 }
1248 
1249 void Assembler::cmovl(Condition cc, Register dst, Operand src) {
1250  if (cc == always) {
1251  movl(dst, src);
1252  } else if (cc == never) {
1253  return;
1254  }
1255  DCHECK_GE(cc, 0);
1256  EnsureSpace ensure_space(this);
1257  // Opcode: 0f 40 + cc /r.
1258  emit_optional_rex_32(dst, src);
1259  emit(0x0F);
1260  emit(0x40 + cc);
1261  emit_operand(dst, src);
1262 }
1263 
1264 
1265 void Assembler::cmpb_al(Immediate imm8) {
1266  DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
1267  EnsureSpace ensure_space(this);
1268  emit(0x3C);
1269  emit(imm8.value_);
1270 }
1271 
1272 void Assembler::lock() {
1273  EnsureSpace ensure_space(this);
1274  emit(0xF0);
1275 }
1276 
1277 void Assembler::cmpxchgb(Operand dst, Register src) {
1278  EnsureSpace ensure_space(this);
1279  if (!src.is_byte_register()) {
1280  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1281  emit_rex_32(src, dst);
1282  } else {
1283  emit_optional_rex_32(src, dst);
1284  }
1285  emit(0x0F);
1286  emit(0xB0);
1287  emit_operand(src, dst);
1288 }
1289 
1290 void Assembler::cmpxchgw(Operand dst, Register src) {
1291  EnsureSpace ensure_space(this);
1292  emit(0x66);
1293  emit_optional_rex_32(src, dst);
1294  emit(0x0F);
1295  emit(0xB1);
1296  emit_operand(src, dst);
1297 }
1298 
1299 void Assembler::emit_cmpxchg(Operand dst, Register src, int size) {
1300  EnsureSpace ensure_space(this);
1301  emit_rex(src, dst, size);
1302  emit(0x0F);
1303  emit(0xB1);
1304  emit_operand(src, dst);
1305 }
1306 
1307 void Assembler::lfence() {
1308  EnsureSpace ensure_space(this);
1309  emit(0x0F);
1310  emit(0xAE);
1311  emit(0xE8);
1312 }
1313 
1314 void Assembler::cpuid() {
1315  EnsureSpace ensure_space(this);
1316  emit(0x0F);
1317  emit(0xA2);
1318 }
1319 
1320 
1321 void Assembler::cqo() {
1322  EnsureSpace ensure_space(this);
1323  emit_rex_64();
1324  emit(0x99);
1325 }
1326 
1327 
1328 void Assembler::emit_dec(Register dst, int size) {
1329  EnsureSpace ensure_space(this);
1330  emit_rex(dst, size);
1331  emit(0xFF);
1332  emit_modrm(0x1, dst);
1333 }
1334 
1335 void Assembler::emit_dec(Operand dst, int size) {
1336  EnsureSpace ensure_space(this);
1337  emit_rex(dst, size);
1338  emit(0xFF);
1339  emit_operand(1, dst);
1340 }
1341 
1342 
1343 void Assembler::decb(Register dst) {
1344  EnsureSpace ensure_space(this);
1345  if (!dst.is_byte_register()) {
1346  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1347  emit_rex_32(dst);
1348  }
1349  emit(0xFE);
1350  emit_modrm(0x1, dst);
1351 }
1352 
1353 void Assembler::decb(Operand dst) {
1354  EnsureSpace ensure_space(this);
1355  emit_optional_rex_32(dst);
1356  emit(0xFE);
1357  emit_operand(1, dst);
1358 }
1359 
1360 
1361 void Assembler::enter(Immediate size) {
1362  EnsureSpace ensure_space(this);
1363  emit(0xC8);
1364  emitw(size.value_); // 16 bit operand, always.
1365  emit(0);
1366 }
1367 
1368 
1369 void Assembler::hlt() {
1370  EnsureSpace ensure_space(this);
1371  emit(0xF4);
1372 }
1373 
1374 
1375 void Assembler::emit_idiv(Register src, int size) {
1376  EnsureSpace ensure_space(this);
1377  emit_rex(src, size);
1378  emit(0xF7);
1379  emit_modrm(0x7, src);
1380 }
1381 
1382 
1383 void Assembler::emit_div(Register src, int size) {
1384  EnsureSpace ensure_space(this);
1385  emit_rex(src, size);
1386  emit(0xF7);
1387  emit_modrm(0x6, src);
1388 }
1389 
1390 
1391 void Assembler::emit_imul(Register src, int size) {
1392  EnsureSpace ensure_space(this);
1393  emit_rex(src, size);
1394  emit(0xF7);
1395  emit_modrm(0x5, src);
1396 }
1397 
1398 void Assembler::emit_imul(Operand src, int size) {
1399  EnsureSpace ensure_space(this);
1400  emit_rex(src, size);
1401  emit(0xF7);
1402  emit_operand(0x5, src);
1403 }
1404 
1405 
1406 void Assembler::emit_imul(Register dst, Register src, int size) {
1407  EnsureSpace ensure_space(this);
1408  emit_rex(dst, src, size);
1409  emit(0x0F);
1410  emit(0xAF);
1411  emit_modrm(dst, src);
1412 }
1413 
1414 void Assembler::emit_imul(Register dst, Operand src, int size) {
1415  EnsureSpace ensure_space(this);
1416  emit_rex(dst, src, size);
1417  emit(0x0F);
1418  emit(0xAF);
1419  emit_operand(dst, src);
1420 }
1421 
1422 
1423 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1424  EnsureSpace ensure_space(this);
1425  emit_rex(dst, src, size);
1426  if (is_int8(imm.value_)) {
1427  emit(0x6B);
1428  emit_modrm(dst, src);
1429  emit(imm.value_);
1430  } else {
1431  emit(0x69);
1432  emit_modrm(dst, src);
1433  emitl(imm.value_);
1434  }
1435 }
1436 
1437 void Assembler::emit_imul(Register dst, Operand src, Immediate imm, int size) {
1438  EnsureSpace ensure_space(this);
1439  emit_rex(dst, src, size);
1440  if (is_int8(imm.value_)) {
1441  emit(0x6B);
1442  emit_operand(dst, src);
1443  emit(imm.value_);
1444  } else {
1445  emit(0x69);
1446  emit_operand(dst, src);
1447  emitl(imm.value_);
1448  }
1449 }
1450 
1451 
1452 void Assembler::emit_inc(Register dst, int size) {
1453  EnsureSpace ensure_space(this);
1454  emit_rex(dst, size);
1455  emit(0xFF);
1456  emit_modrm(0x0, dst);
1457 }
1458 
1459 void Assembler::emit_inc(Operand dst, int size) {
1460  EnsureSpace ensure_space(this);
1461  emit_rex(dst, size);
1462  emit(0xFF);
1463  emit_operand(0, dst);
1464 }
1465 
1466 
1467 void Assembler::int3() {
1468  EnsureSpace ensure_space(this);
1469  emit(0xCC);
1470 }
1471 
1472 
1473 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1474  if (cc == always) {
1475  jmp(L);
1476  return;
1477  } else if (cc == never) {
1478  return;
1479  }
1480  EnsureSpace ensure_space(this);
1481  DCHECK(is_uint4(cc));
1482  if (L->is_bound()) {
1483  const int short_size = 2;
1484  const int long_size = 6;
1485  int offs = L->pos() - pc_offset();
1486  DCHECK_LE(offs, 0);
1487  // Determine whether we can use 1-byte offsets for backwards branches,
1488  // which have a max range of 128 bytes.
1489 
1490  // We also need to check predictable_code_size() flag here, because on x64,
1491  // when the full code generator recompiles code for debugging, some places
1492  // need to be padded out to a certain size. The debugger is keeping track of
1493  // how often it did this so that it can adjust return addresses on the
1494  // stack, but if the size of jump instructions can also change, that's not
1495  // enough and the calculated offsets would be incorrect.
1496  if (is_int8(offs - short_size) && !predictable_code_size()) {
1497  // 0111 tttn #8-bit disp.
1498  emit(0x70 | cc);
1499  emit((offs - short_size) & 0xFF);
1500  } else {
1501  // 0000 1111 1000 tttn #32-bit disp.
1502  emit(0x0F);
1503  emit(0x80 | cc);
1504  emitl(offs - long_size);
1505  }
1506  } else if (distance == Label::kNear) {
1507  // 0111 tttn #8-bit disp
1508  emit(0x70 | cc);
1509  byte disp = 0x00;
1510  if (L->is_near_linked()) {
1511  int offset = L->near_link_pos() - pc_offset();
1512  DCHECK(is_int8(offset));
1513  disp = static_cast<byte>(offset & 0xFF);
1514  }
1515  L->link_to(pc_offset(), Label::kNear);
1516  emit(disp);
1517  } else {
1518  auto jump_opt = jump_optimization_info();
1519  if (V8_UNLIKELY(jump_opt)) {
1520  if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1521  // 0111 tttn #8-bit disp
1522  emit(0x70 | cc);
1523  record_farjmp_position(L, pc_offset());
1524  emit(0);
1525  return;
1526  }
1527  if (jump_opt->is_collecting()) {
1528  farjmp_positions_.push_back(pc_offset() + 2);
1529  }
1530  }
1531  if (L->is_linked()) {
1532  // 0000 1111 1000 tttn #32-bit disp.
1533  emit(0x0F);
1534  emit(0x80 | cc);
1535  emitl(L->pos());
1536  L->link_to(pc_offset() - sizeof(int32_t));
1537  } else {
1538  DCHECK(L->is_unused());
1539  emit(0x0F);
1540  emit(0x80 | cc);
1541  int32_t current = pc_offset();
1542  emitl(current);
1543  L->link_to(current);
1544  }
1545  }
1546 }
1547 
1548 
1549 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1550  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1551  EnsureSpace ensure_space(this);
1552  DCHECK(is_uint4(cc));
1553  emit(0x0F);
1554  emit(0x80 | cc);
1555  emit_runtime_entry(entry, rmode);
1556 }
1557 
1558 
1559 void Assembler::j(Condition cc,
1560  Handle<Code> target,
1561  RelocInfo::Mode rmode) {
1562  if (cc == always) {
1563  jmp(target, rmode);
1564  return;
1565  } else if (cc == never) {
1566  return;
1567  }
1568  EnsureSpace ensure_space(this);
1569  DCHECK(is_uint4(cc));
1570  // 0000 1111 1000 tttn #32-bit disp.
1571  emit(0x0F);
1572  emit(0x80 | cc);
1573  DCHECK(RelocInfo::IsCodeTarget(rmode));
1574  RecordRelocInfo(rmode);
1575  int code_target_index = AddCodeTarget(target);
1576  emitl(code_target_index);
1577 }
1578 
1579 
1580 void Assembler::jmp(Label* L, Label::Distance distance) {
1581  EnsureSpace ensure_space(this);
1582  const int short_size = sizeof(int8_t);
1583  const int long_size = sizeof(int32_t);
1584  if (L->is_bound()) {
1585  int offs = L->pos() - pc_offset() - 1;
1586  DCHECK_LE(offs, 0);
1587  if (is_int8(offs - short_size) && !predictable_code_size()) {
1588  // 1110 1011 #8-bit disp.
1589  emit(0xEB);
1590  emit((offs - short_size) & 0xFF);
1591  } else {
1592  // 1110 1001 #32-bit disp.
1593  emit(0xE9);
1594  emitl(offs - long_size);
1595  }
1596  } else if (distance == Label::kNear) {
1597  emit(0xEB);
1598  byte disp = 0x00;
1599  if (L->is_near_linked()) {
1600  int offset = L->near_link_pos() - pc_offset();
1601  DCHECK(is_int8(offset));
1602  disp = static_cast<byte>(offset & 0xFF);
1603  }
1604  L->link_to(pc_offset(), Label::kNear);
1605  emit(disp);
1606  } else {
1607  auto jump_opt = jump_optimization_info();
1608  if (V8_UNLIKELY(jump_opt)) {
1609  if (jump_opt->is_optimizing() && is_optimizable_farjmp(farjmp_num_++)) {
1610  emit(0xEB);
1611  record_farjmp_position(L, pc_offset());
1612  emit(0);
1613  return;
1614  }
1615  if (jump_opt->is_collecting()) {
1616  farjmp_positions_.push_back(pc_offset() + 1);
1617  }
1618  }
1619  if (L->is_linked()) {
1620  // 1110 1001 #32-bit disp.
1621  emit(0xE9);
1622  emitl(L->pos());
1623  L->link_to(pc_offset() - long_size);
1624  } else {
1625  // 1110 1001 #32-bit disp.
1626  DCHECK(L->is_unused());
1627  emit(0xE9);
1628  int32_t current = pc_offset();
1629  emitl(current);
1630  L->link_to(current);
1631  }
1632  }
1633 }
1634 
1635 
1636 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1637  DCHECK(RelocInfo::IsCodeTarget(rmode));
1638  EnsureSpace ensure_space(this);
1639  // 1110 1001 #32-bit disp.
1640  emit(0xE9);
1641  RecordRelocInfo(rmode);
1642  int code_target_index = AddCodeTarget(target);
1643  emitl(code_target_index);
1644 }
1645 
1646 
1647 void Assembler::jmp(Register target) {
1648  EnsureSpace ensure_space(this);
1649  // Opcode FF/4 r64.
1650  emit_optional_rex_32(target);
1651  emit(0xFF);
1652  emit_modrm(0x4, target);
1653 }
1654 
1655 void Assembler::jmp(Operand src) {
1656  EnsureSpace ensure_space(this);
1657  // Opcode FF/4 m64.
1658  emit_optional_rex_32(src);
1659  emit(0xFF);
1660  emit_operand(0x4, src);
1661 }
1662 
1663 void Assembler::emit_lea(Register dst, Operand src, int size) {
1664  EnsureSpace ensure_space(this);
1665  emit_rex(dst, src, size);
1666  emit(0x8D);
1667  emit_operand(dst, src);
1668 }
1669 
1670 void Assembler::load_rax(Address value, RelocInfo::Mode mode) {
1671  EnsureSpace ensure_space(this);
1672  if (kPointerSize == kInt64Size) {
1673  emit(0x48); // REX.W
1674  emit(0xA1);
1675  emitp(value, mode);
1676  } else {
1677  DCHECK_EQ(kPointerSize, kInt32Size);
1678  emit(0xA1);
1679  emitp(value, mode);
1680  // In 64-bit mode, need to zero extend the operand to 8 bytes.
1681  // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1682  // Developer's Manual Volume 2.
1683  emitl(0);
1684  }
1685 }
1686 
1687 
1688 void Assembler::load_rax(ExternalReference ref) {
1689  load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1690 }
1691 
1692 
1693 void Assembler::leave() {
1694  EnsureSpace ensure_space(this);
1695  emit(0xC9);
1696 }
1697 
1698 void Assembler::movb(Register dst, Operand src) {
1699  EnsureSpace ensure_space(this);
1700  if (!dst.is_byte_register()) {
1701  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1702  emit_rex_32(dst, src);
1703  } else {
1704  emit_optional_rex_32(dst, src);
1705  }
1706  emit(0x8A);
1707  emit_operand(dst, src);
1708 }
1709 
1710 
1711 void Assembler::movb(Register dst, Immediate imm) {
1712  EnsureSpace ensure_space(this);
1713  if (!dst.is_byte_register()) {
1714  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1715  emit_rex_32(dst);
1716  }
1717  emit(0xB0 + dst.low_bits());
1718  emit(imm.value_);
1719 }
1720 
1721 void Assembler::movb(Operand dst, Register src) {
1722  EnsureSpace ensure_space(this);
1723  if (!src.is_byte_register()) {
1724  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1725  emit_rex_32(src, dst);
1726  } else {
1727  emit_optional_rex_32(src, dst);
1728  }
1729  emit(0x88);
1730  emit_operand(src, dst);
1731 }
1732 
1733 void Assembler::movb(Operand dst, Immediate imm) {
1734  EnsureSpace ensure_space(this);
1735  emit_optional_rex_32(dst);
1736  emit(0xC6);
1737  emit_operand(0x0, dst);
1738  emit(static_cast<byte>(imm.value_));
1739 }
1740 
1741 void Assembler::movw(Register dst, Operand src) {
1742  EnsureSpace ensure_space(this);
1743  emit(0x66);
1744  emit_optional_rex_32(dst, src);
1745  emit(0x8B);
1746  emit_operand(dst, src);
1747 }
1748 
1749 void Assembler::movw(Operand dst, Register src) {
1750  EnsureSpace ensure_space(this);
1751  emit(0x66);
1752  emit_optional_rex_32(src, dst);
1753  emit(0x89);
1754  emit_operand(src, dst);
1755 }
1756 
1757 void Assembler::movw(Operand dst, Immediate imm) {
1758  EnsureSpace ensure_space(this);
1759  emit(0x66);
1760  emit_optional_rex_32(dst);
1761  emit(0xC7);
1762  emit_operand(0x0, dst);
1763  emit(static_cast<byte>(imm.value_ & 0xFF));
1764  emit(static_cast<byte>(imm.value_ >> 8));
1765 }
1766 
1767 void Assembler::emit_mov(Register dst, Operand src, int size) {
1768  EnsureSpace ensure_space(this);
1769  emit_rex(dst, src, size);
1770  emit(0x8B);
1771  emit_operand(dst, src);
1772 }
1773 
1774 
1775 void Assembler::emit_mov(Register dst, Register src, int size) {
1776  EnsureSpace ensure_space(this);
1777  if (src.low_bits() == 4) {
1778  emit_rex(src, dst, size);
1779  emit(0x89);
1780  emit_modrm(src, dst);
1781  } else {
1782  emit_rex(dst, src, size);
1783  emit(0x8B);
1784  emit_modrm(dst, src);
1785  }
1786 }
1787 
1788 void Assembler::emit_mov(Operand dst, Register src, int size) {
1789  EnsureSpace ensure_space(this);
1790  emit_rex(src, dst, size);
1791  emit(0x89);
1792  emit_operand(src, dst);
1793 }
1794 
1795 
1796 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1797  EnsureSpace ensure_space(this);
1798  emit_rex(dst, size);
1799  if (size == kInt64Size) {
1800  emit(0xC7);
1801  emit_modrm(0x0, dst);
1802  } else {
1803  DCHECK_EQ(size, kInt32Size);
1804  emit(0xB8 + dst.low_bits());
1805  }
1806  emit(value);
1807 }
1808 
1809 void Assembler::emit_mov(Operand dst, Immediate value, int size) {
1810  EnsureSpace ensure_space(this);
1811  emit_rex(dst, size);
1812  emit(0xC7);
1813  emit_operand(0x0, dst);
1814  emit(value);
1815 }
1816 
1817 void Assembler::movp(Register dst, Address value, RelocInfo::Mode rmode) {
1818  if (constpool_.TryRecordEntry(value, rmode)) {
1819  // Emit rip-relative move with offset = 0
1820  Label label;
1821  emit_mov(dst, Operand(&label, 0), kPointerSize);
1822  bind(&label);
1823  } else {
1824  EnsureSpace ensure_space(this);
1825  emit_rex(dst, kPointerSize);
1826  emit(0xB8 | dst.low_bits());
1827  emitp(value, rmode);
1828  }
1829 }
1830 
1831 void Assembler::movp_heap_number(Register dst, double value) {
1832  EnsureSpace ensure_space(this);
1833  emit_rex(dst, kPointerSize);
1834  emit(0xB8 | dst.low_bits());
1835  RequestHeapObject(HeapObjectRequest(value));
1836  emitp(0, RelocInfo::EMBEDDED_OBJECT);
1837 }
1838 
1839 void Assembler::movp_string(Register dst, const StringConstantBase* str) {
1840  EnsureSpace ensure_space(this);
1841  emit_rex(dst, kPointerSize);
1842  emit(0xB8 | dst.low_bits());
1843  RequestHeapObject(HeapObjectRequest(str));
1844  emitp(0, RelocInfo::EMBEDDED_OBJECT);
1845 }
1846 
1847 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) {
1848  if (constpool_.TryRecordEntry(value, rmode)) {
1849  // Emit rip-relative move with offset = 0
1850  Label label;
1851  emit_mov(dst, Operand(&label, 0), kPointerSize);
1852  bind(&label);
1853  } else {
1854  EnsureSpace ensure_space(this);
1855  emit_rex_64(dst);
1856  emit(0xB8 | dst.low_bits());
1857  if (!RelocInfo::IsNone(rmode)) {
1858  RecordRelocInfo(rmode, value);
1859  }
1860  emitq(value);
1861  }
1862 }
1863 
1864 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) {
1865  movq(dst, static_cast<int64_t>(value), rmode);
1866 }
1867 
1868 // Loads the ip-relative location of the src label into the target location
1869 // (as a 32-bit offset sign extended to 64-bit).
1870 void Assembler::movl(Operand dst, Label* src) {
1871  EnsureSpace ensure_space(this);
1872  emit_optional_rex_32(dst);
1873  emit(0xC7);
1874  emit_operand(0, dst);
1875  if (src->is_bound()) {
1876  int offset = src->pos() - pc_offset() - sizeof(int32_t);
1877  DCHECK_LE(offset, 0);
1878  emitl(offset);
1879  } else if (src->is_linked()) {
1880  emitl(src->pos());
1881  src->link_to(pc_offset() - sizeof(int32_t));
1882  } else {
1883  DCHECK(src->is_unused());
1884  int32_t current = pc_offset();
1885  emitl(current);
1886  src->link_to(current);
1887  }
1888 }
1889 
1890 
1891 void Assembler::movsxbl(Register dst, Register src) {
1892  EnsureSpace ensure_space(this);
1893  if (!src.is_byte_register()) {
1894  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1895  emit_rex_32(dst, src);
1896  } else {
1897  emit_optional_rex_32(dst, src);
1898  }
1899  emit(0x0F);
1900  emit(0xBE);
1901  emit_modrm(dst, src);
1902 }
1903 
1904 void Assembler::movsxbl(Register dst, Operand src) {
1905  EnsureSpace ensure_space(this);
1906  emit_optional_rex_32(dst, src);
1907  emit(0x0F);
1908  emit(0xBE);
1909  emit_operand(dst, src);
1910 }
1911 
1912 void Assembler::movsxbq(Register dst, Operand src) {
1913  EnsureSpace ensure_space(this);
1914  emit_rex_64(dst, src);
1915  emit(0x0F);
1916  emit(0xBE);
1917  emit_operand(dst, src);
1918 }
1919 
1920 void Assembler::movsxbq(Register dst, Register src) {
1921  EnsureSpace ensure_space(this);
1922  emit_rex_64(dst, src);
1923  emit(0x0F);
1924  emit(0xBE);
1925  emit_modrm(dst, src);
1926 }
1927 
1928 void Assembler::movsxwl(Register dst, Register src) {
1929  EnsureSpace ensure_space(this);
1930  emit_optional_rex_32(dst, src);
1931  emit(0x0F);
1932  emit(0xBF);
1933  emit_modrm(dst, src);
1934 }
1935 
1936 void Assembler::movsxwl(Register dst, Operand src) {
1937  EnsureSpace ensure_space(this);
1938  emit_optional_rex_32(dst, src);
1939  emit(0x0F);
1940  emit(0xBF);
1941  emit_operand(dst, src);
1942 }
1943 
1944 void Assembler::movsxwq(Register dst, Operand src) {
1945  EnsureSpace ensure_space(this);
1946  emit_rex_64(dst, src);
1947  emit(0x0F);
1948  emit(0xBF);
1949  emit_operand(dst, src);
1950 }
1951 
1952 void Assembler::movsxwq(Register dst, Register src) {
1953  EnsureSpace ensure_space(this);
1954  emit_rex_64(dst, src);
1955  emit(0x0F);
1956  emit(0xBF);
1957  emit_modrm(dst, src);
1958 }
1959 
1960 void Assembler::movsxlq(Register dst, Register src) {
1961  EnsureSpace ensure_space(this);
1962  emit_rex_64(dst, src);
1963  emit(0x63);
1964  emit_modrm(dst, src);
1965 }
1966 
1967 void Assembler::movsxlq(Register dst, Operand src) {
1968  EnsureSpace ensure_space(this);
1969  emit_rex_64(dst, src);
1970  emit(0x63);
1971  emit_operand(dst, src);
1972 }
1973 
1974 void Assembler::emit_movzxb(Register dst, Operand src, int size) {
1975  EnsureSpace ensure_space(this);
1976  // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1977  // there is no need to make this a 64 bit operation.
1978  emit_optional_rex_32(dst, src);
1979  emit(0x0F);
1980  emit(0xB6);
1981  emit_operand(dst, src);
1982 }
1983 
1984 
1985 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1986  EnsureSpace ensure_space(this);
1987  // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1988  // there is no need to make this a 64 bit operation.
1989  if (!src.is_byte_register()) {
1990  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1991  emit_rex_32(dst, src);
1992  } else {
1993  emit_optional_rex_32(dst, src);
1994  }
1995  emit(0x0F);
1996  emit(0xB6);
1997  emit_modrm(dst, src);
1998 }
1999 
2000 void Assembler::emit_movzxw(Register dst, Operand src, int size) {
2001  EnsureSpace ensure_space(this);
2002  // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
2003  // there is no need to make this a 64 bit operation.
2004  emit_optional_rex_32(dst, src);
2005  emit(0x0F);
2006  emit(0xB7);
2007  emit_operand(dst, src);
2008 }
2009 
2010 
2011 void Assembler::emit_movzxw(Register dst, Register src, int size) {
2012  EnsureSpace ensure_space(this);
2013  // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
2014  // there is no need to make this a 64 bit operation.
2015  emit_optional_rex_32(dst, src);
2016  emit(0x0F);
2017  emit(0xB7);
2018  emit_modrm(dst, src);
2019 }
2020 
2021 
2022 void Assembler::repmovsb() {
2023  EnsureSpace ensure_space(this);
2024  emit(0xF3);
2025  emit(0xA4);
2026 }
2027 
2028 
2029 void Assembler::repmovsw() {
2030  EnsureSpace ensure_space(this);
2031  emit(0x66); // Operand size override.
2032  emit(0xF3);
2033  emit(0xA4);
2034 }
2035 
2036 
2037 void Assembler::emit_repmovs(int size) {
2038  EnsureSpace ensure_space(this);
2039  emit(0xF3);
2040  emit_rex(size);
2041  emit(0xA5);
2042 }
2043 
2044 
2045 void Assembler::mull(Register src) {
2046  EnsureSpace ensure_space(this);
2047  emit_optional_rex_32(src);
2048  emit(0xF7);
2049  emit_modrm(0x4, src);
2050 }
2051 
2052 void Assembler::mull(Operand src) {
2053  EnsureSpace ensure_space(this);
2054  emit_optional_rex_32(src);
2055  emit(0xF7);
2056  emit_operand(0x4, src);
2057 }
2058 
2059 
2060 void Assembler::mulq(Register src) {
2061  EnsureSpace ensure_space(this);
2062  emit_rex_64(src);
2063  emit(0xF7);
2064  emit_modrm(0x4, src);
2065 }
2066 
2067 
2068 void Assembler::emit_neg(Register dst, int size) {
2069  EnsureSpace ensure_space(this);
2070  emit_rex(dst, size);
2071  emit(0xF7);
2072  emit_modrm(0x3, dst);
2073 }
2074 
2075 void Assembler::emit_neg(Operand dst, int size) {
2076  EnsureSpace ensure_space(this);
2077  emit_rex_64(dst);
2078  emit(0xF7);
2079  emit_operand(3, dst);
2080 }
2081 
2082 
2083 void Assembler::nop() {
2084  EnsureSpace ensure_space(this);
2085  emit(0x90);
2086 }
2087 
2088 
2089 void Assembler::emit_not(Register dst, int size) {
2090  EnsureSpace ensure_space(this);
2091  emit_rex(dst, size);
2092  emit(0xF7);
2093  emit_modrm(0x2, dst);
2094 }
2095 
2096 void Assembler::emit_not(Operand dst, int size) {
2097  EnsureSpace ensure_space(this);
2098  emit_rex(dst, size);
2099  emit(0xF7);
2100  emit_operand(2, dst);
2101 }
2102 
2103 
2104 void Assembler::Nop(int n) {
2105  // The recommended muti-byte sequences of NOP instructions from the Intel 64
2106  // and IA-32 Architectures Software Developer's Manual.
2107  //
2108  // Length Assembly Byte Sequence
2109  // 2 bytes 66 NOP 66 90H
2110  // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H
2111  // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
2112  // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
2113  // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
2114  // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
2115  // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
2116  // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00
2117  // 00000000H] 00H
2118 
2119  EnsureSpace ensure_space(this);
2120  while (n > 0) {
2121  switch (n) {
2122  case 2:
2123  emit(0x66);
2124  V8_FALLTHROUGH;
2125  case 1:
2126  emit(0x90);
2127  return;
2128  case 3:
2129  emit(0x0F);
2130  emit(0x1F);
2131  emit(0x00);
2132  return;
2133  case 4:
2134  emit(0x0F);
2135  emit(0x1F);
2136  emit(0x40);
2137  emit(0x00);
2138  return;
2139  case 6:
2140  emit(0x66);
2141  V8_FALLTHROUGH;
2142  case 5:
2143  emit(0x0F);
2144  emit(0x1F);
2145  emit(0x44);
2146  emit(0x00);
2147  emit(0x00);
2148  return;
2149  case 7:
2150  emit(0x0F);
2151  emit(0x1F);
2152  emit(0x80);
2153  emit(0x00);
2154  emit(0x00);
2155  emit(0x00);
2156  emit(0x00);
2157  return;
2158  default:
2159  case 11:
2160  emit(0x66);
2161  n--;
2162  V8_FALLTHROUGH;
2163  case 10:
2164  emit(0x66);
2165  n--;
2166  V8_FALLTHROUGH;
2167  case 9:
2168  emit(0x66);
2169  n--;
2170  V8_FALLTHROUGH;
2171  case 8:
2172  emit(0x0F);
2173  emit(0x1F);
2174  emit(0x84);
2175  emit(0x00);
2176  emit(0x00);
2177  emit(0x00);
2178  emit(0x00);
2179  emit(0x00);
2180  n -= 8;
2181  }
2182  }
2183 }
2184 
2185 
2186 void Assembler::popq(Register dst) {
2187  EnsureSpace ensure_space(this);
2188  emit_optional_rex_32(dst);
2189  emit(0x58 | dst.low_bits());
2190 }
2191 
2192 void Assembler::popq(Operand dst) {
2193  EnsureSpace ensure_space(this);
2194  emit_optional_rex_32(dst);
2195  emit(0x8F);
2196  emit_operand(0, dst);
2197 }
2198 
2199 
2200 void Assembler::popfq() {
2201  EnsureSpace ensure_space(this);
2202  emit(0x9D);
2203 }
2204 
2205 
2206 void Assembler::pushq(Register src) {
2207  EnsureSpace ensure_space(this);
2208  emit_optional_rex_32(src);
2209  emit(0x50 | src.low_bits());
2210 }
2211 
2212 void Assembler::pushq(Operand src) {
2213  EnsureSpace ensure_space(this);
2214  emit_optional_rex_32(src);
2215  emit(0xFF);
2216  emit_operand(6, src);
2217 }
2218 
2219 
2220 void Assembler::pushq(Immediate value) {
2221  EnsureSpace ensure_space(this);
2222  if (is_int8(value.value_)) {
2223  emit(0x6A);
2224  emit(value.value_); // Emit low byte of value.
2225  } else {
2226  emit(0x68);
2227  emitl(value.value_);
2228  }
2229 }
2230 
2231 
2232 void Assembler::pushq_imm32(int32_t imm32) {
2233  EnsureSpace ensure_space(this);
2234  emit(0x68);
2235  emitl(imm32);
2236 }
2237 
2238 
2239 void Assembler::pushfq() {
2240  EnsureSpace ensure_space(this);
2241  emit(0x9C);
2242 }
2243 
2244 
2245 void Assembler::ret(int imm16) {
2246  EnsureSpace ensure_space(this);
2247  DCHECK(is_uint16(imm16));
2248  if (imm16 == 0) {
2249  emit(0xC3);
2250  } else {
2251  emit(0xC2);
2252  emit(imm16 & 0xFF);
2253  emit((imm16 >> 8) & 0xFF);
2254  }
2255 }
2256 
2257 
2258 void Assembler::ud2() {
2259  EnsureSpace ensure_space(this);
2260  emit(0x0F);
2261  emit(0x0B);
2262 }
2263 
2264 
2265 void Assembler::setcc(Condition cc, Register reg) {
2266  if (cc > last_condition) {
2267  movb(reg, Immediate(cc == always ? 1 : 0));
2268  return;
2269  }
2270  EnsureSpace ensure_space(this);
2271  DCHECK(is_uint4(cc));
2272  if (!reg.is_byte_register()) {
2273  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2274  emit_rex_32(reg);
2275  }
2276  emit(0x0F);
2277  emit(0x90 | cc);
2278  emit_modrm(0x0, reg);
2279 }
2280 
2281 
2282 void Assembler::shld(Register dst, Register src) {
2283  EnsureSpace ensure_space(this);
2284  emit_rex_64(src, dst);
2285  emit(0x0F);
2286  emit(0xA5);
2287  emit_modrm(src, dst);
2288 }
2289 
2290 
2291 void Assembler::shrd(Register dst, Register src) {
2292  EnsureSpace ensure_space(this);
2293  emit_rex_64(src, dst);
2294  emit(0x0F);
2295  emit(0xAD);
2296  emit_modrm(src, dst);
2297 }
2298 
2299 void Assembler::xchgb(Register reg, Operand op) {
2300  EnsureSpace ensure_space(this);
2301  if (!reg.is_byte_register()) {
2302  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2303  emit_rex_32(reg, op);
2304  } else {
2305  emit_optional_rex_32(reg, op);
2306  }
2307  emit(0x86);
2308  emit_operand(reg, op);
2309 }
2310 
2311 void Assembler::xchgw(Register reg, Operand op) {
2312  EnsureSpace ensure_space(this);
2313  emit(0x66);
2314  emit_optional_rex_32(reg, op);
2315  emit(0x87);
2316  emit_operand(reg, op);
2317 }
2318 
2319 void Assembler::emit_xchg(Register dst, Register src, int size) {
2320  EnsureSpace ensure_space(this);
2321  if (src == rax || dst == rax) { // Single-byte encoding
2322  Register other = src == rax ? dst : src;
2323  emit_rex(other, size);
2324  emit(0x90 | other.low_bits());
2325  } else if (dst.low_bits() == 4) {
2326  emit_rex(dst, src, size);
2327  emit(0x87);
2328  emit_modrm(dst, src);
2329  } else {
2330  emit_rex(src, dst, size);
2331  emit(0x87);
2332  emit_modrm(src, dst);
2333  }
2334 }
2335 
2336 void Assembler::emit_xchg(Register dst, Operand src, int size) {
2337  EnsureSpace ensure_space(this);
2338  emit_rex(dst, src, size);
2339  emit(0x87);
2340  emit_operand(dst, src);
2341 }
2342 
2343 void Assembler::store_rax(Address dst, RelocInfo::Mode mode) {
2344  EnsureSpace ensure_space(this);
2345  if (kPointerSize == kInt64Size) {
2346  emit(0x48); // REX.W
2347  emit(0xA3);
2348  emitp(dst, mode);
2349  } else {
2350  DCHECK_EQ(kPointerSize, kInt32Size);
2351  emit(0xA3);
2352  emitp(dst, mode);
2353  // In 64-bit mode, need to zero extend the operand to 8 bytes.
2354  // See 2.2.1.4 in Intel64 and IA32 Architectures Software
2355  // Developer's Manual Volume 2.
2356  emitl(0);
2357  }
2358 }
2359 
2360 
2361 void Assembler::store_rax(ExternalReference ref) {
2362  store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2363 }
2364 
2365 void Assembler::sub_sp_32(uint32_t imm) {
2366  emit_rex_64();
2367  emit(0x81); // using a literal 32-bit immediate.
2368  emit_modrm(0x5, rsp);
2369  emitl(imm);
2370 }
2371 
2372 void Assembler::testb(Register dst, Register src) {
2373  EnsureSpace ensure_space(this);
2374  emit_test(dst, src, sizeof(int8_t));
2375 }
2376 
2377 void Assembler::testb(Register reg, Immediate mask) {
2378  DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2379  emit_test(reg, mask, sizeof(int8_t));
2380 }
2381 
2382 void Assembler::testb(Operand op, Immediate mask) {
2383  DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2384  emit_test(op, mask, sizeof(int8_t));
2385 }
2386 
2387 void Assembler::testb(Operand op, Register reg) {
2388  emit_test(op, reg, sizeof(int8_t));
2389 }
2390 
2391 void Assembler::testw(Register dst, Register src) {
2392  emit_test(dst, src, sizeof(uint16_t));
2393 }
2394 
2395 void Assembler::testw(Register reg, Immediate mask) {
2396  emit_test(reg, mask, sizeof(int16_t));
2397 }
2398 
2399 void Assembler::testw(Operand op, Immediate mask) {
2400  emit_test(op, mask, sizeof(int16_t));
2401 }
2402 
2403 void Assembler::testw(Operand op, Register reg) {
2404  emit_test(op, reg, sizeof(int16_t));
2405 }
2406 
2407 void Assembler::emit_test(Register dst, Register src, int size) {
2408  EnsureSpace ensure_space(this);
2409  if (src.low_bits() == 4) std::swap(dst, src);
2410  if (size == sizeof(int16_t)) {
2411  emit(0x66);
2412  size = sizeof(int32_t);
2413  }
2414  bool byte_operand = size == sizeof(int8_t);
2415  if (byte_operand) {
2416  size = sizeof(int32_t);
2417  if (!src.is_byte_register() || !dst.is_byte_register()) {
2418  emit_rex_32(dst, src);
2419  }
2420  } else {
2421  emit_rex(dst, src, size);
2422  }
2423  emit(byte_operand ? 0x84 : 0x85);
2424  emit_modrm(dst, src);
2425 }
2426 
2427 
2428 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2429  if (is_uint8(mask.value_)) {
2430  size = sizeof(int8_t);
2431  } else if (is_uint16(mask.value_)) {
2432  size = sizeof(int16_t);
2433  }
2434  EnsureSpace ensure_space(this);
2435  bool half_word = size == sizeof(int16_t);
2436  if (half_word) {
2437  emit(0x66);
2438  size = sizeof(int32_t);
2439  }
2440  bool byte_operand = size == sizeof(int8_t);
2441  if (byte_operand) {
2442  size = sizeof(int32_t);
2443  if (!reg.is_byte_register()) emit_rex_32(reg);
2444  } else {
2445  emit_rex(reg, size);
2446  }
2447  if (reg == rax) {
2448  emit(byte_operand ? 0xA8 : 0xA9);
2449  } else {
2450  emit(byte_operand ? 0xF6 : 0xF7);
2451  emit_modrm(0x0, reg);
2452  }
2453  if (byte_operand) {
2454  emit(mask.value_);
2455  } else if (half_word) {
2456  emitw(mask.value_);
2457  } else {
2458  emit(mask);
2459  }
2460 }
2461 
2462 void Assembler::emit_test(Operand op, Immediate mask, int size) {
2463  if (is_uint8(mask.value_)) {
2464  size = sizeof(int8_t);
2465  } else if (is_uint16(mask.value_)) {
2466  size = sizeof(int16_t);
2467  }
2468  EnsureSpace ensure_space(this);
2469  bool half_word = size == sizeof(int16_t);
2470  if (half_word) {
2471  emit(0x66);
2472  size = sizeof(int32_t);
2473  }
2474  bool byte_operand = size == sizeof(int8_t);
2475  if (byte_operand) {
2476  size = sizeof(int32_t);
2477  }
2478  emit_rex(rax, op, size);
2479  emit(byte_operand ? 0xF6 : 0xF7);
2480  emit_operand(rax, op); // Operation code 0
2481  if (byte_operand) {
2482  emit(mask.value_);
2483  } else if (half_word) {
2484  emitw(mask.value_);
2485  } else {
2486  emit(mask);
2487  }
2488 }
2489 
2490 void Assembler::emit_test(Operand op, Register reg, int size) {
2491  EnsureSpace ensure_space(this);
2492  if (size == sizeof(int16_t)) {
2493  emit(0x66);
2494  size = sizeof(int32_t);
2495  }
2496  bool byte_operand = size == sizeof(int8_t);
2497  if (byte_operand) {
2498  size = sizeof(int32_t);
2499  if (!reg.is_byte_register()) {
2500  // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2501  emit_rex_32(reg, op);
2502  } else {
2503  emit_optional_rex_32(reg, op);
2504  }
2505  } else {
2506  emit_rex(reg, op, size);
2507  }
2508  emit(byte_operand ? 0x84 : 0x85);
2509  emit_operand(reg, op);
2510 }
2511 
2512 
2513 // FPU instructions.
2514 
2515 
2516 void Assembler::fld(int i) {
2517  EnsureSpace ensure_space(this);
2518  emit_farith(0xD9, 0xC0, i);
2519 }
2520 
2521 
2522 void Assembler::fld1() {
2523  EnsureSpace ensure_space(this);
2524  emit(0xD9);
2525  emit(0xE8);
2526 }
2527 
2528 
2529 void Assembler::fldz() {
2530  EnsureSpace ensure_space(this);
2531  emit(0xD9);
2532  emit(0xEE);
2533 }
2534 
2535 
2536 void Assembler::fldpi() {
2537  EnsureSpace ensure_space(this);
2538  emit(0xD9);
2539  emit(0xEB);
2540 }
2541 
2542 
2543 void Assembler::fldln2() {
2544  EnsureSpace ensure_space(this);
2545  emit(0xD9);
2546  emit(0xED);
2547 }
2548 
2549 void Assembler::fld_s(Operand adr) {
2550  EnsureSpace ensure_space(this);
2551  emit_optional_rex_32(adr);
2552  emit(0xD9);
2553  emit_operand(0, adr);
2554 }
2555 
2556 void Assembler::fld_d(Operand adr) {
2557  EnsureSpace ensure_space(this);
2558  emit_optional_rex_32(adr);
2559  emit(0xDD);
2560  emit_operand(0, adr);
2561 }
2562 
2563 void Assembler::fstp_s(Operand adr) {
2564  EnsureSpace ensure_space(this);
2565  emit_optional_rex_32(adr);
2566  emit(0xD9);
2567  emit_operand(3, adr);
2568 }
2569 
2570 void Assembler::fstp_d(Operand adr) {
2571  EnsureSpace ensure_space(this);
2572  emit_optional_rex_32(adr);
2573  emit(0xDD);
2574  emit_operand(3, adr);
2575 }
2576 
2577 
2578 void Assembler::fstp(int index) {
2579  DCHECK(is_uint3(index));
2580  EnsureSpace ensure_space(this);
2581  emit_farith(0xDD, 0xD8, index);
2582 }
2583 
2584 void Assembler::fild_s(Operand adr) {
2585  EnsureSpace ensure_space(this);
2586  emit_optional_rex_32(adr);
2587  emit(0xDB);
2588  emit_operand(0, adr);
2589 }
2590 
2591 void Assembler::fild_d(Operand adr) {
2592  EnsureSpace ensure_space(this);
2593  emit_optional_rex_32(adr);
2594  emit(0xDF);
2595  emit_operand(5, adr);
2596 }
2597 
2598 void Assembler::fistp_s(Operand adr) {
2599  EnsureSpace ensure_space(this);
2600  emit_optional_rex_32(adr);
2601  emit(0xDB);
2602  emit_operand(3, adr);
2603 }
2604 
2605 void Assembler::fisttp_s(Operand adr) {
2606  DCHECK(IsEnabled(SSE3));
2607  EnsureSpace ensure_space(this);
2608  emit_optional_rex_32(adr);
2609  emit(0xDB);
2610  emit_operand(1, adr);
2611 }
2612 
2613 void Assembler::fisttp_d(Operand adr) {
2614  DCHECK(IsEnabled(SSE3));
2615  EnsureSpace ensure_space(this);
2616  emit_optional_rex_32(adr);
2617  emit(0xDD);
2618  emit_operand(1, adr);
2619 }
2620 
2621 void Assembler::fist_s(Operand adr) {
2622  EnsureSpace ensure_space(this);
2623  emit_optional_rex_32(adr);
2624  emit(0xDB);
2625  emit_operand(2, adr);
2626 }
2627 
2628 void Assembler::fistp_d(Operand adr) {
2629  EnsureSpace ensure_space(this);
2630  emit_optional_rex_32(adr);
2631  emit(0xDF);
2632  emit_operand(7, adr);
2633 }
2634 
2635 
2636 void Assembler::fabs() {
2637  EnsureSpace ensure_space(this);
2638  emit(0xD9);
2639  emit(0xE1);
2640 }
2641 
2642 
2643 void Assembler::fchs() {
2644  EnsureSpace ensure_space(this);
2645  emit(0xD9);
2646  emit(0xE0);
2647 }
2648 
2649 
2650 void Assembler::fcos() {
2651  EnsureSpace ensure_space(this);
2652  emit(0xD9);
2653  emit(0xFF);
2654 }
2655 
2656 
2657 void Assembler::fsin() {
2658  EnsureSpace ensure_space(this);
2659  emit(0xD9);
2660  emit(0xFE);
2661 }
2662 
2663 
2664 void Assembler::fptan() {
2665  EnsureSpace ensure_space(this);
2666  emit(0xD9);
2667  emit(0xF2);
2668 }
2669 
2670 
2671 void Assembler::fyl2x() {
2672  EnsureSpace ensure_space(this);
2673  emit(0xD9);
2674  emit(0xF1);
2675 }
2676 
2677 
2678 void Assembler::f2xm1() {
2679  EnsureSpace ensure_space(this);
2680  emit(0xD9);
2681  emit(0xF0);
2682 }
2683 
2684 
2685 void Assembler::fscale() {
2686  EnsureSpace ensure_space(this);
2687  emit(0xD9);
2688  emit(0xFD);
2689 }
2690 
2691 
2692 void Assembler::fninit() {
2693  EnsureSpace ensure_space(this);
2694  emit(0xDB);
2695  emit(0xE3);
2696 }
2697 
2698 
2699 void Assembler::fadd(int i) {
2700  EnsureSpace ensure_space(this);
2701  emit_farith(0xDC, 0xC0, i);
2702 }
2703 
2704 
2705 void Assembler::fsub(int i) {
2706  EnsureSpace ensure_space(this);
2707  emit_farith(0xDC, 0xE8, i);
2708 }
2709 
2710 void Assembler::fisub_s(Operand adr) {
2711  EnsureSpace ensure_space(this);
2712  emit_optional_rex_32(adr);
2713  emit(0xDA);
2714  emit_operand(4, adr);
2715 }
2716 
2717 
2718 void Assembler::fmul(int i) {
2719  EnsureSpace ensure_space(this);
2720  emit_farith(0xDC, 0xC8, i);
2721 }
2722 
2723 
2724 void Assembler::fdiv(int i) {
2725  EnsureSpace ensure_space(this);
2726  emit_farith(0xDC, 0xF8, i);
2727 }
2728 
2729 
2730 void Assembler::faddp(int i) {
2731  EnsureSpace ensure_space(this);
2732  emit_farith(0xDE, 0xC0, i);
2733 }
2734 
2735 
2736 void Assembler::fsubp(int i) {
2737  EnsureSpace ensure_space(this);
2738  emit_farith(0xDE, 0xE8, i);
2739 }
2740 
2741 
2742 void Assembler::fsubrp(int i) {
2743  EnsureSpace ensure_space(this);
2744  emit_farith(0xDE, 0xE0, i);
2745 }
2746 
2747 
2748 void Assembler::fmulp(int i) {
2749  EnsureSpace ensure_space(this);
2750  emit_farith(0xDE, 0xC8, i);
2751 }
2752 
2753 
2754 void Assembler::fdivp(int i) {
2755  EnsureSpace ensure_space(this);
2756  emit_farith(0xDE, 0xF8, i);
2757 }
2758 
2759 
2760 void Assembler::fprem() {
2761  EnsureSpace ensure_space(this);
2762  emit(0xD9);
2763  emit(0xF8);
2764 }
2765 
2766 
2767 void Assembler::fprem1() {
2768  EnsureSpace ensure_space(this);
2769  emit(0xD9);
2770  emit(0xF5);
2771 }
2772 
2773 
2774 void Assembler::fxch(int i) {
2775  EnsureSpace ensure_space(this);
2776  emit_farith(0xD9, 0xC8, i);
2777 }
2778 
2779 
2780 void Assembler::fincstp() {
2781  EnsureSpace ensure_space(this);
2782  emit(0xD9);
2783  emit(0xF7);
2784 }
2785 
2786 
2787 void Assembler::ffree(int i) {
2788  EnsureSpace ensure_space(this);
2789  emit_farith(0xDD, 0xC0, i);
2790 }
2791 
2792 
2793 void Assembler::ftst() {
2794  EnsureSpace ensure_space(this);
2795  emit(0xD9);
2796  emit(0xE4);
2797 }
2798 
2799 
2800 void Assembler::fucomp(int i) {
2801  EnsureSpace ensure_space(this);
2802  emit_farith(0xDD, 0xE8, i);
2803 }
2804 
2805 
2806 void Assembler::fucompp() {
2807  EnsureSpace ensure_space(this);
2808  emit(0xDA);
2809  emit(0xE9);
2810 }
2811 
2812 
2813 void Assembler::fucomi(int i) {
2814  EnsureSpace ensure_space(this);
2815  emit(0xDB);
2816  emit(0xE8 + i);
2817 }
2818 
2819 
2820 void Assembler::fucomip() {
2821  EnsureSpace ensure_space(this);
2822  emit(0xDF);
2823  emit(0xE9);
2824 }
2825 
2826 
2827 void Assembler::fcompp() {
2828  EnsureSpace ensure_space(this);
2829  emit(0xDE);
2830  emit(0xD9);
2831 }
2832 
2833 
2834 void Assembler::fnstsw_ax() {
2835  EnsureSpace ensure_space(this);
2836  emit(0xDF);
2837  emit(0xE0);
2838 }
2839 
2840 
2841 void Assembler::fwait() {
2842  EnsureSpace ensure_space(this);
2843  emit(0x9B);
2844 }
2845 
2846 
2847 void Assembler::frndint() {
2848  EnsureSpace ensure_space(this);
2849  emit(0xD9);
2850  emit(0xFC);
2851 }
2852 
2853 
2854 void Assembler::fnclex() {
2855  EnsureSpace ensure_space(this);
2856  emit(0xDB);
2857  emit(0xE2);
2858 }
2859 
2860 
2861 void Assembler::sahf() {
2862  // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2863  // in 64-bit mode. Test CpuID.
2864  DCHECK(IsEnabled(SAHF));
2865  EnsureSpace ensure_space(this);
2866  emit(0x9E);
2867 }
2868 
2869 
2870 void Assembler::emit_farith(int b1, int b2, int i) {
2871  DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2872  DCHECK(is_uint3(i)); // illegal stack offset
2873  emit(b1);
2874  emit(b2 + i);
2875 }
2876 
2877 
2878 // SSE operations.
2879 
2880 void Assembler::andps(XMMRegister dst, XMMRegister src) {
2881  EnsureSpace ensure_space(this);
2882  emit_optional_rex_32(dst, src);
2883  emit(0x0F);
2884  emit(0x54);
2885  emit_sse_operand(dst, src);
2886 }
2887 
2888 void Assembler::andps(XMMRegister dst, Operand src) {
2889  EnsureSpace ensure_space(this);
2890  emit_optional_rex_32(dst, src);
2891  emit(0x0F);
2892  emit(0x54);
2893  emit_sse_operand(dst, src);
2894 }
2895 
2896 
2897 void Assembler::orps(XMMRegister dst, XMMRegister src) {
2898  EnsureSpace ensure_space(this);
2899  emit_optional_rex_32(dst, src);
2900  emit(0x0F);
2901  emit(0x56);
2902  emit_sse_operand(dst, src);
2903 }
2904 
2905 void Assembler::orps(XMMRegister dst, Operand src) {
2906  EnsureSpace ensure_space(this);
2907  emit_optional_rex_32(dst, src);
2908  emit(0x0F);
2909  emit(0x56);
2910  emit_sse_operand(dst, src);
2911 }
2912 
2913 
2914 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2915  DCHECK(!IsEnabled(AVX));
2916  EnsureSpace ensure_space(this);
2917  emit_optional_rex_32(dst, src);
2918  emit(0x0F);
2919  emit(0x57);
2920  emit_sse_operand(dst, src);
2921 }
2922 
2923 void Assembler::xorps(XMMRegister dst, Operand src) {
2924  DCHECK(!IsEnabled(AVX));
2925  EnsureSpace ensure_space(this);
2926  emit_optional_rex_32(dst, src);
2927  emit(0x0F);
2928  emit(0x57);
2929  emit_sse_operand(dst, src);
2930 }
2931 
2932 
2933 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2934  EnsureSpace ensure_space(this);
2935  emit_optional_rex_32(dst, src);
2936  emit(0x0F);
2937  emit(0x58);
2938  emit_sse_operand(dst, src);
2939 }
2940 
2941 void Assembler::addps(XMMRegister dst, Operand src) {
2942  EnsureSpace ensure_space(this);
2943  emit_optional_rex_32(dst, src);
2944  emit(0x0F);
2945  emit(0x58);
2946  emit_sse_operand(dst, src);
2947 }
2948 
2949 
2950 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2951  EnsureSpace ensure_space(this);
2952  emit_optional_rex_32(dst, src);
2953  emit(0x0F);
2954  emit(0x5C);
2955  emit_sse_operand(dst, src);
2956 }
2957 
2958 void Assembler::subps(XMMRegister dst, Operand src) {
2959  EnsureSpace ensure_space(this);
2960  emit_optional_rex_32(dst, src);
2961  emit(0x0F);
2962  emit(0x5C);
2963  emit_sse_operand(dst, src);
2964 }
2965 
2966 
2967 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
2968  EnsureSpace ensure_space(this);
2969  emit_optional_rex_32(dst, src);
2970  emit(0x0F);
2971  emit(0x59);
2972  emit_sse_operand(dst, src);
2973 }
2974 
2975 void Assembler::mulps(XMMRegister dst, Operand src) {
2976  EnsureSpace ensure_space(this);
2977  emit_optional_rex_32(dst, src);
2978  emit(0x0F);
2979  emit(0x59);
2980  emit_sse_operand(dst, src);
2981 }
2982 
2983 
2984 void Assembler::divps(XMMRegister dst, XMMRegister src) {
2985  EnsureSpace ensure_space(this);
2986  emit_optional_rex_32(dst, src);
2987  emit(0x0F);
2988  emit(0x5E);
2989  emit_sse_operand(dst, src);
2990 }
2991 
2992 void Assembler::divps(XMMRegister dst, Operand src) {
2993  EnsureSpace ensure_space(this);
2994  emit_optional_rex_32(dst, src);
2995  emit(0x0F);
2996  emit(0x5E);
2997  emit_sse_operand(dst, src);
2998 }
2999 
3000 
3001 // SSE 2 operations.
3002 
3003 void Assembler::movd(XMMRegister dst, Register src) {
3004  DCHECK(!IsEnabled(AVX));
3005  EnsureSpace ensure_space(this);
3006  emit(0x66);
3007  emit_optional_rex_32(dst, src);
3008  emit(0x0F);
3009  emit(0x6E);
3010  emit_sse_operand(dst, src);
3011 }
3012 
3013 void Assembler::movd(XMMRegister dst, Operand src) {
3014  DCHECK(!IsEnabled(AVX));
3015  EnsureSpace ensure_space(this);
3016  emit(0x66);
3017  emit_optional_rex_32(dst, src);
3018  emit(0x0F);
3019  emit(0x6E);
3020  emit_sse_operand(dst, src);
3021 }
3022 
3023 
3024 void Assembler::movd(Register dst, XMMRegister src) {
3025  DCHECK(!IsEnabled(AVX));
3026  EnsureSpace ensure_space(this);
3027  emit(0x66);
3028  emit_optional_rex_32(src, dst);
3029  emit(0x0F);
3030  emit(0x7E);
3031  emit_sse_operand(src, dst);
3032 }
3033 
3034 
3035 void Assembler::movq(XMMRegister dst, Register src) {
3036  DCHECK(!IsEnabled(AVX));
3037  EnsureSpace ensure_space(this);
3038  emit(0x66);
3039  emit_rex_64(dst, src);
3040  emit(0x0F);
3041  emit(0x6E);
3042  emit_sse_operand(dst, src);
3043 }
3044 
3045 
3046 void Assembler::movq(Register dst, XMMRegister src) {
3047  DCHECK(!IsEnabled(AVX));
3048  EnsureSpace ensure_space(this);
3049  emit(0x66);
3050  emit_rex_64(src, dst);
3051  emit(0x0F);
3052  emit(0x7E);
3053  emit_sse_operand(src, dst);
3054 }
3055 
3056 
3057 void Assembler::movq(XMMRegister dst, XMMRegister src) {
3058  DCHECK(!IsEnabled(AVX));
3059  EnsureSpace ensure_space(this);
3060  if (dst.low_bits() == 4) {
3061  // Avoid unnecessary SIB byte.
3062  emit(0xF3);
3063  emit_optional_rex_32(dst, src);
3064  emit(0x0F);
3065  emit(0x7E);
3066  emit_sse_operand(dst, src);
3067  } else {
3068  emit(0x66);
3069  emit_optional_rex_32(src, dst);
3070  emit(0x0F);
3071  emit(0xD6);
3072  emit_sse_operand(src, dst);
3073  }
3074 }
3075 
3076 void Assembler::movdqa(Operand dst, XMMRegister src) {
3077  EnsureSpace ensure_space(this);
3078  emit(0x66);
3079  emit_rex_64(src, dst);
3080  emit(0x0F);
3081  emit(0x7F);
3082  emit_sse_operand(src, dst);
3083 }
3084 
3085 void Assembler::movdqa(XMMRegister dst, Operand src) {
3086  EnsureSpace ensure_space(this);
3087  emit(0x66);
3088  emit_rex_64(dst, src);
3089  emit(0x0F);
3090  emit(0x6F);
3091  emit_sse_operand(dst, src);
3092 }
3093 
3094 void Assembler::movdqu(Operand dst, XMMRegister src) {
3095  EnsureSpace ensure_space(this);
3096  emit(0xF3);
3097  emit_rex_64(src, dst);
3098  emit(0x0F);
3099  emit(0x7F);
3100  emit_sse_operand(src, dst);
3101 }
3102 
3103 void Assembler::movdqu(XMMRegister dst, Operand src) {
3104  EnsureSpace ensure_space(this);
3105  emit(0xF3);
3106  emit_rex_64(dst, src);
3107  emit(0x0F);
3108  emit(0x6F);
3109  emit_sse_operand(dst, src);
3110 }
3111 
3112 
3113 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
3114  DCHECK(IsEnabled(SSE4_1));
3115  DCHECK(is_uint8(imm8));
3116  EnsureSpace ensure_space(this);
3117  emit(0x66);
3118  emit_optional_rex_32(src, dst);
3119  emit(0x0F);
3120  emit(0x3A);
3121  emit(0x17);
3122  emit_sse_operand(src, dst);
3123  emit(imm8);
3124 }
3125 
3126 void Assembler::pextrb(Register dst, XMMRegister src, int8_t imm8) {
3127  DCHECK(IsEnabled(SSE4_1));
3128  DCHECK(is_uint8(imm8));
3129  EnsureSpace ensure_space(this);
3130  emit(0x66);
3131  emit_optional_rex_32(src, dst);
3132  emit(0x0F);
3133  emit(0x3A);
3134  emit(0x14);
3135  emit_sse_operand(src, dst);
3136  emit(imm8);
3137 }
3138 
3139 void Assembler::pextrb(Operand dst, XMMRegister src, int8_t imm8) {
3140  DCHECK(IsEnabled(SSE4_1));
3141  DCHECK(is_uint8(imm8));
3142  EnsureSpace ensure_space(this);
3143  emit(0x66);
3144  emit_optional_rex_32(src, dst);
3145  emit(0x0F);
3146  emit(0x3A);
3147  emit(0x14);
3148  emit_sse_operand(src, dst);
3149  emit(imm8);
3150 }
3151 
3152 void Assembler::pinsrw(XMMRegister dst, Register src, int8_t imm8) {
3153  DCHECK(is_uint8(imm8));
3154  EnsureSpace ensure_space(this);
3155  emit(0x66);
3156  emit_optional_rex_32(dst, src);
3157  emit(0x0F);
3158  emit(0xC4);
3159  emit_sse_operand(dst, src);
3160  emit(imm8);
3161 }
3162 
3163 void Assembler::pinsrw(XMMRegister dst, Operand src, int8_t imm8) {
3164  DCHECK(is_uint8(imm8));
3165  EnsureSpace ensure_space(this);
3166  emit(0x66);
3167  emit_optional_rex_32(dst, src);
3168  emit(0x0F);
3169  emit(0xC4);
3170  emit_sse_operand(dst, src);
3171  emit(imm8);
3172 }
3173 
3174 void Assembler::pextrw(Register dst, XMMRegister src, int8_t imm8) {
3175  DCHECK(IsEnabled(SSE4_1));
3176  DCHECK(is_uint8(imm8));
3177  EnsureSpace ensure_space(this);
3178  emit(0x66);
3179  emit_optional_rex_32(src, dst);
3180  emit(0x0F);
3181  emit(0x3A);
3182  emit(0x15);
3183  emit_sse_operand(src, dst);
3184  emit(imm8);
3185 }
3186 
3187 void Assembler::pextrw(Operand dst, XMMRegister src, int8_t imm8) {
3188  DCHECK(IsEnabled(SSE4_1));
3189  DCHECK(is_uint8(imm8));
3190  EnsureSpace ensure_space(this);
3191  emit(0x66);
3192  emit_optional_rex_32(src, dst);
3193  emit(0x0F);
3194  emit(0x3A);
3195  emit(0x15);
3196  emit_sse_operand(src, dst);
3197  emit(imm8);
3198 }
3199 
3200 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) {
3201  DCHECK(IsEnabled(SSE4_1));
3202  EnsureSpace ensure_space(this);
3203  emit(0x66);
3204  emit_optional_rex_32(src, dst);
3205  emit(0x0F);
3206  emit(0x3A);
3207  emit(0x16);
3208  emit_sse_operand(src, dst);
3209  emit(imm8);
3210 }
3211 
3212 void Assembler::pextrd(Operand dst, XMMRegister src, int8_t imm8) {
3213  DCHECK(IsEnabled(SSE4_1));
3214  EnsureSpace ensure_space(this);
3215  emit(0x66);
3216  emit_optional_rex_32(src, dst);
3217  emit(0x0F);
3218  emit(0x3A);
3219  emit(0x16);
3220  emit_sse_operand(src, dst);
3221  emit(imm8);
3222 }
3223 
3224 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) {
3225  DCHECK(IsEnabled(SSE4_1));
3226  EnsureSpace ensure_space(this);
3227  emit(0x66);
3228  emit_optional_rex_32(dst, src);
3229  emit(0x0F);
3230  emit(0x3A);
3231  emit(0x22);
3232  emit_sse_operand(dst, src);
3233  emit(imm8);
3234 }
3235 
3236 void Assembler::pinsrd(XMMRegister dst, Operand src, int8_t imm8) {
3237  DCHECK(IsEnabled(SSE4_1));
3238  EnsureSpace ensure_space(this);
3239  emit(0x66);
3240  emit_optional_rex_32(dst, src);
3241  emit(0x0F);
3242  emit(0x3A);
3243  emit(0x22);
3244  emit_sse_operand(dst, src);
3245  emit(imm8);
3246 }
3247 
3248 void Assembler::pinsrb(XMMRegister dst, Register src, int8_t imm8) {
3249  DCHECK(IsEnabled(SSE4_1));
3250  EnsureSpace ensure_space(this);
3251  emit(0x66);
3252  emit_optional_rex_32(dst, src);
3253  emit(0x0F);
3254  emit(0x3A);
3255  emit(0x20);
3256  emit_sse_operand(dst, src);
3257  emit(imm8);
3258 }
3259 
3260 void Assembler::pinsrb(XMMRegister dst, Operand src, int8_t imm8) {
3261  DCHECK(IsEnabled(SSE4_1));
3262  EnsureSpace ensure_space(this);
3263  emit(0x66);
3264  emit_optional_rex_32(dst, src);
3265  emit(0x0F);
3266  emit(0x3A);
3267  emit(0x20);
3268  emit_sse_operand(dst, src);
3269  emit(imm8);
3270 }
3271 
3272 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
3273  DCHECK(CpuFeatures::IsSupported(SSE4_1));
3274  DCHECK(is_uint8(imm8));
3275  EnsureSpace ensure_space(this);
3276  emit(0x66);
3277  emit_optional_rex_32(dst, src);
3278  emit(0x0F);
3279  emit(0x3A);
3280  emit(0x21);
3281  emit_sse_operand(dst, src);
3282  emit(imm8);
3283 }
3284 
3285 void Assembler::movsd(Operand dst, XMMRegister src) {
3286  DCHECK(!IsEnabled(AVX));
3287  EnsureSpace ensure_space(this);
3288  emit(0xF2); // double
3289  emit_optional_rex_32(src, dst);
3290  emit(0x0F);
3291  emit(0x11); // store
3292  emit_sse_operand(src, dst);
3293 }
3294 
3295 
3296 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3297  DCHECK(!IsEnabled(AVX));
3298  EnsureSpace ensure_space(this);
3299  emit(0xF2); // double
3300  emit_optional_rex_32(dst, src);
3301  emit(0x0F);
3302  emit(0x10); // load
3303  emit_sse_operand(dst, src);
3304 }
3305 
3306 void Assembler::movsd(XMMRegister dst, Operand src) {
3307  DCHECK(!IsEnabled(AVX));
3308  EnsureSpace ensure_space(this);
3309  emit(0xF2); // double
3310  emit_optional_rex_32(dst, src);
3311  emit(0x0F);
3312  emit(0x10); // load
3313  emit_sse_operand(dst, src);
3314 }
3315 
3316 
3317 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
3318  DCHECK(!IsEnabled(AVX));
3319  EnsureSpace ensure_space(this);
3320  if (src.low_bits() == 4) {
3321  // Try to avoid an unnecessary SIB byte.
3322  emit_optional_rex_32(src, dst);
3323  emit(0x0F);
3324  emit(0x29);
3325  emit_sse_operand(src, dst);
3326  } else {
3327  emit_optional_rex_32(dst, src);
3328  emit(0x0F);
3329  emit(0x28);
3330  emit_sse_operand(dst, src);
3331  }
3332 }
3333 
3334 
3335 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
3336  DCHECK(is_uint8(imm8));
3337  EnsureSpace ensure_space(this);
3338  emit_optional_rex_32(dst, src);
3339  emit(0x0F);
3340  emit(0xC6);
3341  emit_sse_operand(dst, src);
3342  emit(imm8);
3343 }
3344 
3345 
3346 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
3347  DCHECK(!IsEnabled(AVX));
3348  EnsureSpace ensure_space(this);
3349  if (src.low_bits() == 4) {
3350  // Try to avoid an unnecessary SIB byte.
3351  emit(0x66);
3352  emit_optional_rex_32(src, dst);
3353  emit(0x0F);
3354  emit(0x29);
3355  emit_sse_operand(src, dst);
3356  } else {
3357  emit(0x66);
3358  emit_optional_rex_32(dst, src);
3359  emit(0x0F);
3360  emit(0x28);
3361  emit_sse_operand(dst, src);
3362  }
3363 }
3364 
3365 void Assembler::movupd(XMMRegister dst, Operand src) {
3366  EnsureSpace ensure_space(this);
3367  emit(0x66);
3368  emit_optional_rex_32(dst, src);
3369  emit(0x0F);
3370  emit(0x10);
3371  emit_sse_operand(dst, src);
3372 }
3373 
3374 void Assembler::movupd(Operand dst, XMMRegister src) {
3375  EnsureSpace ensure_space(this);
3376  emit(0x66);
3377  emit_optional_rex_32(src, dst);
3378  emit(0x0F);
3379  emit(0x11);
3380  emit_sse_operand(src, dst);
3381 }
3382 
3383 void Assembler::addss(XMMRegister dst, XMMRegister src) {
3384  EnsureSpace ensure_space(this);
3385  emit(0xF3);
3386  emit_optional_rex_32(dst, src);
3387  emit(0x0F);
3388  emit(0x58);
3389  emit_sse_operand(dst, src);
3390 }
3391 
3392 void Assembler::addss(XMMRegister dst, Operand src) {
3393  EnsureSpace ensure_space(this);
3394  emit(0xF3);
3395  emit_optional_rex_32(dst, src);
3396  emit(0x0F);
3397  emit(0x58);
3398  emit_sse_operand(dst, src);
3399 }
3400 
3401 
3402 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3403  EnsureSpace ensure_space(this);
3404  emit(0xF3);
3405  emit_optional_rex_32(dst, src);
3406  emit(0x0F);
3407  emit(0x5C);
3408  emit_sse_operand(dst, src);
3409 }
3410 
3411 void Assembler::subss(XMMRegister dst, Operand src) {
3412  EnsureSpace ensure_space(this);
3413  emit(0xF3);
3414  emit_optional_rex_32(dst, src);
3415  emit(0x0F);
3416  emit(0x5C);
3417  emit_sse_operand(dst, src);
3418 }
3419 
3420 
3421 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3422  EnsureSpace ensure_space(this);
3423  emit(0xF3);
3424  emit_optional_rex_32(dst, src);
3425  emit(0x0F);
3426  emit(0x59);
3427  emit_sse_operand(dst, src);
3428 }
3429 
3430 void Assembler::mulss(XMMRegister dst, Operand src) {
3431  EnsureSpace ensure_space(this);
3432  emit(0xF3);
3433  emit_optional_rex_32(dst, src);
3434  emit(0x0F);
3435  emit(0x59);
3436  emit_sse_operand(dst, src);
3437 }
3438 
3439 
3440 void Assembler::divss(XMMRegister dst, XMMRegister src) {
3441  EnsureSpace ensure_space(this);
3442  emit(0xF3);
3443  emit_optional_rex_32(dst, src);
3444  emit(0x0F);
3445  emit(0x5E);
3446  emit_sse_operand(dst, src);
3447 }
3448 
3449 void Assembler::divss(XMMRegister dst, Operand src) {
3450  EnsureSpace ensure_space(this);
3451  emit(0xF3);
3452  emit_optional_rex_32(dst, src);
3453  emit(0x0F);
3454  emit(0x5E);
3455  emit_sse_operand(dst, src);
3456 }
3457 
3458 
3459 void Assembler::maxss(XMMRegister dst, XMMRegister src) {
3460  EnsureSpace ensure_space(this);
3461  emit(0xF3);
3462  emit_optional_rex_32(dst, src);
3463  emit(0x0F);
3464  emit(0x5F);
3465  emit_sse_operand(dst, src);
3466 }
3467 
3468 void Assembler::maxss(XMMRegister dst, Operand src) {
3469  EnsureSpace ensure_space(this);
3470  emit(0xF3);
3471  emit_optional_rex_32(dst, src);
3472  emit(0x0F);
3473  emit(0x5F);
3474  emit_sse_operand(dst, src);
3475 }
3476 
3477 
3478 void Assembler::minss(XMMRegister dst, XMMRegister src) {
3479  EnsureSpace ensure_space(this);
3480  emit(0xF3);
3481  emit_optional_rex_32(dst, src);
3482  emit(0x0F);
3483  emit(0x5D);
3484  emit_sse_operand(dst, src);
3485 }
3486 
3487 void Assembler::minss(XMMRegister dst, Operand src) {
3488  EnsureSpace ensure_space(this);
3489  emit(0xF3);
3490  emit_optional_rex_32(dst, src);
3491  emit(0x0F);
3492  emit(0x5D);
3493  emit_sse_operand(dst, src);
3494 }
3495 
3496 
3497 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3498  EnsureSpace ensure_space(this);
3499  emit(0xF3);
3500  emit_optional_rex_32(dst, src);
3501  emit(0x0F);
3502  emit(0x51);
3503  emit_sse_operand(dst, src);
3504 }
3505 
3506 void Assembler::sqrtss(XMMRegister dst, Operand src) {
3507  EnsureSpace ensure_space(this);
3508  emit(0xF3);
3509  emit_optional_rex_32(dst, src);
3510  emit(0x0F);
3511  emit(0x51);
3512  emit_sse_operand(dst, src);
3513 }
3514 
3515 
3516 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3517  DCHECK(!IsEnabled(AVX));
3518  EnsureSpace ensure_space(this);
3519  emit_optional_rex_32(dst, src);
3520  emit(0x0F);
3521  emit(0x2E);
3522  emit_sse_operand(dst, src);
3523 }
3524 
3525 void Assembler::ucomiss(XMMRegister dst, Operand src) {
3526  DCHECK(!IsEnabled(AVX));
3527  EnsureSpace ensure_space(this);
3528  emit_optional_rex_32(dst, src);
3529  emit(0x0F);
3530  emit(0x2E);
3531  emit_sse_operand(dst, src);
3532 }
3533 
3534 
3535 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3536  DCHECK(!IsEnabled(AVX));
3537  EnsureSpace ensure_space(this);
3538  emit(0xF3); // single
3539  emit_optional_rex_32(dst, src);
3540  emit(0x0F);
3541  emit(0x10); // load
3542  emit_sse_operand(dst, src);
3543 }
3544 
3545 void Assembler::movss(XMMRegister dst, Operand src) {
3546  DCHECK(!IsEnabled(AVX));
3547  EnsureSpace ensure_space(this);
3548  emit(0xF3); // single
3549  emit_optional_rex_32(dst, src);
3550  emit(0x0F);
3551  emit(0x10); // load
3552  emit_sse_operand(dst, src);
3553 }
3554 
3555 void Assembler::movss(Operand src, XMMRegister dst) {
3556  DCHECK(!IsEnabled(AVX));
3557  EnsureSpace ensure_space(this);
3558  emit(0xF3); // single
3559  emit_optional_rex_32(dst, src);
3560  emit(0x0F);
3561  emit(0x11); // store
3562  emit_sse_operand(dst, src);
3563 }
3564 
3565 
3566 void Assembler::psllq(XMMRegister reg, byte imm8) {
3567  DCHECK(!IsEnabled(AVX));
3568  EnsureSpace ensure_space(this);
3569  emit(0x66);
3570  emit_optional_rex_32(reg);
3571  emit(0x0F);
3572  emit(0x73);
3573  emit_sse_operand(rsi, reg); // rsi == 6
3574  emit(imm8);
3575 }
3576 
3577 
3578 void Assembler::psrlq(XMMRegister reg, byte imm8) {
3579  DCHECK(!IsEnabled(AVX));
3580  EnsureSpace ensure_space(this);
3581  emit(0x66);
3582  emit_optional_rex_32(reg);
3583  emit(0x0F);
3584  emit(0x73);
3585  emit_sse_operand(rdx, reg); // rdx == 2
3586  emit(imm8);
3587 }
3588 
3589 void Assembler::psllw(XMMRegister reg, byte imm8) {
3590  EnsureSpace ensure_space(this);
3591  emit(0x66);
3592  emit_optional_rex_32(reg);
3593  emit(0x0F);
3594  emit(0x71);
3595  emit_sse_operand(rsi, reg); // rsi == 6
3596  emit(imm8);
3597 }
3598 
3599 void Assembler::pslld(XMMRegister reg, byte imm8) {
3600  EnsureSpace ensure_space(this);
3601  emit(0x66);
3602  emit_optional_rex_32(reg);
3603  emit(0x0F);
3604  emit(0x72);
3605  emit_sse_operand(rsi, reg); // rsi == 6
3606  emit(imm8);
3607 }
3608 
3609 void Assembler::psrlw(XMMRegister reg, byte imm8) {
3610  EnsureSpace ensure_space(this);
3611  emit(0x66);
3612  emit_optional_rex_32(reg);
3613  emit(0x0F);
3614  emit(0x71);
3615  emit_sse_operand(rdx, reg); // rdx == 2
3616  emit(imm8);
3617 }
3618 
3619 void Assembler::psrld(XMMRegister reg, byte imm8) {
3620  EnsureSpace ensure_space(this);
3621  emit(0x66);
3622  emit_optional_rex_32(reg);
3623  emit(0x0F);
3624  emit(0x72);
3625  emit_sse_operand(rdx, reg); // rdx == 2
3626  emit(imm8);
3627 }
3628 
3629 void Assembler::psraw(XMMRegister reg, byte imm8) {
3630  EnsureSpace ensure_space(this);
3631  emit(0x66);
3632  emit_optional_rex_32(reg);
3633  emit(0x0F);
3634  emit(0x71);
3635  emit_sse_operand(rsp, reg); // rsp == 4
3636  emit(imm8);
3637 }
3638 
3639 void Assembler::psrad(XMMRegister reg, byte imm8) {
3640  EnsureSpace ensure_space(this);
3641  emit(0x66);
3642  emit_optional_rex_32(reg);
3643  emit(0x0F);
3644  emit(0x72);
3645  emit_sse_operand(rsp, reg); // rsp == 4
3646  emit(imm8);
3647 }
3648 
3649 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3650  EnsureSpace ensure_space(this);
3651  emit_optional_rex_32(dst, src);
3652  emit(0x0F);
3653  emit(0xC2);
3654  emit_sse_operand(dst, src);
3655  emit(cmp);
3656 }
3657 
3658 void Assembler::cmpps(XMMRegister dst, Operand src, int8_t cmp) {
3659  EnsureSpace ensure_space(this);
3660  emit_optional_rex_32(dst, src);
3661  emit(0x0F);
3662  emit(0xC2);
3663  emit_sse_operand(dst, src);
3664  emit(cmp);
3665 }
3666 
3667 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3668  EnsureSpace ensure_space(this);
3669  emit_optional_rex_32(dst, src);
3670  emit(0x66);
3671  emit(0x0F);
3672  emit(0xC2);
3673  emit_sse_operand(dst, src);
3674  emit(cmp);
3675 }
3676 
3677 void Assembler::cmppd(XMMRegister dst, Operand src, int8_t cmp) {
3678  EnsureSpace ensure_space(this);
3679  emit_optional_rex_32(dst, src);
3680  emit(0x66);
3681  emit(0x0F);
3682  emit(0xC2);
3683  emit_sse_operand(dst, src);
3684  emit(cmp);
3685 }
3686 
3687 void Assembler::cvttss2si(Register dst, Operand src) {
3688  DCHECK(!IsEnabled(AVX));
3689  EnsureSpace ensure_space(this);
3690  emit(0xF3);
3691  emit_optional_rex_32(dst, src);
3692  emit(0x0F);
3693  emit(0x2C);
3694  emit_operand(dst, src);
3695 }
3696 
3697 
3698 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3699  DCHECK(!IsEnabled(AVX));
3700  EnsureSpace ensure_space(this);
3701  emit(0xF3);
3702  emit_optional_rex_32(dst, src);
3703  emit(0x0F);
3704  emit(0x2C);
3705  emit_sse_operand(dst, src);
3706 }
3707 
3708 void Assembler::cvttsd2si(Register dst, Operand src) {
3709  DCHECK(!IsEnabled(AVX));
3710  EnsureSpace ensure_space(this);
3711  emit(0xF2);
3712  emit_optional_rex_32(dst, src);
3713  emit(0x0F);
3714  emit(0x2C);
3715  emit_operand(dst, src);
3716 }
3717 
3718 
3719 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3720  DCHECK(!IsEnabled(AVX));
3721  EnsureSpace ensure_space(this);
3722  emit(0xF2);
3723  emit_optional_rex_32(dst, src);
3724  emit(0x0F);
3725  emit(0x2C);
3726  emit_sse_operand(dst, src);
3727 }
3728 
3729 
3730 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3731  DCHECK(!IsEnabled(AVX));
3732  EnsureSpace ensure_space(this);
3733  emit(0xF3);
3734  emit_rex_64(dst, src);
3735  emit(0x0F);
3736  emit(0x2C);
3737  emit_sse_operand(dst, src);
3738 }
3739 
3740 void Assembler::cvttss2siq(Register dst, Operand src) {
3741  DCHECK(!IsEnabled(AVX));
3742  EnsureSpace ensure_space(this);
3743  emit(0xF3);
3744  emit_rex_64(dst, src);
3745  emit(0x0F);
3746  emit(0x2C);
3747  emit_sse_operand(dst, src);
3748 }
3749 
3750 
3751 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3752  DCHECK(!IsEnabled(AVX));
3753  EnsureSpace ensure_space(this);
3754  emit(0xF2);
3755  emit_rex_64(dst, src);
3756  emit(0x0F);
3757  emit(0x2C);
3758  emit_sse_operand(dst, src);
3759 }
3760 
3761 void Assembler::cvttsd2siq(Register dst, Operand src) {
3762  DCHECK(!IsEnabled(AVX));
3763  EnsureSpace ensure_space(this);
3764  emit(0xF2);
3765  emit_rex_64(dst, src);
3766  emit(0x0F);
3767  emit(0x2C);
3768  emit_sse_operand(dst, src);
3769 }
3770 
3771 void Assembler::cvttps2dq(XMMRegister dst, Operand src) {
3772  EnsureSpace ensure_space(this);
3773  emit(0xF3);
3774  emit_rex_64(dst, src);
3775  emit(0x0F);
3776  emit(0x5B);
3777  emit_sse_operand(dst, src);
3778 }
3779 
3780 void Assembler::cvttps2dq(XMMRegister dst, XMMRegister src) {
3781  EnsureSpace ensure_space(this);
3782  emit(0xF3);
3783  emit_rex_64(dst, src);
3784  emit(0x0F);
3785  emit(0x5B);
3786  emit_sse_operand(dst, src);
3787 }
3788 
3789 void Assembler::cvtlsi2sd(XMMRegister dst, Operand src) {
3790  DCHECK(!IsEnabled(AVX));
3791  EnsureSpace ensure_space(this);
3792  emit(0xF2);
3793  emit_optional_rex_32(dst, src);
3794  emit(0x0F);
3795  emit(0x2A);
3796  emit_sse_operand(dst, src);
3797 }
3798 
3799 
3800 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3801  DCHECK(!IsEnabled(AVX));
3802  EnsureSpace ensure_space(this);
3803  emit(0xF2);
3804  emit_optional_rex_32(dst, src);
3805  emit(0x0F);
3806  emit(0x2A);
3807  emit_sse_operand(dst, src);
3808 }
3809 
3810 void Assembler::cvtlsi2ss(XMMRegister dst, Operand src) {
3811  DCHECK(!IsEnabled(AVX));
3812  EnsureSpace ensure_space(this);
3813  emit(0xF3);
3814  emit_optional_rex_32(dst, src);
3815  emit(0x0F);
3816  emit(0x2A);
3817  emit_sse_operand(dst, src);
3818 }
3819 
3820 
3821 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3822  EnsureSpace ensure_space(this);
3823  emit(0xF3);
3824  emit_optional_rex_32(dst, src);
3825  emit(0x0F);
3826  emit(0x2A);
3827  emit_sse_operand(dst, src);
3828 }
3829 
3830 void Assembler::cvtqsi2ss(XMMRegister dst, Operand src) {
3831  DCHECK(!IsEnabled(AVX));
3832  EnsureSpace ensure_space(this);
3833  emit(0xF3);
3834  emit_rex_64(dst, src);
3835  emit(0x0F);
3836  emit(0x2A);
3837  emit_sse_operand(dst, src);
3838 }
3839 
3840 
3841 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3842  DCHECK(!IsEnabled(AVX));
3843  EnsureSpace ensure_space(this);
3844  emit(0xF3);
3845  emit_rex_64(dst, src);
3846  emit(0x0F);
3847  emit(0x2A);
3848  emit_sse_operand(dst, src);
3849 }
3850 
3851 void Assembler::cvtqsi2sd(XMMRegister dst, Operand src) {
3852  DCHECK(!IsEnabled(AVX));
3853  EnsureSpace ensure_space(this);
3854  emit(0xF2);
3855  emit_rex_64(dst, src);
3856  emit(0x0F);
3857  emit(0x2A);
3858  emit_sse_operand(dst, src);
3859 }
3860 
3861 
3862 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3863  DCHECK(!IsEnabled(AVX));
3864  EnsureSpace ensure_space(this);
3865  emit(0xF2);
3866  emit_rex_64(dst, src);
3867  emit(0x0F);
3868  emit(0x2A);
3869  emit_sse_operand(dst, src);
3870 }
3871 
3872 
3873 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
3874  DCHECK(!IsEnabled(AVX));
3875  EnsureSpace ensure_space(this);
3876  emit(0xF3);
3877  emit_optional_rex_32(dst, src);
3878  emit(0x0F);
3879  emit(0x5A);
3880  emit_sse_operand(dst, src);
3881 }
3882 
3883 void Assembler::cvtss2sd(XMMRegister dst, Operand src) {
3884  DCHECK(!IsEnabled(AVX));
3885  EnsureSpace ensure_space(this);
3886  emit(0xF3);
3887  emit_optional_rex_32(dst, src);
3888  emit(0x0F);
3889  emit(0x5A);
3890  emit_sse_operand(dst, src);
3891 }
3892 
3893 
3894 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
3895  DCHECK(!IsEnabled(AVX));
3896  EnsureSpace ensure_space(this);
3897  emit(0xF2);
3898  emit_optional_rex_32(dst, src);
3899  emit(0x0F);
3900  emit(0x5A);
3901  emit_sse_operand(dst, src);
3902 }
3903 
3904 void Assembler::cvtsd2ss(XMMRegister dst, Operand src) {
3905  DCHECK(!IsEnabled(AVX));
3906  EnsureSpace ensure_space(this);
3907  emit(0xF2);
3908  emit_optional_rex_32(dst, src);
3909  emit(0x0F);
3910  emit(0x5A);
3911  emit_sse_operand(dst, src);
3912 }
3913 
3914 
3915 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3916  DCHECK(!IsEnabled(AVX));
3917  EnsureSpace ensure_space(this);
3918  emit(0xF2);
3919  emit_optional_rex_32(dst, src);
3920  emit(0x0F);
3921  emit(0x2D);
3922  emit_sse_operand(dst, src);
3923 }
3924 
3925 
3926 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3927  DCHECK(!IsEnabled(AVX));
3928  EnsureSpace ensure_space(this);
3929  emit(0xF2);
3930  emit_rex_64(dst, src);
3931  emit(0x0F);
3932  emit(0x2D);
3933  emit_sse_operand(dst, src);
3934 }
3935 
3936 
3937 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
3938  EnsureSpace ensure_space(this);
3939  emit(0xF2);
3940  emit_optional_rex_32(dst, src);
3941  emit(0x0F);
3942  emit(0x58);
3943  emit_sse_operand(dst, src);
3944 }
3945 
3946 void Assembler::addsd(XMMRegister dst, Operand src) {
3947  EnsureSpace ensure_space(this);
3948  emit(0xF2);
3949  emit_optional_rex_32(dst, src);
3950  emit(0x0F);
3951  emit(0x58);
3952  emit_sse_operand(dst, src);
3953 }
3954 
3955 
3956 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3957  EnsureSpace ensure_space(this);
3958  emit(0xF2);
3959  emit_optional_rex_32(dst, src);
3960  emit(0x0F);
3961  emit(0x59);
3962  emit_sse_operand(dst, src);
3963 }
3964 
3965 void Assembler::mulsd(XMMRegister dst, Operand src) {
3966  EnsureSpace ensure_space(this);
3967  emit(0xF2);
3968  emit_optional_rex_32(dst, src);
3969  emit(0x0F);
3970  emit(0x59);
3971  emit_sse_operand(dst, src);
3972 }
3973 
3974 
3975 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3976  EnsureSpace ensure_space(this);
3977  emit(0xF2);
3978  emit_optional_rex_32(dst, src);
3979  emit(0x0F);
3980  emit(0x5C);
3981  emit_sse_operand(dst, src);
3982 }
3983 
3984 void Assembler::subsd(XMMRegister dst, Operand src) {
3985  EnsureSpace ensure_space(this);
3986  emit(0xF2);
3987  emit_optional_rex_32(dst, src);
3988  emit(0x0F);
3989  emit(0x5C);
3990  emit_sse_operand(dst, src);
3991 }
3992 
3993 
3994 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
3995  EnsureSpace ensure_space(this);
3996  emit(0xF2);
3997  emit_optional_rex_32(dst, src);
3998  emit(0x0F);
3999  emit(0x5E);
4000  emit_sse_operand(dst, src);
4001 }
4002 
4003 void Assembler::divsd(XMMRegister dst, Operand src) {
4004  EnsureSpace ensure_space(this);
4005  emit(0xF2);
4006  emit_optional_rex_32(dst, src);
4007  emit(0x0F);
4008  emit(0x5E);
4009  emit_sse_operand(dst, src);
4010 }
4011 
4012 
4013 void Assembler::maxsd(XMMRegister dst, XMMRegister src) {
4014  EnsureSpace ensure_space(this);
4015  emit(0xF2);
4016  emit_optional_rex_32(dst, src);
4017  emit(0x0F);
4018  emit(0x5F);
4019  emit_sse_operand(dst, src);
4020 }
4021 
4022 void Assembler::maxsd(XMMRegister dst, Operand src) {
4023  EnsureSpace ensure_space(this);
4024  emit(0xF2);
4025  emit_optional_rex_32(dst, src);
4026  emit(0x0F);
4027  emit(0x5F);
4028  emit_sse_operand(dst, src);
4029 }
4030 
4031 
4032 void Assembler::minsd(XMMRegister dst, XMMRegister src) {
4033  EnsureSpace ensure_space(this);
4034  emit(0xF2);
4035  emit_optional_rex_32(dst, src);
4036  emit(0x0F);
4037  emit(0x5D);
4038  emit_sse_operand(dst, src);
4039 }
4040 
4041 void Assembler::minsd(XMMRegister dst, Operand src) {
4042  EnsureSpace ensure_space(this);
4043  emit(0xF2);
4044  emit_optional_rex_32(dst, src);
4045  emit(0x0F);
4046  emit(0x5D);
4047  emit_sse_operand(dst, src);
4048 }
4049 
4050 
4051 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4052  EnsureSpace ensure_space(this);
4053  emit(0x66);
4054  emit_optional_rex_32(dst, src);
4055  emit(0x0F);
4056  emit(0x54);
4057  emit_sse_operand(dst, src);
4058 }
4059 
4060 void Assembler::andpd(XMMRegister dst, Operand src) {
4061  EnsureSpace ensure_space(this);
4062  emit(0x66);
4063  emit_optional_rex_32(dst, src);
4064  emit(0x0F);
4065  emit(0x54);
4066  emit_sse_operand(dst, src);
4067 }
4068 
4069 
4070 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
4071  EnsureSpace ensure_space(this);
4072  emit(0x66);
4073  emit_optional_rex_32(dst, src);
4074  emit(0x0F);
4075  emit(0x56);
4076  emit_sse_operand(dst, src);
4077 }
4078 
4079 void Assembler::orpd(XMMRegister dst, Operand src) {
4080  EnsureSpace ensure_space(this);
4081  emit(0x66);
4082  emit_optional_rex_32(dst, src);
4083  emit(0x0F);
4084  emit(0x56);
4085  emit_sse_operand(dst, src);
4086 }
4087 
4088 
4089 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4090  DCHECK(!IsEnabled(AVX));
4091  EnsureSpace ensure_space(this);
4092  emit(0x66);
4093  emit_optional_rex_32(dst, src);
4094  emit(0x0F);
4095  emit(0x57);
4096  emit_sse_operand(dst, src);
4097 }
4098 
4099 void Assembler::xorpd(XMMRegister dst, Operand src) {
4100  DCHECK(!IsEnabled(AVX));
4101  EnsureSpace ensure_space(this);
4102  emit(0x66);
4103  emit_optional_rex_32(dst, src);
4104  emit(0x0F);
4105  emit(0x57);
4106  emit_sse_operand(dst, src);
4107 }
4108 
4109 
4110 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4111  DCHECK(!IsEnabled(AVX));
4112  EnsureSpace ensure_space(this);
4113  emit(0xF2);
4114  emit_optional_rex_32(dst, src);
4115  emit(0x0F);
4116  emit(0x51);
4117  emit_sse_operand(dst, src);
4118 }
4119 
4120 void Assembler::sqrtsd(XMMRegister dst, Operand src) {
4121  DCHECK(!IsEnabled(AVX));
4122  EnsureSpace ensure_space(this);
4123  emit(0xF2);
4124  emit_optional_rex_32(dst, src);
4125  emit(0x0F);
4126  emit(0x51);
4127  emit_sse_operand(dst, src);
4128 }
4129 
4130 void Assembler::haddps(XMMRegister dst, XMMRegister src) {
4131  DCHECK(IsEnabled(SSE3));
4132  EnsureSpace ensure_space(this);
4133  emit(0xF2);
4134  emit_optional_rex_32(dst, src);
4135  emit(0x0F);
4136  emit(0x7C);
4137  emit_sse_operand(dst, src);
4138 }
4139 
4140 void Assembler::haddps(XMMRegister dst, Operand src) {
4141  DCHECK(IsEnabled(SSE3));
4142  EnsureSpace ensure_space(this);
4143  emit(0xF2);
4144  emit_optional_rex_32(dst, src);
4145  emit(0x0F);
4146  emit(0x7C);
4147  emit_sse_operand(dst, src);
4148 }
4149 
4150 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4151  DCHECK(!IsEnabled(AVX));
4152  EnsureSpace ensure_space(this);
4153  emit(0x66);
4154  emit_optional_rex_32(dst, src);
4155  emit(0x0F);
4156  emit(0x2E);
4157  emit_sse_operand(dst, src);
4158 }
4159 
4160 void Assembler::ucomisd(XMMRegister dst, Operand src) {
4161  DCHECK(!IsEnabled(AVX));
4162  EnsureSpace ensure_space(this);
4163  emit(0x66);
4164  emit_optional_rex_32(dst, src);
4165  emit(0x0F);
4166  emit(0x2E);
4167  emit_sse_operand(dst, src);
4168 }
4169 
4170 
4171 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
4172  EnsureSpace ensure_space(this);
4173  emit(0xF2);
4174  emit_optional_rex_32(dst, src);
4175  emit(0x0F);
4176  emit(0xC2);
4177  emit_sse_operand(dst, src);
4178  emit(0x01); // LT == 1
4179 }
4180 
4181 
4182 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
4183  DCHECK(!IsEnabled(AVX));
4184  DCHECK(IsEnabled(SSE4_1));
4185  EnsureSpace ensure_space(this);
4186  emit(0x66);
4187  emit_optional_rex_32(dst, src);
4188  emit(0x0F);
4189  emit(0x3A);
4190  emit(0x0A);
4191  emit_sse_operand(dst, src);
4192  // Mask precision exception.
4193  emit(static_cast<byte>(mode) | 0x8);
4194 }
4195 
4196 
4197 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
4198  DCHECK(!IsEnabled(AVX));
4199  DCHECK(IsEnabled(SSE4_1));
4200  EnsureSpace ensure_space(this);
4201  emit(0x66);
4202  emit_optional_rex_32(dst, src);
4203  emit(0x0F);
4204  emit(0x3A);
4205  emit(0x0B);
4206  emit_sse_operand(dst, src);
4207  // Mask precision exception.
4208  emit(static_cast<byte>(mode) | 0x8);
4209 }
4210 
4211 
4212 void Assembler::movmskpd(Register dst, XMMRegister src) {
4213  EnsureSpace ensure_space(this);
4214  emit(0x66);
4215  emit_optional_rex_32(dst, src);
4216  emit(0x0F);
4217  emit(0x50);
4218  emit_sse_operand(dst, src);
4219 }
4220 
4221 
4222 void Assembler::movmskps(Register dst, XMMRegister src) {
4223  EnsureSpace ensure_space(this);
4224  emit_optional_rex_32(dst, src);
4225  emit(0x0F);
4226  emit(0x50);
4227  emit_sse_operand(dst, src);
4228 }
4229 
4230 
4231 // AVX instructions
4232 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
4233  XMMRegister src2) {
4234  DCHECK(IsEnabled(FMA3));
4235  EnsureSpace ensure_space(this);
4236  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
4237  emit(op);
4238  emit_sse_operand(dst, src2);
4239 }
4240 
4241 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
4242  Operand src2) {
4243  DCHECK(IsEnabled(FMA3));
4244  EnsureSpace ensure_space(this);
4245  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
4246  emit(op);
4247  emit_sse_operand(dst, src2);
4248 }
4249 
4250 
4251 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
4252  XMMRegister src2) {
4253  DCHECK(IsEnabled(FMA3));
4254  EnsureSpace ensure_space(this);
4255  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
4256  emit(op);
4257  emit_sse_operand(dst, src2);
4258 }
4259 
4260 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
4261  Operand src2) {
4262  DCHECK(IsEnabled(FMA3));
4263  EnsureSpace ensure_space(this);
4264  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
4265  emit(op);
4266  emit_sse_operand(dst, src2);
4267 }
4268 
4269 
4270 void Assembler::vmovd(XMMRegister dst, Register src) {
4271  DCHECK(IsEnabled(AVX));
4272  EnsureSpace ensure_space(this);
4273  XMMRegister isrc = XMMRegister::from_code(src.code());
4274  emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
4275  emit(0x6E);
4276  emit_sse_operand(dst, src);
4277 }
4278 
4279 void Assembler::vmovd(XMMRegister dst, Operand src) {
4280  DCHECK(IsEnabled(AVX));
4281  EnsureSpace ensure_space(this);
4282  emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
4283  emit(0x6E);
4284  emit_sse_operand(dst, src);
4285 }
4286 
4287 
4288 void Assembler::vmovd(Register dst, XMMRegister src) {
4289  DCHECK(IsEnabled(AVX));
4290  EnsureSpace ensure_space(this);
4291  XMMRegister idst = XMMRegister::from_code(dst.code());
4292  emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
4293  emit(0x7E);
4294  emit_sse_operand(src, dst);
4295 }
4296 
4297 
4298 void Assembler::vmovq(XMMRegister dst, Register src) {
4299  DCHECK(IsEnabled(AVX));
4300  EnsureSpace ensure_space(this);
4301  XMMRegister isrc = XMMRegister::from_code(src.code());
4302  emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
4303  emit(0x6E);
4304  emit_sse_operand(dst, src);
4305 }
4306 
4307 void Assembler::vmovq(XMMRegister dst, Operand src) {
4308  DCHECK(IsEnabled(AVX));
4309  EnsureSpace ensure_space(this);
4310  emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
4311  emit(0x6E);
4312  emit_sse_operand(dst, src);
4313 }
4314 
4315 
4316 void Assembler::vmovq(Register dst, XMMRegister src) {
4317  DCHECK(IsEnabled(AVX));
4318  EnsureSpace ensure_space(this);
4319  XMMRegister idst = XMMRegister::from_code(dst.code());
4320  emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
4321  emit(0x7E);
4322  emit_sse_operand(src, dst);
4323 }
4324 
4325 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
4326  XMMRegister src2, SIMDPrefix pp, LeadingOpcode m,
4327  VexW w) {
4328  DCHECK(IsEnabled(AVX));
4329  EnsureSpace ensure_space(this);
4330  emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4331  emit(op);
4332  emit_sse_operand(dst, src2);
4333 }
4334 
4335 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1, Operand src2,
4336  SIMDPrefix pp, LeadingOpcode m, VexW w) {
4337  DCHECK(IsEnabled(AVX));
4338  EnsureSpace ensure_space(this);
4339  emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4340  emit(op);
4341  emit_sse_operand(dst, src2);
4342 }
4343 
4344 
4345 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
4346  XMMRegister src2) {
4347  DCHECK(IsEnabled(AVX));
4348  EnsureSpace ensure_space(this);
4349  emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4350  emit(op);
4351  emit_sse_operand(dst, src2);
4352 }
4353 
4354 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
4355  DCHECK(IsEnabled(AVX));
4356  EnsureSpace ensure_space(this);
4357  emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4358  emit(op);
4359  emit_sse_operand(dst, src2);
4360 }
4361 
4362 
4363 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
4364  XMMRegister src2) {
4365  DCHECK(IsEnabled(AVX));
4366  EnsureSpace ensure_space(this);
4367  emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4368  emit(op);
4369  emit_sse_operand(dst, src2);
4370 }
4371 
4372 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
4373  DCHECK(IsEnabled(AVX));
4374  EnsureSpace ensure_space(this);
4375  emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4376  emit(op);
4377  emit_sse_operand(dst, src2);
4378 }
4379 
4380 
4381 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
4382  DCHECK(IsEnabled(AVX));
4383  EnsureSpace ensure_space(this);
4384  emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4385  emit(0x2E);
4386  emit_sse_operand(dst, src);
4387 }
4388 
4389 void Assembler::vucomiss(XMMRegister dst, Operand src) {
4390  DCHECK(IsEnabled(AVX));
4391  EnsureSpace ensure_space(this);
4392  emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4393  emit(0x2E);
4394  emit_sse_operand(dst, src);
4395 }
4396 
4397 
4398 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
4399  XMMRegister src2) {
4400  DCHECK(IsEnabled(AVX));
4401  EnsureSpace ensure_space(this);
4402  emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4403  emit(op);
4404  emit_sse_operand(dst, src2);
4405 }
4406 
4407 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1, Operand src2) {
4408  DCHECK(IsEnabled(AVX));
4409  EnsureSpace ensure_space(this);
4410  emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4411  emit(op);
4412  emit_sse_operand(dst, src2);
4413 }
4414 
4415 
4416 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
4417  DCHECK(IsEnabled(BMI1));
4418  EnsureSpace ensure_space(this);
4419  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4420  emit(op);
4421  emit_modrm(reg, rm);
4422 }
4423 
4424 void Assembler::bmi1q(byte op, Register reg, Register vreg, Operand rm) {
4425  DCHECK(IsEnabled(BMI1));
4426  EnsureSpace ensure_space(this);
4427  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4428  emit(op);
4429  emit_operand(reg, rm);
4430 }
4431 
4432 
4433 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
4434  DCHECK(IsEnabled(BMI1));
4435  EnsureSpace ensure_space(this);
4436  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4437  emit(op);
4438  emit_modrm(reg, rm);
4439 }
4440 
4441 void Assembler::bmi1l(byte op, Register reg, Register vreg, Operand rm) {
4442  DCHECK(IsEnabled(BMI1));
4443  EnsureSpace ensure_space(this);
4444  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4445  emit(op);
4446  emit_operand(reg, rm);
4447 }
4448 
4449 
4450 void Assembler::tzcntq(Register dst, Register src) {
4451  DCHECK(IsEnabled(BMI1));
4452  EnsureSpace ensure_space(this);
4453  emit(0xF3);
4454  emit_rex_64(dst, src);
4455  emit(0x0F);
4456  emit(0xBC);
4457  emit_modrm(dst, src);
4458 }
4459 
4460 void Assembler::tzcntq(Register dst, Operand src) {
4461  DCHECK(IsEnabled(BMI1));
4462  EnsureSpace ensure_space(this);
4463  emit(0xF3);
4464  emit_rex_64(dst, src);
4465  emit(0x0F);
4466  emit(0xBC);
4467  emit_operand(dst, src);
4468 }
4469 
4470 
4471 void Assembler::tzcntl(Register dst, Register src) {
4472  DCHECK(IsEnabled(BMI1));
4473  EnsureSpace ensure_space(this);
4474  emit(0xF3);
4475  emit_optional_rex_32(dst, src);
4476  emit(0x0F);
4477  emit(0xBC);
4478  emit_modrm(dst, src);
4479 }
4480 
4481 void Assembler::tzcntl(Register dst, Operand src) {
4482  DCHECK(IsEnabled(BMI1));
4483  EnsureSpace ensure_space(this);
4484  emit(0xF3);
4485  emit_optional_rex_32(dst, src);
4486  emit(0x0F);
4487  emit(0xBC);
4488  emit_operand(dst, src);
4489 }
4490 
4491 
4492 void Assembler::lzcntq(Register dst, Register src) {
4493  DCHECK(IsEnabled(LZCNT));
4494  EnsureSpace ensure_space(this);
4495  emit(0xF3);
4496  emit_rex_64(dst, src);
4497  emit(0x0F);
4498  emit(0xBD);
4499  emit_modrm(dst, src);
4500 }
4501 
4502 void Assembler::lzcntq(Register dst, Operand src) {
4503  DCHECK(IsEnabled(LZCNT));
4504  EnsureSpace ensure_space(this);
4505  emit(0xF3);
4506  emit_rex_64(dst, src);
4507  emit(0x0F);
4508  emit(0xBD);
4509  emit_operand(dst, src);
4510 }
4511 
4512 
4513 void Assembler::lzcntl(Register dst, Register src) {
4514  DCHECK(IsEnabled(LZCNT));
4515  EnsureSpace ensure_space(this);
4516  emit(0xF3);
4517  emit_optional_rex_32(dst, src);
4518  emit(0x0F);
4519  emit(0xBD);
4520  emit_modrm(dst, src);
4521 }
4522 
4523 void Assembler::lzcntl(Register dst, Operand src) {
4524  DCHECK(IsEnabled(LZCNT));
4525  EnsureSpace ensure_space(this);
4526  emit(0xF3);
4527  emit_optional_rex_32(dst, src);
4528  emit(0x0F);
4529  emit(0xBD);
4530  emit_operand(dst, src);
4531 }
4532 
4533 
4534 void Assembler::popcntq(Register dst, Register src) {
4535  DCHECK(IsEnabled(POPCNT));
4536  EnsureSpace ensure_space(this);
4537  emit(0xF3);
4538  emit_rex_64(dst, src);
4539  emit(0x0F);
4540  emit(0xB8);
4541  emit_modrm(dst, src);
4542 }
4543 
4544 void Assembler::popcntq(Register dst, Operand src) {
4545  DCHECK(IsEnabled(POPCNT));
4546  EnsureSpace ensure_space(this);
4547  emit(0xF3);
4548  emit_rex_64(dst, src);
4549  emit(0x0F);
4550  emit(0xB8);
4551  emit_operand(dst, src);
4552 }
4553 
4554 
4555 void Assembler::popcntl(Register dst, Register src) {
4556  DCHECK(IsEnabled(POPCNT));
4557  EnsureSpace ensure_space(this);
4558  emit(0xF3);
4559  emit_optional_rex_32(dst, src);
4560  emit(0x0F);
4561  emit(0xB8);
4562  emit_modrm(dst, src);
4563 }
4564 
4565 void Assembler::popcntl(Register dst, Operand src) {
4566  DCHECK(IsEnabled(POPCNT));
4567  EnsureSpace ensure_space(this);
4568  emit(0xF3);
4569  emit_optional_rex_32(dst, src);
4570  emit(0x0F);
4571  emit(0xB8);
4572  emit_operand(dst, src);
4573 }
4574 
4575 
4576 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4577  Register rm) {
4578  DCHECK(IsEnabled(BMI2));
4579  EnsureSpace ensure_space(this);
4580  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4581  emit(op);
4582  emit_modrm(reg, rm);
4583 }
4584 
4585 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4586  Operand rm) {
4587  DCHECK(IsEnabled(BMI2));
4588  EnsureSpace ensure_space(this);
4589  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4590  emit(op);
4591  emit_operand(reg, rm);
4592 }
4593 
4594 
4595 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4596  Register rm) {
4597  DCHECK(IsEnabled(BMI2));
4598  EnsureSpace ensure_space(this);
4599  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4600  emit(op);
4601  emit_modrm(reg, rm);
4602 }
4603 
4604 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4605  Operand rm) {
4606  DCHECK(IsEnabled(BMI2));
4607  EnsureSpace ensure_space(this);
4608  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4609  emit(op);
4610  emit_operand(reg, rm);
4611 }
4612 
4613 
4614 void Assembler::rorxq(Register dst, Register src, byte imm8) {
4615  DCHECK(IsEnabled(BMI2));
4616  DCHECK(is_uint8(imm8));
4617  Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4618  EnsureSpace ensure_space(this);
4619  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4620  emit(0xF0);
4621  emit_modrm(dst, src);
4622  emit(imm8);
4623 }
4624 
4625 void Assembler::rorxq(Register dst, Operand src, byte imm8) {
4626  DCHECK(IsEnabled(BMI2));
4627  DCHECK(is_uint8(imm8));
4628  Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4629  EnsureSpace ensure_space(this);
4630  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4631  emit(0xF0);
4632  emit_operand(dst, src);
4633  emit(imm8);
4634 }
4635 
4636 
4637 void Assembler::rorxl(Register dst, Register src, byte imm8) {
4638  DCHECK(IsEnabled(BMI2));
4639  DCHECK(is_uint8(imm8));
4640  Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4641  EnsureSpace ensure_space(this);
4642  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4643  emit(0xF0);
4644  emit_modrm(dst, src);
4645  emit(imm8);
4646 }
4647 
4648 void Assembler::rorxl(Register dst, Operand src, byte imm8) {
4649  DCHECK(IsEnabled(BMI2));
4650  DCHECK(is_uint8(imm8));
4651  Register vreg = Register::from_code<0>(); // VEX.vvvv unused
4652  EnsureSpace ensure_space(this);
4653  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4654  emit(0xF0);
4655  emit_operand(dst, src);
4656  emit(imm8);
4657 }
4658 
4659 void Assembler::pause() {
4660  emit(0xF3);
4661  emit(0x90);
4662 }
4663 
4664 void Assembler::minps(XMMRegister dst, XMMRegister src) {
4665  EnsureSpace ensure_space(this);
4666  emit_optional_rex_32(dst, src);
4667  emit(0x0F);
4668  emit(0x5D);
4669  emit_sse_operand(dst, src);
4670 }
4671 
4672 void Assembler::minps(XMMRegister dst, Operand src) {
4673  EnsureSpace ensure_space(this);
4674  emit_optional_rex_32(dst, src);
4675  emit(0x0F);
4676  emit(0x5D);
4677  emit_sse_operand(dst, src);
4678 }
4679 
4680 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
4681  EnsureSpace ensure_space(this);
4682  emit_optional_rex_32(dst, src);
4683  emit(0x0F);
4684  emit(0x5F);
4685  emit_sse_operand(dst, src);
4686 }
4687 
4688 void Assembler::maxps(XMMRegister dst, Operand src) {
4689  EnsureSpace ensure_space(this);
4690  emit_optional_rex_32(dst, src);
4691  emit(0x0F);
4692  emit(0x5F);
4693  emit_sse_operand(dst, src);
4694 }
4695 
4696 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4697  EnsureSpace ensure_space(this);
4698  emit_optional_rex_32(dst, src);
4699  emit(0x0F);
4700  emit(0x53);
4701  emit_sse_operand(dst, src);
4702 }
4703 
4704 void Assembler::rcpps(XMMRegister dst, Operand src) {
4705  EnsureSpace ensure_space(this);
4706  emit_optional_rex_32(dst, src);
4707  emit(0x0F);
4708  emit(0x53);
4709  emit_sse_operand(dst, src);
4710 }
4711 
4712 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) {
4713  EnsureSpace ensure_space(this);
4714  emit_optional_rex_32(dst, src);
4715  emit(0x0F);
4716  emit(0x52);
4717  emit_sse_operand(dst, src);
4718 }
4719 
4720 void Assembler::rsqrtps(XMMRegister dst, Operand src) {
4721  EnsureSpace ensure_space(this);
4722  emit_optional_rex_32(dst, src);
4723  emit(0x0F);
4724  emit(0x52);
4725  emit_sse_operand(dst, src);
4726 }
4727 
4728 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) {
4729  EnsureSpace ensure_space(this);
4730  emit_optional_rex_32(dst, src);
4731  emit(0x0F);
4732  emit(0x51);
4733  emit_sse_operand(dst, src);
4734 }
4735 
4736 void Assembler::sqrtps(XMMRegister dst, Operand src) {
4737  EnsureSpace ensure_space(this);
4738  emit_optional_rex_32(dst, src);
4739  emit(0x0F);
4740  emit(0x51);
4741  emit_sse_operand(dst, src);
4742 }
4743 
4744 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
4745  EnsureSpace ensure_space(this);
4746  emit_optional_rex_32(dst, src);
4747  emit(0x0F);
4748  emit(0x5B);
4749  emit_sse_operand(dst, src);
4750 }
4751 
4752 void Assembler::cvtdq2ps(XMMRegister dst, Operand src) {
4753  EnsureSpace ensure_space(this);
4754  emit_optional_rex_32(dst, src);
4755  emit(0x0F);
4756  emit(0x5B);
4757  emit_sse_operand(dst, src);
4758 }
4759 
4760 void Assembler::movups(XMMRegister dst, XMMRegister src) {
4761  EnsureSpace ensure_space(this);
4762  if (src.low_bits() == 4) {
4763  // Try to avoid an unnecessary SIB byte.
4764  emit_optional_rex_32(src, dst);
4765  emit(0x0F);
4766  emit(0x11);
4767  emit_sse_operand(src, dst);
4768  } else {
4769  emit_optional_rex_32(dst, src);
4770  emit(0x0F);
4771  emit(0x10);
4772  emit_sse_operand(dst, src);
4773  }
4774 }
4775 
4776 void Assembler::movups(XMMRegister dst, Operand src) {
4777  EnsureSpace ensure_space(this);
4778  emit_optional_rex_32(dst, src);
4779  emit(0x0F);
4780  emit(0x10);
4781  emit_sse_operand(dst, src);
4782 }
4783 
4784 void Assembler::movups(Operand dst, XMMRegister src) {
4785  EnsureSpace ensure_space(this);
4786  emit_optional_rex_32(src, dst);
4787  emit(0x0F);
4788  emit(0x11);
4789  emit_sse_operand(src, dst);
4790 }
4791 
4792 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4793  byte escape, byte opcode) {
4794  EnsureSpace ensure_space(this);
4795  emit(prefix);
4796  emit_optional_rex_32(dst, src);
4797  emit(escape);
4798  emit(opcode);
4799  emit_sse_operand(dst, src);
4800 }
4801 
4802 void Assembler::sse2_instr(XMMRegister dst, Operand src, byte prefix,
4803  byte escape, byte opcode) {
4804  EnsureSpace ensure_space(this);
4805  emit(prefix);
4806  emit_optional_rex_32(dst, src);
4807  emit(escape);
4808  emit(opcode);
4809  emit_sse_operand(dst, src);
4810 }
4811 
4812 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
4813  byte escape1, byte escape2, byte opcode) {
4814  DCHECK(IsEnabled(SSSE3));
4815  EnsureSpace ensure_space(this);
4816  emit(prefix);
4817  emit_optional_rex_32(dst, src);
4818  emit(escape1);
4819  emit(escape2);
4820  emit(opcode);
4821  emit_sse_operand(dst, src);
4822 }
4823 
4824 void Assembler::ssse3_instr(XMMRegister dst, Operand src, byte prefix,
4825  byte escape1, byte escape2, byte opcode) {
4826  DCHECK(IsEnabled(SSSE3));
4827  EnsureSpace ensure_space(this);
4828  emit(prefix);
4829  emit_optional_rex_32(dst, src);
4830  emit(escape1);
4831  emit(escape2);
4832  emit(opcode);
4833  emit_sse_operand(dst, src);
4834 }
4835 
4836 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
4837  byte escape1, byte escape2, byte opcode) {
4838  DCHECK(IsEnabled(SSE4_1));
4839  EnsureSpace ensure_space(this);
4840  emit(prefix);
4841  emit_optional_rex_32(dst, src);
4842  emit(escape1);
4843  emit(escape2);
4844  emit(opcode);
4845  emit_sse_operand(dst, src);
4846 }
4847 
4848 void Assembler::sse4_instr(XMMRegister dst, Operand src, byte prefix,
4849  byte escape1, byte escape2, byte opcode) {
4850  DCHECK(IsEnabled(SSE4_1));
4851  EnsureSpace ensure_space(this);
4852  emit(prefix);
4853  emit_optional_rex_32(dst, src);
4854  emit(escape1);
4855  emit(escape2);
4856  emit(opcode);
4857  emit_sse_operand(dst, src);
4858 }
4859 
4860 void Assembler::lddqu(XMMRegister dst, Operand src) {
4861  DCHECK(IsEnabled(SSE3));
4862  EnsureSpace ensure_space(this);
4863  emit(0xF2);
4864  emit_optional_rex_32(dst, src);
4865  emit(0x0F);
4866  emit(0xF0);
4867  emit_sse_operand(dst, src);
4868 }
4869 
4870 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4871  EnsureSpace ensure_space(this);
4872  emit(0x66);
4873  emit_optional_rex_32(dst);
4874  emit(0x0F);
4875  emit(0x73);
4876  emit_sse_operand(dst);
4877  emit(shift);
4878 }
4879 
4880 void Assembler::pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4881  EnsureSpace ensure_space(this);
4882  emit(0xF3);
4883  emit_optional_rex_32(dst, src);
4884  emit(0x0F);
4885  emit(0x70);
4886  emit_sse_operand(dst, src);
4887  emit(shuffle);
4888 }
4889 
4890 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4891  EnsureSpace ensure_space(this);
4892  emit(0xF2);
4893  emit_optional_rex_32(dst, src);
4894  emit(0x0F);
4895  emit(0x70);
4896  emit_sse_operand(dst, src);
4897  emit(shuffle);
4898 }
4899 
4900 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4901  EnsureSpace ensure_space(this);
4902  emit(0x66);
4903  emit_optional_rex_32(dst, src);
4904  emit(0x0F);
4905  emit(0x70);
4906  emit_sse_operand(dst, src);
4907  emit(shuffle);
4908 }
4909 
4910 void Assembler::pshufd(XMMRegister dst, Operand src, uint8_t shuffle) {
4911  EnsureSpace ensure_space(this);
4912  emit(0x66);
4913  emit_optional_rex_32(dst, src);
4914  emit(0x0F);
4915  emit(0x70);
4916  emit_sse_operand(dst, src);
4917  emit(shuffle);
4918 }
4919 
4920 void Assembler::emit_sse_operand(XMMRegister reg, Operand adr) {
4921  Register ireg = Register::from_code(reg.code());
4922  emit_operand(ireg, adr);
4923 }
4924 
4925 void Assembler::emit_sse_operand(Register reg, Operand adr) {
4926  emit_operand(reg, adr);
4927 }
4928 
4929 
4930 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4931  emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4932 }
4933 
4934 
4935 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4936  emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4937 }
4938 
4939 
4940 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4941  emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4942 }
4943 
4944 void Assembler::emit_sse_operand(XMMRegister dst) {
4945  emit(0xD8 | dst.low_bits());
4946 }
4947 
4948 void Assembler::db(uint8_t data) {
4949  EnsureSpace ensure_space(this);
4950  emit(data);
4951 }
4952 
4953 
4954 void Assembler::dd(uint32_t data) {
4955  EnsureSpace ensure_space(this);
4956  emitl(data);
4957 }
4958 
4959 
4960 void Assembler::dq(uint64_t data) {
4961  EnsureSpace ensure_space(this);
4962  emitq(data);
4963 }
4964 
4965 
4966 void Assembler::dq(Label* label) {
4967  EnsureSpace ensure_space(this);
4968  if (label->is_bound()) {
4969  internal_reference_positions_.push_back(pc_offset());
4970  emitp(reinterpret_cast<Address>(buffer_) + label->pos(),
4971  RelocInfo::INTERNAL_REFERENCE);
4972  } else {
4973  RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4974  emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4975  if (label->is_linked()) {
4976  emitl(label->pos());
4977  label->link_to(pc_offset() - sizeof(int32_t));
4978  } else {
4979  DCHECK(label->is_unused());
4980  int32_t current = pc_offset();
4981  emitl(current);
4982  label->link_to(current);
4983  }
4984  }
4985 }
4986 
4987 
4988 // Relocation information implementations.
4989 
4990 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4991  if (!ShouldRecordRelocInfo(rmode)) return;
4992  RelocInfo rinfo(reinterpret_cast<Address>(pc_), rmode, data, Code());
4993  reloc_info_writer.Write(&rinfo);
4994 }
4995 
4996 const int RelocInfo::kApplyMask =
4997  RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
4998  RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
4999  RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) |
5000  RelocInfo::ModeMask(RelocInfo::WASM_CALL);
5001 
5002 bool RelocInfo::IsCodedSpecially() {
5003  // The deserializer needs to know whether a pointer is specially coded. Being
5004  // specially coded on x64 means that it is a relative 32 bit address, as used
5005  // by branch instructions.
5006  return (1 << rmode_) & kApplyMask;
5007 }
5008 
5009 
5010 bool RelocInfo::IsInConstantPool() {
5011  return false;
5012 }
5013 
5014 int RelocInfo::GetDeoptimizationId(Isolate* isolate, DeoptimizeKind kind) {
5015  DCHECK(IsRuntimeEntry(rmode_));
5016  return Deoptimizer::GetDeoptimizationId(isolate, target_address(), kind);
5017 }
5018 
5019 } // namespace internal
5020 } // namespace v8
5021 
5022 #endif // V8_TARGET_ARCH_X64
int64_t
v8
Definition: libplatform.h:13
uint32_t