V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
assembler-arm64.h
1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef V8_ARM64_ASSEMBLER_ARM64_H_
6 #define V8_ARM64_ASSEMBLER_ARM64_H_
7 
8 #include <deque>
9 #include <list>
10 #include <map>
11 #include <vector>
12 
13 #include "src/arm64/constants-arm64.h"
14 #include "src/arm64/instructions-arm64.h"
15 #include "src/assembler.h"
16 #include "src/base/optional.h"
17 #include "src/constant-pool.h"
18 #include "src/globals.h"
19 #include "src/utils.h"
20 
21 // Windows arm64 SDK defines mvn to NEON intrinsic neon_not which will not
22 // be used here.
23 #if defined(V8_OS_WIN) && defined(mvn)
24 #undef mvn
25 #endif
26 
27 namespace v8 {
28 namespace internal {
29 
30 // -----------------------------------------------------------------------------
31 // Registers.
32 // clang-format off
33 #define GENERAL_REGISTER_CODE_LIST(R) \
34  R(0) R(1) R(2) R(3) R(4) R(5) R(6) R(7) \
35  R(8) R(9) R(10) R(11) R(12) R(13) R(14) R(15) \
36  R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23) \
37  R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
38 
39 #define GENERAL_REGISTERS(R) \
40  R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
41  R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
42  R(x16) R(x17) R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) \
43  R(x24) R(x25) R(x26) R(x27) R(x28) R(x29) R(x30) R(x31)
44 
45 #if defined(V8_OS_WIN)
46 // x18 is reserved as platform register on Windows ARM64.
47 #define ALLOCATABLE_GENERAL_REGISTERS(R) \
48  R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
49  R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
50  R(x19) R(x20) R(x21) R(x22) R(x23) R(x24) R(x25) \
51  R(x27) R(x28)
52 #else
53 #define ALLOCATABLE_GENERAL_REGISTERS(R) \
54  R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \
55  R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \
56  R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) R(x24) R(x25) \
57  R(x27) R(x28)
58 #endif
59 
60 #define FLOAT_REGISTERS(V) \
61  V(s0) V(s1) V(s2) V(s3) V(s4) V(s5) V(s6) V(s7) \
62  V(s8) V(s9) V(s10) V(s11) V(s12) V(s13) V(s14) V(s15) \
63  V(s16) V(s17) V(s18) V(s19) V(s20) V(s21) V(s22) V(s23) \
64  V(s24) V(s25) V(s26) V(s27) V(s28) V(s29) V(s30) V(s31)
65 
66 #define DOUBLE_REGISTERS(R) \
67  R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \
68  R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d15) \
69  R(d16) R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) \
70  R(d24) R(d25) R(d26) R(d27) R(d28) R(d29) R(d30) R(d31)
71 
72 #define SIMD128_REGISTERS(V) \
73  V(q0) V(q1) V(q2) V(q3) V(q4) V(q5) V(q6) V(q7) \
74  V(q8) V(q9) V(q10) V(q11) V(q12) V(q13) V(q14) V(q15) \
75  V(q16) V(q17) V(q18) V(q19) V(q20) V(q21) V(q22) V(q23) \
76  V(q24) V(q25) V(q26) V(q27) V(q28) V(q29) V(q30) V(q31)
77 
78 // Register d29 could be allocated, but we keep an even length list here, in
79 // order to make stack alignment easier for save and restore.
80 #define ALLOCATABLE_DOUBLE_REGISTERS(R) \
81  R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \
82  R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d16) \
83  R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) R(d24) \
84  R(d25) R(d26) R(d27) R(d28)
85 // clang-format on
86 
87 constexpr int kRegListSizeInBits = sizeof(RegList) * kBitsPerByte;
88 
89 const int kNumRegs = kNumberOfRegisters;
90 // Registers x0-x17 are caller-saved.
91 const int kNumJSCallerSaved = 18;
92 const RegList kJSCallerSaved = 0x3ffff;
93 
94 // Number of registers for which space is reserved in safepoints. Must be a
95 // multiple of eight.
96 // TODO(all): Refine this number.
97 const int kNumSafepointRegisters = 32;
98 
99 // Define the list of registers actually saved at safepoints.
100 // Note that the number of saved registers may be smaller than the reserved
101 // space, i.e. kNumSafepointSavedRegisters <= kNumSafepointRegisters.
102 #define kSafepointSavedRegisters CPURegList::GetSafepointSavedRegisters().list()
103 #define kNumSafepointSavedRegisters \
104  CPURegList::GetSafepointSavedRegisters().Count()
105 
106 // Some CPURegister methods can return Register and VRegister types, so we
107 // need to declare them in advance.
108 class Register;
109 class VRegister;
110 
111 enum RegisterCode {
112 #define REGISTER_CODE(R) kRegCode_##R,
113  GENERAL_REGISTERS(REGISTER_CODE)
114 #undef REGISTER_CODE
115  kRegAfterLast
116 };
117 
118 class CPURegister : public RegisterBase<CPURegister, kRegAfterLast> {
119  public:
120  enum RegisterType {
121  kRegister,
122  kVRegister,
123  kNoRegister
124  };
125 
126  static constexpr CPURegister no_reg() {
127  return CPURegister{0, 0, kNoRegister};
128  }
129 
130  template <int code, int size, RegisterType type>
131  static constexpr CPURegister Create() {
132  static_assert(IsValid(code, size, type), "Cannot create invalid registers");
133  return CPURegister{code, size, type};
134  }
135 
136  static CPURegister Create(int code, int size, RegisterType type) {
137  DCHECK(IsValid(code, size, type));
138  return CPURegister{code, size, type};
139  }
140 
141  RegisterType type() const { return reg_type_; }
142  int SizeInBits() const {
143  DCHECK(IsValid());
144  return reg_size_;
145  }
146  int SizeInBytes() const {
147  DCHECK(IsValid());
148  DCHECK_EQ(SizeInBits() % 8, 0);
149  return reg_size_ / 8;
150  }
151  bool Is8Bits() const {
152  DCHECK(IsValid());
153  return reg_size_ == 8;
154  }
155  bool Is16Bits() const {
156  DCHECK(IsValid());
157  return reg_size_ == 16;
158  }
159  bool Is32Bits() const {
160  DCHECK(IsValid());
161  return reg_size_ == 32;
162  }
163  bool Is64Bits() const {
164  DCHECK(IsValid());
165  return reg_size_ == 64;
166  }
167  bool Is128Bits() const {
168  DCHECK(IsValid());
169  return reg_size_ == 128;
170  }
171  bool IsValid() const { return reg_type_ != kNoRegister; }
172  bool IsNone() const { return reg_type_ == kNoRegister; }
173  bool Is(const CPURegister& other) const {
174  return Aliases(other) && (reg_size_ == other.reg_size_);
175  }
176  bool Aliases(const CPURegister& other) const {
177  return (reg_code_ == other.reg_code_) && (reg_type_ == other.reg_type_);
178  }
179 
180  bool IsZero() const;
181  bool IsSP() const;
182 
183  bool IsRegister() const { return reg_type_ == kRegister; }
184  bool IsVRegister() const { return reg_type_ == kVRegister; }
185 
186  bool IsFPRegister() const { return IsS() || IsD(); }
187 
188  bool IsW() const { return IsRegister() && Is32Bits(); }
189  bool IsX() const { return IsRegister() && Is64Bits(); }
190 
191  // These assertions ensure that the size and type of the register are as
192  // described. They do not consider the number of lanes that make up a vector.
193  // So, for example, Is8B() implies IsD(), and Is1D() implies IsD, but IsD()
194  // does not imply Is1D() or Is8B().
195  // Check the number of lanes, ie. the format of the vector, using methods such
196  // as Is8B(), Is1D(), etc. in the VRegister class.
197  bool IsV() const { return IsVRegister(); }
198  bool IsB() const { return IsV() && Is8Bits(); }
199  bool IsH() const { return IsV() && Is16Bits(); }
200  bool IsS() const { return IsV() && Is32Bits(); }
201  bool IsD() const { return IsV() && Is64Bits(); }
202  bool IsQ() const { return IsV() && Is128Bits(); }
203 
204  Register Reg() const;
205  VRegister VReg() const;
206 
207  Register X() const;
208  Register W() const;
209  VRegister V() const;
210  VRegister B() const;
211  VRegister H() const;
212  VRegister D() const;
213  VRegister S() const;
214  VRegister Q() const;
215 
216  bool IsSameSizeAndType(const CPURegister& other) const;
217 
218  bool is(const CPURegister& other) const { return Is(other); }
219  bool is_valid() const { return IsValid(); }
220 
221  protected:
222  int reg_size_;
223  RegisterType reg_type_;
224 
225 #if defined(V8_OS_WIN) && !defined(__clang__)
226  // MSVC has problem to parse template base class as friend class.
227  friend RegisterBase;
228 #else
229  friend class RegisterBase;
230 #endif
231 
232  constexpr CPURegister(int code, int size, RegisterType type)
233  : RegisterBase(code), reg_size_(size), reg_type_(type) {}
234 
235  static constexpr bool IsValidRegister(int code, int size) {
236  return (size == kWRegSizeInBits || size == kXRegSizeInBits) &&
237  (code < kNumberOfRegisters || code == kSPRegInternalCode);
238  }
239 
240  static constexpr bool IsValidVRegister(int code, int size) {
241  return (size == kBRegSizeInBits || size == kHRegSizeInBits ||
242  size == kSRegSizeInBits || size == kDRegSizeInBits ||
243  size == kQRegSizeInBits) &&
244  code < kNumberOfVRegisters;
245  }
246 
247  static constexpr bool IsValid(int code, int size, RegisterType type) {
248  return (type == kRegister && IsValidRegister(code, size)) ||
249  (type == kVRegister && IsValidVRegister(code, size));
250  }
251 
252  static constexpr bool IsNone(int code, int size, RegisterType type) {
253  return type == kNoRegister && code == 0 && size == 0;
254  }
255 };
256 
257 ASSERT_TRIVIALLY_COPYABLE(CPURegister);
258 
259 class Register : public CPURegister {
260  public:
261  static constexpr Register no_reg() { return Register(CPURegister::no_reg()); }
262 
263  template <int code, int size>
264  static constexpr Register Create() {
265  return Register(CPURegister::Create<code, size, CPURegister::kRegister>());
266  }
267 
268  static Register Create(int code, int size) {
269  return Register(CPURegister::Create(code, size, CPURegister::kRegister));
270  }
271 
272  static Register XRegFromCode(unsigned code);
273  static Register WRegFromCode(unsigned code);
274 
275  static Register from_code(int code) {
276  // Always return an X register.
277  return Register::Create(code, kXRegSizeInBits);
278  }
279 
280  template <int code>
281  static Register from_code() {
282  // Always return an X register.
283  return Register::Create<code, kXRegSizeInBits>();
284  }
285 
286  private:
287  constexpr explicit Register(const CPURegister& r) : CPURegister(r) {}
288 };
289 
290 ASSERT_TRIVIALLY_COPYABLE(Register);
291 
292 constexpr bool kPadArguments = true;
293 constexpr bool kSimpleFPAliasing = true;
294 constexpr bool kSimdMaskRegisters = false;
295 
296 enum DoubleRegisterCode {
297 #define REGISTER_CODE(R) kDoubleCode_##R,
298  DOUBLE_REGISTERS(REGISTER_CODE)
299 #undef REGISTER_CODE
300  kDoubleAfterLast
301 };
302 
303 class VRegister : public CPURegister {
304  public:
305  static constexpr VRegister no_reg() {
306  return VRegister(CPURegister::no_reg(), 0);
307  }
308 
309  template <int code, int size, int lane_count = 1>
310  static constexpr VRegister Create() {
311  static_assert(IsValidLaneCount(lane_count), "Invalid lane count");
312  return VRegister(CPURegister::Create<code, size, kVRegister>(), lane_count);
313  }
314 
315  static VRegister Create(int code, int size, int lane_count = 1) {
316  DCHECK(IsValidLaneCount(lane_count));
317  return VRegister(CPURegister::Create(code, size, CPURegister::kVRegister),
318  lane_count);
319  }
320 
321  static VRegister Create(int reg_code, VectorFormat format) {
322  int reg_size = RegisterSizeInBitsFromFormat(format);
323  int reg_count = IsVectorFormat(format) ? LaneCountFromFormat(format) : 1;
324  return VRegister::Create(reg_code, reg_size, reg_count);
325  }
326 
327  static VRegister BRegFromCode(unsigned code);
328  static VRegister HRegFromCode(unsigned code);
329  static VRegister SRegFromCode(unsigned code);
330  static VRegister DRegFromCode(unsigned code);
331  static VRegister QRegFromCode(unsigned code);
332  static VRegister VRegFromCode(unsigned code);
333 
334  VRegister V8B() const {
335  return VRegister::Create(code(), kDRegSizeInBits, 8);
336  }
337  VRegister V16B() const {
338  return VRegister::Create(code(), kQRegSizeInBits, 16);
339  }
340  VRegister V4H() const {
341  return VRegister::Create(code(), kDRegSizeInBits, 4);
342  }
343  VRegister V8H() const {
344  return VRegister::Create(code(), kQRegSizeInBits, 8);
345  }
346  VRegister V2S() const {
347  return VRegister::Create(code(), kDRegSizeInBits, 2);
348  }
349  VRegister V4S() const {
350  return VRegister::Create(code(), kQRegSizeInBits, 4);
351  }
352  VRegister V2D() const {
353  return VRegister::Create(code(), kQRegSizeInBits, 2);
354  }
355  VRegister V1D() const {
356  return VRegister::Create(code(), kDRegSizeInBits, 1);
357  }
358 
359  bool Is8B() const { return (Is64Bits() && (lane_count_ == 8)); }
360  bool Is16B() const { return (Is128Bits() && (lane_count_ == 16)); }
361  bool Is4H() const { return (Is64Bits() && (lane_count_ == 4)); }
362  bool Is8H() const { return (Is128Bits() && (lane_count_ == 8)); }
363  bool Is2S() const { return (Is64Bits() && (lane_count_ == 2)); }
364  bool Is4S() const { return (Is128Bits() && (lane_count_ == 4)); }
365  bool Is1D() const { return (Is64Bits() && (lane_count_ == 1)); }
366  bool Is2D() const { return (Is128Bits() && (lane_count_ == 2)); }
367 
368  // For consistency, we assert the number of lanes of these scalar registers,
369  // even though there are no vectors of equivalent total size with which they
370  // could alias.
371  bool Is1B() const {
372  DCHECK(!(Is8Bits() && IsVector()));
373  return Is8Bits();
374  }
375  bool Is1H() const {
376  DCHECK(!(Is16Bits() && IsVector()));
377  return Is16Bits();
378  }
379  bool Is1S() const {
380  DCHECK(!(Is32Bits() && IsVector()));
381  return Is32Bits();
382  }
383 
384  bool IsLaneSizeB() const { return LaneSizeInBits() == kBRegSizeInBits; }
385  bool IsLaneSizeH() const { return LaneSizeInBits() == kHRegSizeInBits; }
386  bool IsLaneSizeS() const { return LaneSizeInBits() == kSRegSizeInBits; }
387  bool IsLaneSizeD() const { return LaneSizeInBits() == kDRegSizeInBits; }
388 
389  bool IsScalar() const { return lane_count_ == 1; }
390  bool IsVector() const { return lane_count_ > 1; }
391 
392  bool IsSameFormat(const VRegister& other) const {
393  return (reg_size_ == other.reg_size_) && (lane_count_ == other.lane_count_);
394  }
395 
396  int LaneCount() const { return lane_count_; }
397 
398  unsigned LaneSizeInBytes() const { return SizeInBytes() / lane_count_; }
399 
400  unsigned LaneSizeInBits() const { return LaneSizeInBytes() * 8; }
401 
402  static constexpr int kMaxNumRegisters = kNumberOfVRegisters;
403  STATIC_ASSERT(kMaxNumRegisters == kDoubleAfterLast);
404 
405  static VRegister from_code(int code) {
406  // Always return a D register.
407  return VRegister::Create(code, kDRegSizeInBits);
408  }
409 
410  private:
411  int lane_count_;
412 
413  constexpr explicit VRegister(const CPURegister& r, int lane_count)
414  : CPURegister(r), lane_count_(lane_count) {}
415 
416  static constexpr bool IsValidLaneCount(int lane_count) {
417  return base::bits::IsPowerOfTwo(lane_count) && lane_count <= 16;
418  }
419 };
420 
421 ASSERT_TRIVIALLY_COPYABLE(VRegister);
422 
423 // No*Reg is used to indicate an unused argument, or an error case. Note that
424 // these all compare equal (using the Is() method). The Register and VRegister
425 // variants are provided for convenience.
426 constexpr Register NoReg = Register::no_reg();
427 constexpr VRegister NoVReg = VRegister::no_reg();
428 constexpr CPURegister NoCPUReg = CPURegister::no_reg();
429 constexpr Register no_reg = NoReg;
430 constexpr VRegister no_dreg = NoVReg;
431 
432 #define DEFINE_REGISTER(register_class, name, ...) \
433  constexpr register_class name = register_class::Create<__VA_ARGS__>()
434 #define ALIAS_REGISTER(register_class, alias, name) \
435  constexpr register_class alias = name
436 
437 #define DEFINE_REGISTERS(N) \
438  DEFINE_REGISTER(Register, w##N, N, kWRegSizeInBits); \
439  DEFINE_REGISTER(Register, x##N, N, kXRegSizeInBits);
440 GENERAL_REGISTER_CODE_LIST(DEFINE_REGISTERS)
441 #undef DEFINE_REGISTERS
442 
443 DEFINE_REGISTER(Register, wsp, kSPRegInternalCode, kWRegSizeInBits);
444 DEFINE_REGISTER(Register, sp, kSPRegInternalCode, kXRegSizeInBits);
445 
446 #define DEFINE_VREGISTERS(N) \
447  DEFINE_REGISTER(VRegister, b##N, N, kBRegSizeInBits); \
448  DEFINE_REGISTER(VRegister, h##N, N, kHRegSizeInBits); \
449  DEFINE_REGISTER(VRegister, s##N, N, kSRegSizeInBits); \
450  DEFINE_REGISTER(VRegister, d##N, N, kDRegSizeInBits); \
451  DEFINE_REGISTER(VRegister, q##N, N, kQRegSizeInBits); \
452  DEFINE_REGISTER(VRegister, v##N, N, kQRegSizeInBits);
453 GENERAL_REGISTER_CODE_LIST(DEFINE_VREGISTERS)
454 #undef DEFINE_VREGISTERS
455 
456 #undef DEFINE_REGISTER
457 
458 // Registers aliases.
459 ALIAS_REGISTER(VRegister, v8_, v8); // Avoid conflicts with namespace v8.
460 ALIAS_REGISTER(Register, ip0, x16);
461 ALIAS_REGISTER(Register, ip1, x17);
462 ALIAS_REGISTER(Register, wip0, w16);
463 ALIAS_REGISTER(Register, wip1, w17);
464 // Root register.
465 ALIAS_REGISTER(Register, kRootRegister, x26);
466 ALIAS_REGISTER(Register, rr, x26);
467 // Context pointer register.
468 ALIAS_REGISTER(Register, cp, x27);
469 ALIAS_REGISTER(Register, fp, x29);
470 ALIAS_REGISTER(Register, lr, x30);
471 ALIAS_REGISTER(Register, xzr, x31);
472 ALIAS_REGISTER(Register, wzr, w31);
473 
474 // Register used for padding stack slots.
475 ALIAS_REGISTER(Register, padreg, x31);
476 
477 // Keeps the 0 double value.
478 ALIAS_REGISTER(VRegister, fp_zero, d15);
479 // MacroAssembler fixed V Registers.
480 ALIAS_REGISTER(VRegister, fp_fixed1, d28);
481 ALIAS_REGISTER(VRegister, fp_fixed2, d29);
482 
483 // MacroAssembler scratch V registers.
484 ALIAS_REGISTER(VRegister, fp_scratch, d30);
485 ALIAS_REGISTER(VRegister, fp_scratch1, d30);
486 ALIAS_REGISTER(VRegister, fp_scratch2, d31);
487 
488 #undef ALIAS_REGISTER
489 
490 // AreAliased returns true if any of the named registers overlap. Arguments set
491 // to NoReg are ignored. The system stack pointer may be specified.
492 bool AreAliased(const CPURegister& reg1,
493  const CPURegister& reg2,
494  const CPURegister& reg3 = NoReg,
495  const CPURegister& reg4 = NoReg,
496  const CPURegister& reg5 = NoReg,
497  const CPURegister& reg6 = NoReg,
498  const CPURegister& reg7 = NoReg,
499  const CPURegister& reg8 = NoReg);
500 
501 // AreSameSizeAndType returns true if all of the specified registers have the
502 // same size, and are of the same type. The system stack pointer may be
503 // specified. Arguments set to NoReg are ignored, as are any subsequent
504 // arguments. At least one argument (reg1) must be valid (not NoCPUReg).
505 bool AreSameSizeAndType(
506  const CPURegister& reg1, const CPURegister& reg2 = NoCPUReg,
507  const CPURegister& reg3 = NoCPUReg, const CPURegister& reg4 = NoCPUReg,
508  const CPURegister& reg5 = NoCPUReg, const CPURegister& reg6 = NoCPUReg,
509  const CPURegister& reg7 = NoCPUReg, const CPURegister& reg8 = NoCPUReg);
510 
511 // AreSameFormat returns true if all of the specified VRegisters have the same
512 // vector format. Arguments set to NoVReg are ignored, as are any subsequent
513 // arguments. At least one argument (reg1) must be valid (not NoVReg).
514 bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
515  const VRegister& reg3 = NoVReg,
516  const VRegister& reg4 = NoVReg);
517 
518 // AreConsecutive returns true if all of the specified VRegisters are
519 // consecutive in the register file. Arguments may be set to NoVReg, and if so,
520 // subsequent arguments must also be NoVReg. At least one argument (reg1) must
521 // be valid (not NoVReg).
522 bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
523  const VRegister& reg3 = NoVReg,
524  const VRegister& reg4 = NoVReg);
525 
526 typedef VRegister FloatRegister;
527 typedef VRegister DoubleRegister;
528 typedef VRegister Simd128Register;
529 
530 // -----------------------------------------------------------------------------
531 // Lists of registers.
532 class CPURegList {
533  public:
534  template <typename... CPURegisters>
535  explicit CPURegList(CPURegister reg0, CPURegisters... regs)
536  : list_(CPURegister::ListOf(reg0, regs...)),
537  size_(reg0.SizeInBits()),
538  type_(reg0.type()) {
539  DCHECK(AreSameSizeAndType(reg0, regs...));
540  DCHECK(IsValid());
541  }
542 
543  CPURegList(CPURegister::RegisterType type, int size, RegList list)
544  : list_(list), size_(size), type_(type) {
545  DCHECK(IsValid());
546  }
547 
548  CPURegList(CPURegister::RegisterType type, int size, int first_reg,
549  int last_reg)
550  : size_(size), type_(type) {
551  DCHECK(
552  ((type == CPURegister::kRegister) && (last_reg < kNumberOfRegisters)) ||
553  ((type == CPURegister::kVRegister) &&
554  (last_reg < kNumberOfVRegisters)));
555  DCHECK(last_reg >= first_reg);
556  list_ = (1ULL << (last_reg + 1)) - 1;
557  list_ &= ~((1ULL << first_reg) - 1);
558  DCHECK(IsValid());
559  }
560 
561  CPURegister::RegisterType type() const {
562  DCHECK(IsValid());
563  return type_;
564  }
565 
566  RegList list() const {
567  DCHECK(IsValid());
568  return list_;
569  }
570 
571  inline void set_list(RegList new_list) {
572  DCHECK(IsValid());
573  list_ = new_list;
574  }
575 
576  // Combine another CPURegList into this one. Registers that already exist in
577  // this list are left unchanged. The type and size of the registers in the
578  // 'other' list must match those in this list.
579  void Combine(const CPURegList& other);
580 
581  // Remove every register in the other CPURegList from this one. Registers that
582  // do not exist in this list are ignored. The type of the registers in the
583  // 'other' list must match those in this list.
584  void Remove(const CPURegList& other);
585 
586  // Variants of Combine and Remove which take CPURegisters.
587  void Combine(const CPURegister& other);
588  void Remove(const CPURegister& other1,
589  const CPURegister& other2 = NoCPUReg,
590  const CPURegister& other3 = NoCPUReg,
591  const CPURegister& other4 = NoCPUReg);
592 
593  // Variants of Combine and Remove which take a single register by its code;
594  // the type and size of the register is inferred from this list.
595  void Combine(int code);
596  void Remove(int code);
597 
598  // Remove all callee-saved registers from the list. This can be useful when
599  // preparing registers for an AAPCS64 function call, for example.
600  void RemoveCalleeSaved();
601 
602  CPURegister PopLowestIndex();
603  CPURegister PopHighestIndex();
604 
605  // AAPCS64 callee-saved registers.
606  static CPURegList GetCalleeSaved(int size = kXRegSizeInBits);
607  static CPURegList GetCalleeSavedV(int size = kDRegSizeInBits);
608 
609  // AAPCS64 caller-saved registers. Note that this includes lr.
610  // TODO(all): Determine how we handle d8-d15 being callee-saved, but the top
611  // 64-bits being caller-saved.
612  static CPURegList GetCallerSaved(int size = kXRegSizeInBits);
613  static CPURegList GetCallerSavedV(int size = kDRegSizeInBits);
614 
615  // Registers saved as safepoints.
616  static CPURegList GetSafepointSavedRegisters();
617 
618  bool IsEmpty() const {
619  DCHECK(IsValid());
620  return list_ == 0;
621  }
622 
623  bool IncludesAliasOf(const CPURegister& other1,
624  const CPURegister& other2 = NoCPUReg,
625  const CPURegister& other3 = NoCPUReg,
626  const CPURegister& other4 = NoCPUReg) const {
627  DCHECK(IsValid());
628  RegList list = 0;
629  if (!other1.IsNone() && (other1.type() == type_)) list |= other1.bit();
630  if (!other2.IsNone() && (other2.type() == type_)) list |= other2.bit();
631  if (!other3.IsNone() && (other3.type() == type_)) list |= other3.bit();
632  if (!other4.IsNone() && (other4.type() == type_)) list |= other4.bit();
633  return (list_ & list) != 0;
634  }
635 
636  int Count() const {
637  DCHECK(IsValid());
638  return CountSetBits(list_, kRegListSizeInBits);
639  }
640 
641  int RegisterSizeInBits() const {
642  DCHECK(IsValid());
643  return size_;
644  }
645 
646  int RegisterSizeInBytes() const {
647  int size_in_bits = RegisterSizeInBits();
648  DCHECK_EQ(size_in_bits % kBitsPerByte, 0);
649  return size_in_bits / kBitsPerByte;
650  }
651 
652  int TotalSizeInBytes() const {
653  DCHECK(IsValid());
654  return RegisterSizeInBytes() * Count();
655  }
656 
657  private:
658  RegList list_;
659  int size_;
660  CPURegister::RegisterType type_;
661 
662  bool IsValid() const {
663  constexpr RegList kValidRegisters{0x8000000ffffffff};
664  constexpr RegList kValidVRegisters{0x0000000ffffffff};
665  switch (type_) {
666  case CPURegister::kRegister:
667  return (list_ & kValidRegisters) == list_;
668  case CPURegister::kVRegister:
669  return (list_ & kValidVRegisters) == list_;
670  case CPURegister::kNoRegister:
671  return list_ == 0;
672  default:
673  UNREACHABLE();
674  }
675  }
676 };
677 
678 
679 // AAPCS64 callee-saved registers.
680 #define kCalleeSaved CPURegList::GetCalleeSaved()
681 #define kCalleeSavedV CPURegList::GetCalleeSavedV()
682 
683 // AAPCS64 caller-saved registers. Note that this includes lr.
684 #define kCallerSaved CPURegList::GetCallerSaved()
685 #define kCallerSavedV CPURegList::GetCallerSavedV()
686 
687 // -----------------------------------------------------------------------------
688 // Immediates.
689 class Immediate {
690  public:
691  template<typename T>
692  inline explicit Immediate(Handle<T> handle);
693 
694  // This is allowed to be an implicit constructor because Immediate is
695  // a wrapper class that doesn't normally perform any type conversion.
696  template<typename T>
697  inline Immediate(T value); // NOLINT(runtime/explicit)
698 
699  template<typename T>
700  inline Immediate(T value, RelocInfo::Mode rmode);
701 
702  int64_t value() const { return value_; }
703  RelocInfo::Mode rmode() const { return rmode_; }
704 
705  private:
706  void InitializeHandle(Handle<HeapObject> value);
707 
708  int64_t value_;
709  RelocInfo::Mode rmode_;
710 };
711 
712 
713 // -----------------------------------------------------------------------------
714 // Operands.
715 constexpr int kSmiShift = kSmiTagSize + kSmiShiftSize;
716 constexpr uint64_t kSmiShiftMask = (1ULL << kSmiShift) - 1;
717 
718 // Represents an operand in a machine instruction.
719 class Operand {
720  // TODO(all): If necessary, study more in details which methods
721  // TODO(all): should be inlined or not.
722  public:
723  // rm, {<shift> {#<shift_amount>}}
724  // where <shift> is one of {LSL, LSR, ASR, ROR}.
725  // <shift_amount> is uint6_t.
726  // This is allowed to be an implicit constructor because Operand is
727  // a wrapper class that doesn't normally perform any type conversion.
728  inline Operand(Register reg,
729  Shift shift = LSL,
730  unsigned shift_amount = 0); // NOLINT(runtime/explicit)
731 
732  // rm, <extend> {#<shift_amount>}
733  // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
734  // <shift_amount> is uint2_t.
735  inline Operand(Register reg,
736  Extend extend,
737  unsigned shift_amount = 0);
738 
739  static Operand EmbeddedNumber(double number); // Smi or HeapNumber.
740  static Operand EmbeddedCode(CodeStub* stub);
741  static Operand EmbeddedStringConstant(const StringConstantBase* str);
742 
743  inline bool IsHeapObjectRequest() const;
744  inline HeapObjectRequest heap_object_request() const;
745  inline Immediate immediate_for_heap_object_request() const;
746 
747  template<typename T>
748  inline explicit Operand(Handle<T> handle);
749 
750  // Implicit constructor for all int types, ExternalReference, and Smi.
751  template<typename T>
752  inline Operand(T t); // NOLINT(runtime/explicit)
753 
754  // Implicit constructor for int types.
755  template<typename T>
756  inline Operand(T t, RelocInfo::Mode rmode);
757 
758  inline bool IsImmediate() const;
759  inline bool IsShiftedRegister() const;
760  inline bool IsExtendedRegister() const;
761  inline bool IsZero() const;
762 
763  // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
764  // which helps in the encoding of instructions that use the stack pointer.
765  inline Operand ToExtendedRegister() const;
766 
767  inline Immediate immediate() const;
768  inline int64_t ImmediateValue() const;
769  inline RelocInfo::Mode ImmediateRMode() const;
770  inline Register reg() const;
771  inline Shift shift() const;
772  inline Extend extend() const;
773  inline unsigned shift_amount() const;
774 
775  // Relocation information.
776  bool NeedsRelocation(const Assembler* assembler) const;
777 
778  // Helpers
779  inline static Operand UntagSmi(Register smi);
780  inline static Operand UntagSmiAndScale(Register smi, int scale);
781 
782  private:
783  base::Optional<HeapObjectRequest> heap_object_request_;
784  Immediate immediate_;
785  Register reg_;
786  Shift shift_;
787  Extend extend_;
788  unsigned shift_amount_;
789 };
790 
791 
792 // MemOperand represents a memory operand in a load or store instruction.
793 class MemOperand {
794  public:
795  inline MemOperand();
796  inline explicit MemOperand(Register base,
797  int64_t offset = 0,
798  AddrMode addrmode = Offset);
799  inline explicit MemOperand(Register base,
800  Register regoffset,
801  Shift shift = LSL,
802  unsigned shift_amount = 0);
803  inline explicit MemOperand(Register base,
804  Register regoffset,
805  Extend extend,
806  unsigned shift_amount = 0);
807  inline explicit MemOperand(Register base,
808  const Operand& offset,
809  AddrMode addrmode = Offset);
810 
811  const Register& base() const { return base_; }
812  const Register& regoffset() const { return regoffset_; }
813  int64_t offset() const { return offset_; }
814  AddrMode addrmode() const { return addrmode_; }
815  Shift shift() const { return shift_; }
816  Extend extend() const { return extend_; }
817  unsigned shift_amount() const { return shift_amount_; }
818  inline bool IsImmediateOffset() const;
819  inline bool IsRegisterOffset() const;
820  inline bool IsPreIndex() const;
821  inline bool IsPostIndex() const;
822 
823  // For offset modes, return the offset as an Operand. This helper cannot
824  // handle indexed modes.
825  inline Operand OffsetAsOperand() const;
826 
827  enum PairResult {
828  kNotPair, // Can't use a pair instruction.
829  kPairAB, // Can use a pair instruction (operandA has lower address).
830  kPairBA // Can use a pair instruction (operandB has lower address).
831  };
832  // Check if two MemOperand are consistent for stp/ldp use.
833  static PairResult AreConsistentForPair(const MemOperand& operandA,
834  const MemOperand& operandB,
835  int access_size_log2 = kXRegSizeLog2);
836 
837  private:
838  Register base_;
839  Register regoffset_;
840  int64_t offset_;
841  AddrMode addrmode_;
842  Shift shift_;
843  Extend extend_;
844  unsigned shift_amount_;
845 };
846 
847 
848 class ConstPool {
849  public:
850  explicit ConstPool(Assembler* assm) : assm_(assm), first_use_(-1) {}
851  // Returns true when we need to write RelocInfo and false when we do not.
852  bool RecordEntry(intptr_t data, RelocInfo::Mode mode);
853  int EntryCount() const { return static_cast<int>(entries_.size()); }
854  bool IsEmpty() const { return entries_.empty(); }
855  // Distance in bytes between the current pc and the first instruction
856  // using the pool. If there are no pending entries return kMaxInt.
857  int DistanceToFirstUse();
858  // Offset after which instructions using the pool will be out of range.
859  int MaxPcOffset();
860  // Maximum size the constant pool can be with current entries. It always
861  // includes alignment padding and branch over.
862  int WorstCaseSize();
863  // Size in bytes of the literal pool *if* it is emitted at the current
864  // pc. The size will include the branch over the pool if it was requested.
865  int SizeIfEmittedAtCurrentPc(bool require_jump);
866  // Emit the literal pool at the current pc with a branch over the pool if
867  // requested.
868  void Emit(bool require_jump);
869  // Discard any pending pool entries.
870  void Clear();
871 
872  private:
873  void EmitMarker();
874  void EmitGuard();
875  void EmitEntries();
876 
877  typedef std::map<uint64_t, int> SharedEntryMap;
878  // Adds a shared entry to entries_, using 'entry_map' to determine whether we
879  // already track this entry. Returns true if this is the first time we add
880  // this entry, false otherwise.
881  bool AddSharedEntry(SharedEntryMap& entry_map, uint64_t data, int offset);
882 
883  Assembler* assm_;
884  // Keep track of the first instruction requiring a constant pool entry
885  // since the previous constant pool was emitted.
886  int first_use_;
887 
888  // Map of data to index in entries_ for shared entries.
889  SharedEntryMap shared_entries_;
890 
891  // Map of address of handle to index in entries_. We need to keep track of
892  // code targets separately from other shared entries, as they can be
893  // relocated.
894  SharedEntryMap handle_to_index_map_;
895 
896  // Values, pc offset(s) of entries. Use a vector to preserve the order of
897  // insertion, as the serializer expects code target RelocInfo to point to
898  // constant pool addresses in an ascending order.
899  std::vector<std::pair<uint64_t, std::vector<int> > > entries_;
900 };
901 
902 
903 // -----------------------------------------------------------------------------
904 // Assembler.
905 
906 class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
907  public:
908  // Create an assembler. Instructions and relocation information are emitted
909  // into a buffer, with the instructions starting from the beginning and the
910  // relocation information starting from the end of the buffer. See CodeDesc
911  // for a detailed comment on the layout (globals.h).
912  //
913  // If the provided buffer is nullptr, the assembler allocates and grows its
914  // own buffer, and buffer_size determines the initial buffer size. The buffer
915  // is owned by the assembler and deallocated upon destruction of the
916  // assembler.
917  //
918  // If the provided buffer is not nullptr, the assembler uses the provided
919  // buffer for code generation and assumes its size to be buffer_size. If the
920  // buffer is too small, a fatal error occurs. No deallocation of the buffer is
921  // done upon destruction of the assembler.
922  Assembler(const AssemblerOptions& options, void* buffer, int buffer_size);
923 
924  virtual ~Assembler();
925 
926  virtual void AbortedCodeGeneration() {
927  constpool_.Clear();
928  }
929 
930  // System functions ---------------------------------------------------------
931  // Start generating code from the beginning of the buffer, discarding any code
932  // and data that has already been emitted into the buffer.
933  //
934  // In order to avoid any accidental transfer of state, Reset DCHECKs that the
935  // constant pool is not blocked.
936  void Reset();
937 
938  // GetCode emits any pending (non-emitted) code and fills the descriptor
939  // desc. GetCode() is idempotent; it returns the same result if no other
940  // Assembler functions are invoked in between GetCode() calls.
941  //
942  // The descriptor (desc) can be nullptr. In that case, the code is finalized
943  // as usual, but the descriptor is not populated.
944  void GetCode(Isolate* isolate, CodeDesc* desc);
945 
946  // Insert the smallest number of nop instructions
947  // possible to align the pc offset to a multiple
948  // of m. m must be a power of 2 (>= 4).
949  void Align(int m);
950  // Insert the smallest number of zero bytes possible to align the pc offset
951  // to a mulitple of m. m must be a power of 2 (>= 2).
952  void DataAlign(int m);
953 
954  inline void Unreachable();
955 
956  // Label --------------------------------------------------------------------
957  // Bind a label to the current pc. Note that labels can only be bound once,
958  // and if labels are linked to other instructions, they _must_ be bound
959  // before they go out of scope.
960  void bind(Label* label);
961 
962 
963  // RelocInfo and pools ------------------------------------------------------
964 
965  // Record relocation information for current pc_.
966  enum ConstantPoolMode { NEEDS_POOL_ENTRY, NO_POOL_ENTRY };
967  void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0,
968  ConstantPoolMode constant_pool_mode = NEEDS_POOL_ENTRY);
969 
970  // Generate a B immediate instruction with the corresponding relocation info.
971  // 'offset' is the immediate to encode in the B instruction (so it is the
972  // difference between the target and the PC of the instruction, divided by
973  // the instruction size).
974  void near_jump(int offset, RelocInfo::Mode rmode);
975  // Generate a BL immediate instruction with the corresponding relocation info.
976  // As for near_jump, 'offset' is the immediate to encode in the BL
977  // instruction.
978  void near_call(int offset, RelocInfo::Mode rmode);
979  // Generate a BL immediate instruction with the corresponding relocation info
980  // for the input HeapObjectRequest.
981  void near_call(HeapObjectRequest request);
982 
983  // Return the address in the constant pool of the code target address used by
984  // the branch/call instruction at pc.
985  inline static Address target_pointer_address_at(Address pc);
986 
987  // Read/Modify the code target address in the branch/call instruction at pc.
988  // The isolate argument is unused (and may be nullptr) when skipping flushing.
989  inline static Address target_address_at(Address pc, Address constant_pool);
990  inline static void set_target_address_at(
991  Address pc, Address constant_pool, Address target,
992  ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
993 
994  // Returns the handle for the code object called at 'pc'.
995  // This might need to be temporarily encoded as an offset into code_targets_.
996  inline Handle<Code> code_target_object_handle_at(Address pc);
997 
998  // Returns the target address for a runtime function for the call encoded
999  // at 'pc'.
1000  // Runtime entries can be temporarily encoded as the offset between the
1001  // runtime function entrypoint and the code range start (stored in the
1002  // code_range_start field), in order to be encodable as we generate the code,
1003  // before it is moved into the code space.
1004  inline Address runtime_entry_at(Address pc);
1005 
1006  // Return the code target address at a call site from the return address of
1007  // that call in the instruction stream.
1008  inline static Address target_address_from_return_address(Address pc);
1009 
1010  // This sets the branch destination. 'location' here can be either the pc of
1011  // an immediate branch or the address of an entry in the constant pool.
1012  // This is for calls and branches within generated code.
1013  inline static void deserialization_set_special_target_at(Address location,
1014  Code code,
1015  Address target);
1016 
1017  // Get the size of the special target encoded at 'location'.
1018  inline static int deserialization_special_target_size(Address location);
1019 
1020  // This sets the internal reference at the pc.
1021  inline static void deserialization_set_target_internal_reference_at(
1022  Address pc, Address target,
1023  RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
1024 
1025  // This value is used in the serialization process and must be zero for
1026  // ARM64, as the code target is split across multiple instructions and does
1027  // not exist separately in the code, so the serializer should not step
1028  // forwards in memory after a target is resolved and written.
1029  static constexpr int kSpecialTargetSize = 0;
1030 
1031  // Size of the generated code in bytes
1032  uint64_t SizeOfGeneratedCode() const {
1033  DCHECK((pc_ >= buffer_) && (pc_ < (buffer_ + buffer_size_)));
1034  return pc_ - buffer_;
1035  }
1036 
1037  // Return the code size generated from label to the current position.
1038  uint64_t SizeOfCodeGeneratedSince(const Label* label) {
1039  DCHECK(label->is_bound());
1040  DCHECK(pc_offset() >= label->pos());
1041  DCHECK(pc_offset() < buffer_size_);
1042  return pc_offset() - label->pos();
1043  }
1044 
1045  // Return the number of instructions generated from label to the
1046  // current position.
1047  uint64_t InstructionsGeneratedSince(const Label* label) {
1048  return SizeOfCodeGeneratedSince(label) / kInstrSize;
1049  }
1050 
1051  // Prevent contant pool emission until EndBlockConstPool is called.
1052  // Call to this function can be nested but must be followed by an equal
1053  // number of calls to EndBlockConstpool.
1054  void StartBlockConstPool();
1055 
1056  // Resume constant pool emission. Need to be called as many time as
1057  // StartBlockConstPool to have an effect.
1058  void EndBlockConstPool();
1059 
1060  bool is_const_pool_blocked() const;
1061  static bool IsConstantPoolAt(Instruction* instr);
1062  static int ConstantPoolSizeAt(Instruction* instr);
1063  // See Assembler::CheckConstPool for more info.
1064  void EmitPoolGuard();
1065 
1066  // Prevent veneer pool emission until EndBlockVeneerPool is called.
1067  // Call to this function can be nested but must be followed by an equal
1068  // number of calls to EndBlockConstpool.
1069  void StartBlockVeneerPool();
1070 
1071  // Resume constant pool emission. Need to be called as many time as
1072  // StartBlockVeneerPool to have an effect.
1073  void EndBlockVeneerPool();
1074 
1075  bool is_veneer_pool_blocked() const {
1076  return veneer_pool_blocked_nesting_ > 0;
1077  }
1078 
1079  // Block/resume emission of constant pools and veneer pools.
1080  void StartBlockPools() {
1081  StartBlockConstPool();
1082  StartBlockVeneerPool();
1083  }
1084  void EndBlockPools() {
1085  EndBlockConstPool();
1086  EndBlockVeneerPool();
1087  }
1088 
1089  // Debugging ----------------------------------------------------------------
1090  void RecordComment(const char* msg);
1091 
1092  // Record a deoptimization reason that can be used by a log or cpu profiler.
1093  // Use --trace-deopt to enable.
1094  void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
1095  int id);
1096 
1097  int buffer_space() const;
1098 
1099  // Record the emission of a constant pool.
1100  //
1101  // The emission of constant and veneer pools depends on the size of the code
1102  // generated and the number of RelocInfo recorded.
1103  // The Debug mechanism needs to map code offsets between two versions of a
1104  // function, compiled with and without debugger support (see for example
1105  // Debug::PrepareForBreakPoints()).
1106  // Compiling functions with debugger support generates additional code
1107  // (DebugCodegen::GenerateSlot()). This may affect the emission of the pools
1108  // and cause the version of the code with debugger support to have pools
1109  // generated in different places.
1110  // Recording the position and size of emitted pools allows to correctly
1111  // compute the offset mappings between the different versions of a function in
1112  // all situations.
1113  //
1114  // The parameter indicates the size of the pool (in bytes), including
1115  // the marker and branch over the data.
1116  void RecordConstPool(int size);
1117 
1118  // Instruction set functions ------------------------------------------------
1119 
1120  // Branch / Jump instructions.
1121  // For branches offsets are scaled, i.e. they in instrcutions not in bytes.
1122  // Branch to register.
1123  void br(const Register& xn);
1124 
1125  // Branch-link to register.
1126  void blr(const Register& xn);
1127 
1128  // Branch to register with return hint.
1129  void ret(const Register& xn = lr);
1130 
1131  // Unconditional branch to label.
1132  void b(Label* label);
1133 
1134  // Conditional branch to label.
1135  void b(Label* label, Condition cond);
1136 
1137  // Unconditional branch to PC offset.
1138  void b(int imm26);
1139 
1140  // Conditional branch to PC offset.
1141  void b(int imm19, Condition cond);
1142 
1143  // Branch-link to label / pc offset.
1144  void bl(Label* label);
1145  void bl(int imm26);
1146 
1147  // Compare and branch to label / pc offset if zero.
1148  void cbz(const Register& rt, Label* label);
1149  void cbz(const Register& rt, int imm19);
1150 
1151  // Compare and branch to label / pc offset if not zero.
1152  void cbnz(const Register& rt, Label* label);
1153  void cbnz(const Register& rt, int imm19);
1154 
1155  // Test bit and branch to label / pc offset if zero.
1156  void tbz(const Register& rt, unsigned bit_pos, Label* label);
1157  void tbz(const Register& rt, unsigned bit_pos, int imm14);
1158 
1159  // Test bit and branch to label / pc offset if not zero.
1160  void tbnz(const Register& rt, unsigned bit_pos, Label* label);
1161  void tbnz(const Register& rt, unsigned bit_pos, int imm14);
1162 
1163  // Address calculation instructions.
1164  // Calculate a PC-relative address. Unlike for branches the offset in adr is
1165  // unscaled (i.e. the result can be unaligned).
1166  void adr(const Register& rd, Label* label);
1167  void adr(const Register& rd, int imm21);
1168 
1169  // Data Processing instructions.
1170  // Add.
1171  void add(const Register& rd,
1172  const Register& rn,
1173  const Operand& operand);
1174 
1175  // Add and update status flags.
1176  void adds(const Register& rd,
1177  const Register& rn,
1178  const Operand& operand);
1179 
1180  // Compare negative.
1181  void cmn(const Register& rn, const Operand& operand);
1182 
1183  // Subtract.
1184  void sub(const Register& rd,
1185  const Register& rn,
1186  const Operand& operand);
1187 
1188  // Subtract and update status flags.
1189  void subs(const Register& rd,
1190  const Register& rn,
1191  const Operand& operand);
1192 
1193  // Compare.
1194  void cmp(const Register& rn, const Operand& operand);
1195 
1196  // Negate.
1197  void neg(const Register& rd,
1198  const Operand& operand);
1199 
1200  // Negate and update status flags.
1201  void negs(const Register& rd,
1202  const Operand& operand);
1203 
1204  // Add with carry bit.
1205  void adc(const Register& rd,
1206  const Register& rn,
1207  const Operand& operand);
1208 
1209  // Add with carry bit and update status flags.
1210  void adcs(const Register& rd,
1211  const Register& rn,
1212  const Operand& operand);
1213 
1214  // Subtract with carry bit.
1215  void sbc(const Register& rd,
1216  const Register& rn,
1217  const Operand& operand);
1218 
1219  // Subtract with carry bit and update status flags.
1220  void sbcs(const Register& rd,
1221  const Register& rn,
1222  const Operand& operand);
1223 
1224  // Negate with carry bit.
1225  void ngc(const Register& rd,
1226  const Operand& operand);
1227 
1228  // Negate with carry bit and update status flags.
1229  void ngcs(const Register& rd,
1230  const Operand& operand);
1231 
1232  // Logical instructions.
1233  // Bitwise and (A & B).
1234  void and_(const Register& rd,
1235  const Register& rn,
1236  const Operand& operand);
1237 
1238  // Bitwise and (A & B) and update status flags.
1239  void ands(const Register& rd,
1240  const Register& rn,
1241  const Operand& operand);
1242 
1243  // Bit test, and set flags.
1244  void tst(const Register& rn, const Operand& operand);
1245 
1246  // Bit clear (A & ~B).
1247  void bic(const Register& rd,
1248  const Register& rn,
1249  const Operand& operand);
1250 
1251  // Bit clear (A & ~B) and update status flags.
1252  void bics(const Register& rd,
1253  const Register& rn,
1254  const Operand& operand);
1255 
1256  // Bitwise and.
1257  void and_(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1258 
1259  // Bit clear immediate.
1260  void bic(const VRegister& vd, const int imm8, const int left_shift = 0);
1261 
1262  // Bit clear.
1263  void bic(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1264 
1265  // Bitwise insert if false.
1266  void bif(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1267 
1268  // Bitwise insert if true.
1269  void bit(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1270 
1271  // Bitwise select.
1272  void bsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1273 
1274  // Polynomial multiply.
1275  void pmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1276 
1277  // Vector move immediate.
1278  void movi(const VRegister& vd, const uint64_t imm, Shift shift = LSL,
1279  const int shift_amount = 0);
1280 
1281  // Bitwise not.
1282  void mvn(const VRegister& vd, const VRegister& vn);
1283 
1284  // Vector move inverted immediate.
1285  void mvni(const VRegister& vd, const int imm8, Shift shift = LSL,
1286  const int shift_amount = 0);
1287 
1288  // Signed saturating accumulate of unsigned value.
1289  void suqadd(const VRegister& vd, const VRegister& vn);
1290 
1291  // Unsigned saturating accumulate of signed value.
1292  void usqadd(const VRegister& vd, const VRegister& vn);
1293 
1294  // Absolute value.
1295  void abs(const VRegister& vd, const VRegister& vn);
1296 
1297  // Signed saturating absolute value.
1298  void sqabs(const VRegister& vd, const VRegister& vn);
1299 
1300  // Negate.
1301  void neg(const VRegister& vd, const VRegister& vn);
1302 
1303  // Signed saturating negate.
1304  void sqneg(const VRegister& vd, const VRegister& vn);
1305 
1306  // Bitwise not.
1307  void not_(const VRegister& vd, const VRegister& vn);
1308 
1309  // Extract narrow.
1310  void xtn(const VRegister& vd, const VRegister& vn);
1311 
1312  // Extract narrow (second part).
1313  void xtn2(const VRegister& vd, const VRegister& vn);
1314 
1315  // Signed saturating extract narrow.
1316  void sqxtn(const VRegister& vd, const VRegister& vn);
1317 
1318  // Signed saturating extract narrow (second part).
1319  void sqxtn2(const VRegister& vd, const VRegister& vn);
1320 
1321  // Unsigned saturating extract narrow.
1322  void uqxtn(const VRegister& vd, const VRegister& vn);
1323 
1324  // Unsigned saturating extract narrow (second part).
1325  void uqxtn2(const VRegister& vd, const VRegister& vn);
1326 
1327  // Signed saturating extract unsigned narrow.
1328  void sqxtun(const VRegister& vd, const VRegister& vn);
1329 
1330  // Signed saturating extract unsigned narrow (second part).
1331  void sqxtun2(const VRegister& vd, const VRegister& vn);
1332 
1333  // Move register to register.
1334  void mov(const VRegister& vd, const VRegister& vn);
1335 
1336  // Bitwise not or.
1337  void orn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1338 
1339  // Bitwise exclusive or.
1340  void eor(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1341 
1342  // Bitwise or (A | B).
1343  void orr(const Register& rd, const Register& rn, const Operand& operand);
1344 
1345  // Bitwise or.
1346  void orr(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1347 
1348  // Bitwise or immediate.
1349  void orr(const VRegister& vd, const int imm8, const int left_shift = 0);
1350 
1351  // Bitwise nor (A | ~B).
1352  void orn(const Register& rd, const Register& rn, const Operand& operand);
1353 
1354  // Bitwise eor/xor (A ^ B).
1355  void eor(const Register& rd, const Register& rn, const Operand& operand);
1356 
1357  // Bitwise enor/xnor (A ^ ~B).
1358  void eon(const Register& rd, const Register& rn, const Operand& operand);
1359 
1360  // Logical shift left variable.
1361  void lslv(const Register& rd, const Register& rn, const Register& rm);
1362 
1363  // Logical shift right variable.
1364  void lsrv(const Register& rd, const Register& rn, const Register& rm);
1365 
1366  // Arithmetic shift right variable.
1367  void asrv(const Register& rd, const Register& rn, const Register& rm);
1368 
1369  // Rotate right variable.
1370  void rorv(const Register& rd, const Register& rn, const Register& rm);
1371 
1372  // Bitfield instructions.
1373  // Bitfield move.
1374  void bfm(const Register& rd, const Register& rn, int immr, int imms);
1375 
1376  // Signed bitfield move.
1377  void sbfm(const Register& rd, const Register& rn, int immr, int imms);
1378 
1379  // Unsigned bitfield move.
1380  void ubfm(const Register& rd, const Register& rn, int immr, int imms);
1381 
1382  // Bfm aliases.
1383  // Bitfield insert.
1384  void bfi(const Register& rd, const Register& rn, int lsb, int width) {
1385  DCHECK_GE(width, 1);
1386  DCHECK(lsb + width <= rn.SizeInBits());
1387  bfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
1388  }
1389 
1390  // Bitfield extract and insert low.
1391  void bfxil(const Register& rd, const Register& rn, int lsb, int width) {
1392  DCHECK_GE(width, 1);
1393  DCHECK(lsb + width <= rn.SizeInBits());
1394  bfm(rd, rn, lsb, lsb + width - 1);
1395  }
1396 
1397  // Sbfm aliases.
1398  // Arithmetic shift right.
1399  void asr(const Register& rd, const Register& rn, int shift) {
1400  DCHECK(shift < rd.SizeInBits());
1401  sbfm(rd, rn, shift, rd.SizeInBits() - 1);
1402  }
1403 
1404  // Signed bitfield insert in zero.
1405  void sbfiz(const Register& rd, const Register& rn, int lsb, int width) {
1406  DCHECK_GE(width, 1);
1407  DCHECK(lsb + width <= rn.SizeInBits());
1408  sbfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
1409  }
1410 
1411  // Signed bitfield extract.
1412  void sbfx(const Register& rd, const Register& rn, int lsb, int width) {
1413  DCHECK_GE(width, 1);
1414  DCHECK(lsb + width <= rn.SizeInBits());
1415  sbfm(rd, rn, lsb, lsb + width - 1);
1416  }
1417 
1418  // Signed extend byte.
1419  void sxtb(const Register& rd, const Register& rn) {
1420  sbfm(rd, rn, 0, 7);
1421  }
1422 
1423  // Signed extend halfword.
1424  void sxth(const Register& rd, const Register& rn) {
1425  sbfm(rd, rn, 0, 15);
1426  }
1427 
1428  // Signed extend word.
1429  void sxtw(const Register& rd, const Register& rn) {
1430  sbfm(rd, rn, 0, 31);
1431  }
1432 
1433  // Ubfm aliases.
1434  // Logical shift left.
1435  void lsl(const Register& rd, const Register& rn, int shift) {
1436  int reg_size = rd.SizeInBits();
1437  DCHECK(shift < reg_size);
1438  ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
1439  }
1440 
1441  // Logical shift right.
1442  void lsr(const Register& rd, const Register& rn, int shift) {
1443  DCHECK(shift < rd.SizeInBits());
1444  ubfm(rd, rn, shift, rd.SizeInBits() - 1);
1445  }
1446 
1447  // Unsigned bitfield insert in zero.
1448  void ubfiz(const Register& rd, const Register& rn, int lsb, int width) {
1449  DCHECK_GE(width, 1);
1450  DCHECK(lsb + width <= rn.SizeInBits());
1451  ubfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1);
1452  }
1453 
1454  // Unsigned bitfield extract.
1455  void ubfx(const Register& rd, const Register& rn, int lsb, int width) {
1456  DCHECK_GE(width, 1);
1457  DCHECK(lsb + width <= rn.SizeInBits());
1458  ubfm(rd, rn, lsb, lsb + width - 1);
1459  }
1460 
1461  // Unsigned extend byte.
1462  void uxtb(const Register& rd, const Register& rn) {
1463  ubfm(rd, rn, 0, 7);
1464  }
1465 
1466  // Unsigned extend halfword.
1467  void uxth(const Register& rd, const Register& rn) {
1468  ubfm(rd, rn, 0, 15);
1469  }
1470 
1471  // Unsigned extend word.
1472  void uxtw(const Register& rd, const Register& rn) {
1473  ubfm(rd, rn, 0, 31);
1474  }
1475 
1476  // Extract.
1477  void extr(const Register& rd, const Register& rn, const Register& rm,
1478  int lsb);
1479 
1480  // Conditional select: rd = cond ? rn : rm.
1481  void csel(const Register& rd,
1482  const Register& rn,
1483  const Register& rm,
1484  Condition cond);
1485 
1486  // Conditional select increment: rd = cond ? rn : rm + 1.
1487  void csinc(const Register& rd,
1488  const Register& rn,
1489  const Register& rm,
1490  Condition cond);
1491 
1492  // Conditional select inversion: rd = cond ? rn : ~rm.
1493  void csinv(const Register& rd,
1494  const Register& rn,
1495  const Register& rm,
1496  Condition cond);
1497 
1498  // Conditional select negation: rd = cond ? rn : -rm.
1499  void csneg(const Register& rd,
1500  const Register& rn,
1501  const Register& rm,
1502  Condition cond);
1503 
1504  // Conditional set: rd = cond ? 1 : 0.
1505  void cset(const Register& rd, Condition cond);
1506 
1507  // Conditional set minus: rd = cond ? -1 : 0.
1508  void csetm(const Register& rd, Condition cond);
1509 
1510  // Conditional increment: rd = cond ? rn + 1 : rn.
1511  void cinc(const Register& rd, const Register& rn, Condition cond);
1512 
1513  // Conditional invert: rd = cond ? ~rn : rn.
1514  void cinv(const Register& rd, const Register& rn, Condition cond);
1515 
1516  // Conditional negate: rd = cond ? -rn : rn.
1517  void cneg(const Register& rd, const Register& rn, Condition cond);
1518 
1519  // Extr aliases.
1520  void ror(const Register& rd, const Register& rs, unsigned shift) {
1521  extr(rd, rs, rs, shift);
1522  }
1523 
1524  // Conditional comparison.
1525  // Conditional compare negative.
1526  void ccmn(const Register& rn,
1527  const Operand& operand,
1528  StatusFlags nzcv,
1529  Condition cond);
1530 
1531  // Conditional compare.
1532  void ccmp(const Register& rn,
1533  const Operand& operand,
1534  StatusFlags nzcv,
1535  Condition cond);
1536 
1537  // Multiplication.
1538  // 32 x 32 -> 32-bit and 64 x 64 -> 64-bit multiply.
1539  void mul(const Register& rd, const Register& rn, const Register& rm);
1540 
1541  // 32 + 32 x 32 -> 32-bit and 64 + 64 x 64 -> 64-bit multiply accumulate.
1542  void madd(const Register& rd,
1543  const Register& rn,
1544  const Register& rm,
1545  const Register& ra);
1546 
1547  // -(32 x 32) -> 32-bit and -(64 x 64) -> 64-bit multiply.
1548  void mneg(const Register& rd, const Register& rn, const Register& rm);
1549 
1550  // 32 - 32 x 32 -> 32-bit and 64 - 64 x 64 -> 64-bit multiply subtract.
1551  void msub(const Register& rd,
1552  const Register& rn,
1553  const Register& rm,
1554  const Register& ra);
1555 
1556  // 32 x 32 -> 64-bit multiply.
1557  void smull(const Register& rd, const Register& rn, const Register& rm);
1558 
1559  // Xd = bits<127:64> of Xn * Xm.
1560  void smulh(const Register& rd, const Register& rn, const Register& rm);
1561 
1562  // Signed 32 x 32 -> 64-bit multiply and accumulate.
1563  void smaddl(const Register& rd,
1564  const Register& rn,
1565  const Register& rm,
1566  const Register& ra);
1567 
1568  // Unsigned 32 x 32 -> 64-bit multiply and accumulate.
1569  void umaddl(const Register& rd,
1570  const Register& rn,
1571  const Register& rm,
1572  const Register& ra);
1573 
1574  // Signed 32 x 32 -> 64-bit multiply and subtract.
1575  void smsubl(const Register& rd,
1576  const Register& rn,
1577  const Register& rm,
1578  const Register& ra);
1579 
1580  // Unsigned 32 x 32 -> 64-bit multiply and subtract.
1581  void umsubl(const Register& rd,
1582  const Register& rn,
1583  const Register& rm,
1584  const Register& ra);
1585 
1586  // Signed integer divide.
1587  void sdiv(const Register& rd, const Register& rn, const Register& rm);
1588 
1589  // Unsigned integer divide.
1590  void udiv(const Register& rd, const Register& rn, const Register& rm);
1591 
1592  // Bit count, bit reverse and endian reverse.
1593  void rbit(const Register& rd, const Register& rn);
1594  void rev16(const Register& rd, const Register& rn);
1595  void rev32(const Register& rd, const Register& rn);
1596  void rev(const Register& rd, const Register& rn);
1597  void clz(const Register& rd, const Register& rn);
1598  void cls(const Register& rd, const Register& rn);
1599 
1600  // Memory instructions.
1601 
1602  // Load integer or FP register.
1603  void ldr(const CPURegister& rt, const MemOperand& src);
1604 
1605  // Store integer or FP register.
1606  void str(const CPURegister& rt, const MemOperand& dst);
1607 
1608  // Load word with sign extension.
1609  void ldrsw(const Register& rt, const MemOperand& src);
1610 
1611  // Load byte.
1612  void ldrb(const Register& rt, const MemOperand& src);
1613 
1614  // Store byte.
1615  void strb(const Register& rt, const MemOperand& dst);
1616 
1617  // Load byte with sign extension.
1618  void ldrsb(const Register& rt, const MemOperand& src);
1619 
1620  // Load half-word.
1621  void ldrh(const Register& rt, const MemOperand& src);
1622 
1623  // Store half-word.
1624  void strh(const Register& rt, const MemOperand& dst);
1625 
1626  // Load half-word with sign extension.
1627  void ldrsh(const Register& rt, const MemOperand& src);
1628 
1629  // Load integer or FP register pair.
1630  void ldp(const CPURegister& rt, const CPURegister& rt2,
1631  const MemOperand& src);
1632 
1633  // Store integer or FP register pair.
1634  void stp(const CPURegister& rt, const CPURegister& rt2,
1635  const MemOperand& dst);
1636 
1637  // Load word pair with sign extension.
1638  void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
1639 
1640  // Load literal to register from a pc relative address.
1641  void ldr_pcrel(const CPURegister& rt, int imm19);
1642 
1643  // Load literal to register.
1644  void ldr(const CPURegister& rt, const Immediate& imm);
1645  void ldr(const CPURegister& rt, const Operand& operand);
1646 
1647  // Load-acquire word.
1648  void ldar(const Register& rt, const Register& rn);
1649 
1650  // Load-acquire exclusive word.
1651  void ldaxr(const Register& rt, const Register& rn);
1652 
1653  // Store-release word.
1654  void stlr(const Register& rt, const Register& rn);
1655 
1656  // Store-release exclusive word.
1657  void stlxr(const Register& rs, const Register& rt, const Register& rn);
1658 
1659  // Load-acquire byte.
1660  void ldarb(const Register& rt, const Register& rn);
1661 
1662  // Load-acquire exclusive byte.
1663  void ldaxrb(const Register& rt, const Register& rn);
1664 
1665  // Store-release byte.
1666  void stlrb(const Register& rt, const Register& rn);
1667 
1668  // Store-release exclusive byte.
1669  void stlxrb(const Register& rs, const Register& rt, const Register& rn);
1670 
1671  // Load-acquire half-word.
1672  void ldarh(const Register& rt, const Register& rn);
1673 
1674  // Load-acquire exclusive half-word.
1675  void ldaxrh(const Register& rt, const Register& rn);
1676 
1677  // Store-release half-word.
1678  void stlrh(const Register& rt, const Register& rn);
1679 
1680  // Store-release exclusive half-word.
1681  void stlxrh(const Register& rs, const Register& rt, const Register& rn);
1682 
1683  // Move instructions. The default shift of -1 indicates that the move
1684  // instruction will calculate an appropriate 16-bit immediate and left shift
1685  // that is equal to the 64-bit immediate argument. If an explicit left shift
1686  // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
1687  //
1688  // For movk, an explicit shift can be used to indicate which half word should
1689  // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
1690  // half word with zero, whereas movk(x0, 0, 48) will overwrite the
1691  // most-significant.
1692 
1693  // Move and keep.
1694  void movk(const Register& rd, uint64_t imm, int shift = -1) {
1695  MoveWide(rd, imm, shift, MOVK);
1696  }
1697 
1698  // Move with non-zero.
1699  void movn(const Register& rd, uint64_t imm, int shift = -1) {
1700  MoveWide(rd, imm, shift, MOVN);
1701  }
1702 
1703  // Move with zero.
1704  void movz(const Register& rd, uint64_t imm, int shift = -1) {
1705  MoveWide(rd, imm, shift, MOVZ);
1706  }
1707 
1708  // Misc instructions.
1709  // Monitor debug-mode breakpoint.
1710  void brk(int code);
1711 
1712  // Halting debug-mode breakpoint.
1713  void hlt(int code);
1714 
1715  // Move register to register.
1716  void mov(const Register& rd, const Register& rn);
1717 
1718  // Move NOT(operand) to register.
1719  void mvn(const Register& rd, const Operand& operand);
1720 
1721  // System instructions.
1722  // Move to register from system register.
1723  void mrs(const Register& rt, SystemRegister sysreg);
1724 
1725  // Move from register to system register.
1726  void msr(SystemRegister sysreg, const Register& rt);
1727 
1728  // System hint.
1729  void hint(SystemHint code);
1730 
1731  // Data memory barrier
1732  void dmb(BarrierDomain domain, BarrierType type);
1733 
1734  // Data synchronization barrier
1735  void dsb(BarrierDomain domain, BarrierType type);
1736 
1737  // Instruction synchronization barrier
1738  void isb();
1739 
1740  // Conditional speculation barrier.
1741  void csdb();
1742 
1743  // Alias for system instructions.
1744  void nop() { hint(NOP); }
1745 
1746  // Different nop operations are used by the code generator to detect certain
1747  // states of the generated code.
1748  enum NopMarkerTypes {
1749  DEBUG_BREAK_NOP,
1750  INTERRUPT_CODE_NOP,
1751  ADR_FAR_NOP,
1752  FIRST_NOP_MARKER = DEBUG_BREAK_NOP,
1753  LAST_NOP_MARKER = ADR_FAR_NOP
1754  };
1755 
1756  void nop(NopMarkerTypes n) {
1757  DCHECK((FIRST_NOP_MARKER <= n) && (n <= LAST_NOP_MARKER));
1758  mov(Register::XRegFromCode(n), Register::XRegFromCode(n));
1759  }
1760 
1761  // Add.
1762  void add(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1763 
1764  // Unsigned halving add.
1765  void uhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1766 
1767  // Subtract.
1768  void sub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1769 
1770  // Signed halving add.
1771  void shadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1772 
1773  // Multiply by scalar element.
1774  void mul(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1775  int vm_index);
1776 
1777  // Multiply-add by scalar element.
1778  void mla(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1779  int vm_index);
1780 
1781  // Multiply-subtract by scalar element.
1782  void mls(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1783  int vm_index);
1784 
1785  // Signed long multiply-add by scalar element.
1786  void smlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1787  int vm_index);
1788 
1789  // Signed long multiply-add by scalar element (second part).
1790  void smlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1791  int vm_index);
1792 
1793  // Unsigned long multiply-add by scalar element.
1794  void umlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1795  int vm_index);
1796 
1797  // Unsigned long multiply-add by scalar element (second part).
1798  void umlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1799  int vm_index);
1800 
1801  // Signed long multiply-sub by scalar element.
1802  void smlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1803  int vm_index);
1804 
1805  // Signed long multiply-sub by scalar element (second part).
1806  void smlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1807  int vm_index);
1808 
1809  // Unsigned long multiply-sub by scalar element.
1810  void umlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1811  int vm_index);
1812 
1813  // Unsigned long multiply-sub by scalar element (second part).
1814  void umlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1815  int vm_index);
1816 
1817  // Signed long multiply by scalar element.
1818  void smull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1819  int vm_index);
1820 
1821  // Signed long multiply by scalar element (second part).
1822  void smull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1823  int vm_index);
1824 
1825  // Unsigned long multiply by scalar element.
1826  void umull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1827  int vm_index);
1828 
1829  // Unsigned long multiply by scalar element (second part).
1830  void umull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1831  int vm_index);
1832 
1833  // Add narrow returning high half.
1834  void addhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1835 
1836  // Add narrow returning high half (second part).
1837  void addhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1838 
1839  // Signed saturating double long multiply by element.
1840  void sqdmull(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1841  int vm_index);
1842 
1843  // Signed saturating double long multiply by element (second part).
1844  void sqdmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1845  int vm_index);
1846 
1847  // Signed saturating doubling long multiply-add by element.
1848  void sqdmlal(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1849  int vm_index);
1850 
1851  // Signed saturating doubling long multiply-add by element (second part).
1852  void sqdmlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1853  int vm_index);
1854 
1855  // Signed saturating doubling long multiply-sub by element.
1856  void sqdmlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1857  int vm_index);
1858 
1859  // Signed saturating doubling long multiply-sub by element (second part).
1860  void sqdmlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1861  int vm_index);
1862 
1863  // Compare bitwise to zero.
1864  void cmeq(const VRegister& vd, const VRegister& vn, int value);
1865 
1866  // Compare signed greater than or equal to zero.
1867  void cmge(const VRegister& vd, const VRegister& vn, int value);
1868 
1869  // Compare signed greater than zero.
1870  void cmgt(const VRegister& vd, const VRegister& vn, int value);
1871 
1872  // Compare signed less than or equal to zero.
1873  void cmle(const VRegister& vd, const VRegister& vn, int value);
1874 
1875  // Compare signed less than zero.
1876  void cmlt(const VRegister& vd, const VRegister& vn, int value);
1877 
1878  // Unsigned rounding halving add.
1879  void urhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1880 
1881  // Compare equal.
1882  void cmeq(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1883 
1884  // Compare signed greater than or equal.
1885  void cmge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1886 
1887  // Compare signed greater than.
1888  void cmgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1889 
1890  // Compare unsigned higher.
1891  void cmhi(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1892 
1893  // Compare unsigned higher or same.
1894  void cmhs(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1895 
1896  // Compare bitwise test bits nonzero.
1897  void cmtst(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1898 
1899  // Signed shift left by register.
1900  void sshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1901 
1902  // Unsigned shift left by register.
1903  void ushl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1904 
1905  // Signed saturating doubling long multiply-subtract.
1906  void sqdmlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1907 
1908  // Signed saturating doubling long multiply-subtract (second part).
1909  void sqdmlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1910 
1911  // Signed saturating doubling long multiply.
1912  void sqdmull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1913 
1914  // Signed saturating doubling long multiply (second part).
1915  void sqdmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1916 
1917  // Signed saturating doubling multiply returning high half.
1918  void sqdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1919 
1920  // Signed saturating rounding doubling multiply returning high half.
1921  void sqrdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1922 
1923  // Signed saturating doubling multiply element returning high half.
1924  void sqdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1925  int vm_index);
1926 
1927  // Signed saturating rounding doubling multiply element returning high half.
1928  void sqrdmulh(const VRegister& vd, const VRegister& vn, const VRegister& vm,
1929  int vm_index);
1930 
1931  // Unsigned long multiply long.
1932  void umull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1933 
1934  // Unsigned long multiply (second part).
1935  void umull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1936 
1937  // Rounding add narrow returning high half.
1938  void raddhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1939 
1940  // Subtract narrow returning high half.
1941  void subhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1942 
1943  // Subtract narrow returning high half (second part).
1944  void subhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1945 
1946  // Rounding add narrow returning high half (second part).
1947  void raddhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1948 
1949  // Rounding subtract narrow returning high half.
1950  void rsubhn(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1951 
1952  // Rounding subtract narrow returning high half (second part).
1953  void rsubhn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1954 
1955  // Signed saturating shift left by register.
1956  void sqshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1957 
1958  // Unsigned saturating shift left by register.
1959  void uqshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1960 
1961  // Signed rounding shift left by register.
1962  void srshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1963 
1964  // Unsigned rounding shift left by register.
1965  void urshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1966 
1967  // Signed saturating rounding shift left by register.
1968  void sqrshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1969 
1970  // Unsigned saturating rounding shift left by register.
1971  void uqrshl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1972 
1973  // Signed absolute difference.
1974  void sabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1975 
1976  // Unsigned absolute difference and accumulate.
1977  void uaba(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1978 
1979  // Shift left by immediate and insert.
1980  void sli(const VRegister& vd, const VRegister& vn, int shift);
1981 
1982  // Shift right by immediate and insert.
1983  void sri(const VRegister& vd, const VRegister& vn, int shift);
1984 
1985  // Signed maximum.
1986  void smax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1987 
1988  // Signed pairwise maximum.
1989  void smaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
1990 
1991  // Add across vector.
1992  void addv(const VRegister& vd, const VRegister& vn);
1993 
1994  // Signed add long across vector.
1995  void saddlv(const VRegister& vd, const VRegister& vn);
1996 
1997  // Unsigned add long across vector.
1998  void uaddlv(const VRegister& vd, const VRegister& vn);
1999 
2000  // FP maximum number across vector.
2001  void fmaxnmv(const VRegister& vd, const VRegister& vn);
2002 
2003  // FP maximum across vector.
2004  void fmaxv(const VRegister& vd, const VRegister& vn);
2005 
2006  // FP minimum number across vector.
2007  void fminnmv(const VRegister& vd, const VRegister& vn);
2008 
2009  // FP minimum across vector.
2010  void fminv(const VRegister& vd, const VRegister& vn);
2011 
2012  // Signed maximum across vector.
2013  void smaxv(const VRegister& vd, const VRegister& vn);
2014 
2015  // Signed minimum.
2016  void smin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2017 
2018  // Signed minimum pairwise.
2019  void sminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2020 
2021  // Signed minimum across vector.
2022  void sminv(const VRegister& vd, const VRegister& vn);
2023 
2024  // One-element structure store from one register.
2025  void st1(const VRegister& vt, const MemOperand& src);
2026 
2027  // One-element structure store from two registers.
2028  void st1(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
2029 
2030  // One-element structure store from three registers.
2031  void st1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2032  const MemOperand& src);
2033 
2034  // One-element structure store from four registers.
2035  void st1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2036  const VRegister& vt4, const MemOperand& src);
2037 
2038  // One-element single structure store from one lane.
2039  void st1(const VRegister& vt, int lane, const MemOperand& src);
2040 
2041  // Two-element structure store from two registers.
2042  void st2(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
2043 
2044  // Two-element single structure store from two lanes.
2045  void st2(const VRegister& vt, const VRegister& vt2, int lane,
2046  const MemOperand& src);
2047 
2048  // Three-element structure store from three registers.
2049  void st3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2050  const MemOperand& src);
2051 
2052  // Three-element single structure store from three lanes.
2053  void st3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2054  int lane, const MemOperand& src);
2055 
2056  // Four-element structure store from four registers.
2057  void st4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2058  const VRegister& vt4, const MemOperand& src);
2059 
2060  // Four-element single structure store from four lanes.
2061  void st4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2062  const VRegister& vt4, int lane, const MemOperand& src);
2063 
2064  // Unsigned add long.
2065  void uaddl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2066 
2067  // Unsigned add long (second part).
2068  void uaddl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2069 
2070  // Unsigned add wide.
2071  void uaddw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2072 
2073  // Unsigned add wide (second part).
2074  void uaddw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2075 
2076  // Signed add long.
2077  void saddl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2078 
2079  // Signed add long (second part).
2080  void saddl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2081 
2082  // Signed add wide.
2083  void saddw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2084 
2085  // Signed add wide (second part).
2086  void saddw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2087 
2088  // Unsigned subtract long.
2089  void usubl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2090 
2091  // Unsigned subtract long (second part).
2092  void usubl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2093 
2094  // Unsigned subtract wide.
2095  void usubw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2096 
2097  // Signed subtract long.
2098  void ssubl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2099 
2100  // Signed subtract long (second part).
2101  void ssubl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2102 
2103  // Signed integer subtract wide.
2104  void ssubw(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2105 
2106  // Signed integer subtract wide (second part).
2107  void ssubw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2108 
2109  // Unsigned subtract wide (second part).
2110  void usubw2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2111 
2112  // Unsigned maximum.
2113  void umax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2114 
2115  // Unsigned pairwise maximum.
2116  void umaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2117 
2118  // Unsigned maximum across vector.
2119  void umaxv(const VRegister& vd, const VRegister& vn);
2120 
2121  // Unsigned minimum.
2122  void umin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2123 
2124  // Unsigned pairwise minimum.
2125  void uminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2126 
2127  // Unsigned minimum across vector.
2128  void uminv(const VRegister& vd, const VRegister& vn);
2129 
2130  // Transpose vectors (primary).
2131  void trn1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2132 
2133  // Transpose vectors (secondary).
2134  void trn2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2135 
2136  // Unzip vectors (primary).
2137  void uzp1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2138 
2139  // Unzip vectors (secondary).
2140  void uzp2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2141 
2142  // Zip vectors (primary).
2143  void zip1(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2144 
2145  // Zip vectors (secondary).
2146  void zip2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2147 
2148  // Signed shift right by immediate.
2149  void sshr(const VRegister& vd, const VRegister& vn, int shift);
2150 
2151  // Unsigned shift right by immediate.
2152  void ushr(const VRegister& vd, const VRegister& vn, int shift);
2153 
2154  // Signed rounding shift right by immediate.
2155  void srshr(const VRegister& vd, const VRegister& vn, int shift);
2156 
2157  // Unsigned rounding shift right by immediate.
2158  void urshr(const VRegister& vd, const VRegister& vn, int shift);
2159 
2160  // Signed shift right by immediate and accumulate.
2161  void ssra(const VRegister& vd, const VRegister& vn, int shift);
2162 
2163  // Unsigned shift right by immediate and accumulate.
2164  void usra(const VRegister& vd, const VRegister& vn, int shift);
2165 
2166  // Signed rounding shift right by immediate and accumulate.
2167  void srsra(const VRegister& vd, const VRegister& vn, int shift);
2168 
2169  // Unsigned rounding shift right by immediate and accumulate.
2170  void ursra(const VRegister& vd, const VRegister& vn, int shift);
2171 
2172  // Shift right narrow by immediate.
2173  void shrn(const VRegister& vd, const VRegister& vn, int shift);
2174 
2175  // Shift right narrow by immediate (second part).
2176  void shrn2(const VRegister& vd, const VRegister& vn, int shift);
2177 
2178  // Rounding shift right narrow by immediate.
2179  void rshrn(const VRegister& vd, const VRegister& vn, int shift);
2180 
2181  // Rounding shift right narrow by immediate (second part).
2182  void rshrn2(const VRegister& vd, const VRegister& vn, int shift);
2183 
2184  // Unsigned saturating shift right narrow by immediate.
2185  void uqshrn(const VRegister& vd, const VRegister& vn, int shift);
2186 
2187  // Unsigned saturating shift right narrow by immediate (second part).
2188  void uqshrn2(const VRegister& vd, const VRegister& vn, int shift);
2189 
2190  // Unsigned saturating rounding shift right narrow by immediate.
2191  void uqrshrn(const VRegister& vd, const VRegister& vn, int shift);
2192 
2193  // Unsigned saturating rounding shift right narrow by immediate (second part).
2194  void uqrshrn2(const VRegister& vd, const VRegister& vn, int shift);
2195 
2196  // Signed saturating shift right narrow by immediate.
2197  void sqshrn(const VRegister& vd, const VRegister& vn, int shift);
2198 
2199  // Signed saturating shift right narrow by immediate (second part).
2200  void sqshrn2(const VRegister& vd, const VRegister& vn, int shift);
2201 
2202  // Signed saturating rounded shift right narrow by immediate.
2203  void sqrshrn(const VRegister& vd, const VRegister& vn, int shift);
2204 
2205  // Signed saturating rounded shift right narrow by immediate (second part).
2206  void sqrshrn2(const VRegister& vd, const VRegister& vn, int shift);
2207 
2208  // Signed saturating shift right unsigned narrow by immediate.
2209  void sqshrun(const VRegister& vd, const VRegister& vn, int shift);
2210 
2211  // Signed saturating shift right unsigned narrow by immediate (second part).
2212  void sqshrun2(const VRegister& vd, const VRegister& vn, int shift);
2213 
2214  // Signed sat rounded shift right unsigned narrow by immediate.
2215  void sqrshrun(const VRegister& vd, const VRegister& vn, int shift);
2216 
2217  // Signed sat rounded shift right unsigned narrow by immediate (second part).
2218  void sqrshrun2(const VRegister& vd, const VRegister& vn, int shift);
2219 
2220  // FP reciprocal step.
2221  void frecps(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2222 
2223  // FP reciprocal estimate.
2224  void frecpe(const VRegister& vd, const VRegister& vn);
2225 
2226  // FP reciprocal square root estimate.
2227  void frsqrte(const VRegister& vd, const VRegister& vn);
2228 
2229  // FP reciprocal square root step.
2230  void frsqrts(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2231 
2232  // Signed absolute difference and accumulate long.
2233  void sabal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2234 
2235  // Signed absolute difference and accumulate long (second part).
2236  void sabal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2237 
2238  // Unsigned absolute difference and accumulate long.
2239  void uabal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2240 
2241  // Unsigned absolute difference and accumulate long (second part).
2242  void uabal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2243 
2244  // Signed absolute difference long.
2245  void sabdl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2246 
2247  // Signed absolute difference long (second part).
2248  void sabdl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2249 
2250  // Unsigned absolute difference long.
2251  void uabdl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2252 
2253  // Unsigned absolute difference long (second part).
2254  void uabdl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2255 
2256  // Polynomial multiply long.
2257  void pmull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2258 
2259  // Polynomial multiply long (second part).
2260  void pmull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2261 
2262  // Signed long multiply-add.
2263  void smlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2264 
2265  // Signed long multiply-add (second part).
2266  void smlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2267 
2268  // Unsigned long multiply-add.
2269  void umlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2270 
2271  // Unsigned long multiply-add (second part).
2272  void umlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2273 
2274  // Signed long multiply-sub.
2275  void smlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2276 
2277  // Signed long multiply-sub (second part).
2278  void smlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2279 
2280  // Unsigned long multiply-sub.
2281  void umlsl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2282 
2283  // Unsigned long multiply-sub (second part).
2284  void umlsl2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2285 
2286  // Signed long multiply.
2287  void smull(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2288 
2289  // Signed long multiply (second part).
2290  void smull2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2291 
2292  // Signed saturating doubling long multiply-add.
2293  void sqdmlal(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2294 
2295  // Signed saturating doubling long multiply-add (second part).
2296  void sqdmlal2(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2297 
2298  // Unsigned absolute difference.
2299  void uabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2300 
2301  // Signed absolute difference and accumulate.
2302  void saba(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2303 
2304  // FP instructions.
2305  // Move immediate to FP register.
2306  void fmov(const VRegister& fd, double imm);
2307  void fmov(const VRegister& fd, float imm);
2308 
2309  // Move FP register to register.
2310  void fmov(const Register& rd, const VRegister& fn);
2311 
2312  // Move register to FP register.
2313  void fmov(const VRegister& fd, const Register& rn);
2314 
2315  // Move FP register to FP register.
2316  void fmov(const VRegister& fd, const VRegister& fn);
2317 
2318  // Move 64-bit register to top half of 128-bit FP register.
2319  void fmov(const VRegister& vd, int index, const Register& rn);
2320 
2321  // Move top half of 128-bit FP register to 64-bit register.
2322  void fmov(const Register& rd, const VRegister& vn, int index);
2323 
2324  // FP add.
2325  void fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2326 
2327  // FP subtract.
2328  void fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2329 
2330  // FP multiply.
2331  void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2332 
2333  // FP compare equal to zero.
2334  void fcmeq(const VRegister& vd, const VRegister& vn, double imm);
2335 
2336  // FP greater than zero.
2337  void fcmgt(const VRegister& vd, const VRegister& vn, double imm);
2338 
2339  // FP greater than or equal to zero.
2340  void fcmge(const VRegister& vd, const VRegister& vn, double imm);
2341 
2342  // FP less than or equal to zero.
2343  void fcmle(const VRegister& vd, const VRegister& vn, double imm);
2344 
2345  // FP less than to zero.
2346  void fcmlt(const VRegister& vd, const VRegister& vn, double imm);
2347 
2348  // FP absolute difference.
2349  void fabd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2350 
2351  // FP pairwise add vector.
2352  void faddp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2353 
2354  // FP pairwise add scalar.
2355  void faddp(const VRegister& vd, const VRegister& vn);
2356 
2357  // FP pairwise maximum scalar.
2358  void fmaxp(const VRegister& vd, const VRegister& vn);
2359 
2360  // FP pairwise maximum number scalar.
2361  void fmaxnmp(const VRegister& vd, const VRegister& vn);
2362 
2363  // FP pairwise minimum number scalar.
2364  void fminnmp(const VRegister& vd, const VRegister& vn);
2365 
2366  // FP vector multiply accumulate.
2367  void fmla(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2368 
2369  // FP vector multiply subtract.
2370  void fmls(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2371 
2372  // FP vector multiply extended.
2373  void fmulx(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2374 
2375  // FP absolute greater than or equal.
2376  void facge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2377 
2378  // FP absolute greater than.
2379  void facgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2380 
2381  // FP multiply by element.
2382  void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2383  int vm_index);
2384 
2385  // FP fused multiply-add to accumulator by element.
2386  void fmla(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2387  int vm_index);
2388 
2389  // FP fused multiply-sub from accumulator by element.
2390  void fmls(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2391  int vm_index);
2392 
2393  // FP multiply extended by element.
2394  void fmulx(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2395  int vm_index);
2396 
2397  // FP compare equal.
2398  void fcmeq(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2399 
2400  // FP greater than.
2401  void fcmgt(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2402 
2403  // FP greater than or equal.
2404  void fcmge(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2405 
2406  // FP pairwise maximum vector.
2407  void fmaxp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2408 
2409  // FP pairwise minimum vector.
2410  void fminp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2411 
2412  // FP pairwise minimum scalar.
2413  void fminp(const VRegister& vd, const VRegister& vn);
2414 
2415  // FP pairwise maximum number vector.
2416  void fmaxnmp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2417 
2418  // FP pairwise minimum number vector.
2419  void fminnmp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2420 
2421  // FP fused multiply-add.
2422  void fmadd(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2423  const VRegister& va);
2424 
2425  // FP fused multiply-subtract.
2426  void fmsub(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2427  const VRegister& va);
2428 
2429  // FP fused multiply-add and negate.
2430  void fnmadd(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2431  const VRegister& va);
2432 
2433  // FP fused multiply-subtract and negate.
2434  void fnmsub(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2435  const VRegister& va);
2436 
2437  // FP multiply-negate scalar.
2438  void fnmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2439 
2440  // FP reciprocal exponent scalar.
2441  void frecpx(const VRegister& vd, const VRegister& vn);
2442 
2443  // FP divide.
2444  void fdiv(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2445 
2446  // FP maximum.
2447  void fmax(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2448 
2449  // FP minimum.
2450  void fmin(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2451 
2452  // FP maximum.
2453  void fmaxnm(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2454 
2455  // FP minimum.
2456  void fminnm(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2457 
2458  // FP absolute.
2459  void fabs(const VRegister& vd, const VRegister& vn);
2460 
2461  // FP negate.
2462  void fneg(const VRegister& vd, const VRegister& vn);
2463 
2464  // FP square root.
2465  void fsqrt(const VRegister& vd, const VRegister& vn);
2466 
2467  // FP round to integer nearest with ties to away.
2468  void frinta(const VRegister& vd, const VRegister& vn);
2469 
2470  // FP round to integer, implicit rounding.
2471  void frinti(const VRegister& vd, const VRegister& vn);
2472 
2473  // FP round to integer toward minus infinity.
2474  void frintm(const VRegister& vd, const VRegister& vn);
2475 
2476  // FP round to integer nearest with ties to even.
2477  void frintn(const VRegister& vd, const VRegister& vn);
2478 
2479  // FP round to integer towards plus infinity.
2480  void frintp(const VRegister& vd, const VRegister& vn);
2481 
2482  // FP round to integer, exact, implicit rounding.
2483  void frintx(const VRegister& vd, const VRegister& vn);
2484 
2485  // FP round to integer towards zero.
2486  void frintz(const VRegister& vd, const VRegister& vn);
2487 
2488  // FP compare registers.
2489  void fcmp(const VRegister& vn, const VRegister& vm);
2490 
2491  // FP compare immediate.
2492  void fcmp(const VRegister& vn, double value);
2493 
2494  // FP conditional compare.
2495  void fccmp(const VRegister& vn, const VRegister& vm, StatusFlags nzcv,
2496  Condition cond);
2497 
2498  // FP conditional select.
2499  void fcsel(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2500  Condition cond);
2501 
2502  // Common FP Convert functions.
2503  void NEONFPConvertToInt(const Register& rd, const VRegister& vn, Instr op);
2504  void NEONFPConvertToInt(const VRegister& vd, const VRegister& vn, Instr op);
2505 
2506  // FP convert between precisions.
2507  void fcvt(const VRegister& vd, const VRegister& vn);
2508 
2509  // FP convert to higher precision.
2510  void fcvtl(const VRegister& vd, const VRegister& vn);
2511 
2512  // FP convert to higher precision (second part).
2513  void fcvtl2(const VRegister& vd, const VRegister& vn);
2514 
2515  // FP convert to lower precision.
2516  void fcvtn(const VRegister& vd, const VRegister& vn);
2517 
2518  // FP convert to lower prevision (second part).
2519  void fcvtn2(const VRegister& vd, const VRegister& vn);
2520 
2521  // FP convert to lower precision, rounding to odd.
2522  void fcvtxn(const VRegister& vd, const VRegister& vn);
2523 
2524  // FP convert to lower precision, rounding to odd (second part).
2525  void fcvtxn2(const VRegister& vd, const VRegister& vn);
2526 
2527  // FP convert to signed integer, nearest with ties to away.
2528  void fcvtas(const Register& rd, const VRegister& vn);
2529 
2530  // FP convert to unsigned integer, nearest with ties to away.
2531  void fcvtau(const Register& rd, const VRegister& vn);
2532 
2533  // FP convert to signed integer, nearest with ties to away.
2534  void fcvtas(const VRegister& vd, const VRegister& vn);
2535 
2536  // FP convert to unsigned integer, nearest with ties to away.
2537  void fcvtau(const VRegister& vd, const VRegister& vn);
2538 
2539  // FP convert to signed integer, round towards -infinity.
2540  void fcvtms(const Register& rd, const VRegister& vn);
2541 
2542  // FP convert to unsigned integer, round towards -infinity.
2543  void fcvtmu(const Register& rd, const VRegister& vn);
2544 
2545  // FP convert to signed integer, round towards -infinity.
2546  void fcvtms(const VRegister& vd, const VRegister& vn);
2547 
2548  // FP convert to unsigned integer, round towards -infinity.
2549  void fcvtmu(const VRegister& vd, const VRegister& vn);
2550 
2551  // FP convert to signed integer, nearest with ties to even.
2552  void fcvtns(const Register& rd, const VRegister& vn);
2553 
2554  // FP convert to unsigned integer, nearest with ties to even.
2555  void fcvtnu(const Register& rd, const VRegister& vn);
2556 
2557  // FP convert to signed integer, nearest with ties to even.
2558  void fcvtns(const VRegister& rd, const VRegister& vn);
2559 
2560  // FP convert to unsigned integer, nearest with ties to even.
2561  void fcvtnu(const VRegister& rd, const VRegister& vn);
2562 
2563  // FP convert to signed integer or fixed-point, round towards zero.
2564  void fcvtzs(const Register& rd, const VRegister& vn, int fbits = 0);
2565 
2566  // FP convert to unsigned integer or fixed-point, round towards zero.
2567  void fcvtzu(const Register& rd, const VRegister& vn, int fbits = 0);
2568 
2569  // FP convert to signed integer or fixed-point, round towards zero.
2570  void fcvtzs(const VRegister& vd, const VRegister& vn, int fbits = 0);
2571 
2572  // FP convert to unsigned integer or fixed-point, round towards zero.
2573  void fcvtzu(const VRegister& vd, const VRegister& vn, int fbits = 0);
2574 
2575  // FP convert to signed integer, round towards +infinity.
2576  void fcvtps(const Register& rd, const VRegister& vn);
2577 
2578  // FP convert to unsigned integer, round towards +infinity.
2579  void fcvtpu(const Register& rd, const VRegister& vn);
2580 
2581  // FP convert to signed integer, round towards +infinity.
2582  void fcvtps(const VRegister& vd, const VRegister& vn);
2583 
2584  // FP convert to unsigned integer, round towards +infinity.
2585  void fcvtpu(const VRegister& vd, const VRegister& vn);
2586 
2587  // Convert signed integer or fixed point to FP.
2588  void scvtf(const VRegister& fd, const Register& rn, int fbits = 0);
2589 
2590  // Convert unsigned integer or fixed point to FP.
2591  void ucvtf(const VRegister& fd, const Register& rn, int fbits = 0);
2592 
2593  // Convert signed integer or fixed-point to FP.
2594  void scvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
2595 
2596  // Convert unsigned integer or fixed-point to FP.
2597  void ucvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
2598 
2599  // Extract vector from pair of vectors.
2600  void ext(const VRegister& vd, const VRegister& vn, const VRegister& vm,
2601  int index);
2602 
2603  // Duplicate vector element to vector or scalar.
2604  void dup(const VRegister& vd, const VRegister& vn, int vn_index);
2605 
2606  // Duplicate general-purpose register to vector.
2607  void dup(const VRegister& vd, const Register& rn);
2608 
2609  // Insert vector element from general-purpose register.
2610  void ins(const VRegister& vd, int vd_index, const Register& rn);
2611 
2612  // Move general-purpose register to a vector element.
2613  void mov(const VRegister& vd, int vd_index, const Register& rn);
2614 
2615  // Unsigned move vector element to general-purpose register.
2616  void umov(const Register& rd, const VRegister& vn, int vn_index);
2617 
2618  // Move vector element to general-purpose register.
2619  void mov(const Register& rd, const VRegister& vn, int vn_index);
2620 
2621  // Move vector element to scalar.
2622  void mov(const VRegister& vd, const VRegister& vn, int vn_index);
2623 
2624  // Insert vector element from another vector element.
2625  void ins(const VRegister& vd, int vd_index, const VRegister& vn,
2626  int vn_index);
2627 
2628  // Move vector element to another vector element.
2629  void mov(const VRegister& vd, int vd_index, const VRegister& vn,
2630  int vn_index);
2631 
2632  // Signed move vector element to general-purpose register.
2633  void smov(const Register& rd, const VRegister& vn, int vn_index);
2634 
2635  // One-element structure load to one register.
2636  void ld1(const VRegister& vt, const MemOperand& src);
2637 
2638  // One-element structure load to two registers.
2639  void ld1(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
2640 
2641  // One-element structure load to three registers.
2642  void ld1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2643  const MemOperand& src);
2644 
2645  // One-element structure load to four registers.
2646  void ld1(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2647  const VRegister& vt4, const MemOperand& src);
2648 
2649  // One-element single structure load to one lane.
2650  void ld1(const VRegister& vt, int lane, const MemOperand& src);
2651 
2652  // One-element single structure load to all lanes.
2653  void ld1r(const VRegister& vt, const MemOperand& src);
2654 
2655  // Two-element structure load.
2656  void ld2(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
2657 
2658  // Two-element single structure load to one lane.
2659  void ld2(const VRegister& vt, const VRegister& vt2, int lane,
2660  const MemOperand& src);
2661 
2662  // Two-element single structure load to all lanes.
2663  void ld2r(const VRegister& vt, const VRegister& vt2, const MemOperand& src);
2664 
2665  // Three-element structure load.
2666  void ld3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2667  const MemOperand& src);
2668 
2669  // Three-element single structure load to one lane.
2670  void ld3(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2671  int lane, const MemOperand& src);
2672 
2673  // Three-element single structure load to all lanes.
2674  void ld3r(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2675  const MemOperand& src);
2676 
2677  // Four-element structure load.
2678  void ld4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2679  const VRegister& vt4, const MemOperand& src);
2680 
2681  // Four-element single structure load to one lane.
2682  void ld4(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2683  const VRegister& vt4, int lane, const MemOperand& src);
2684 
2685  // Four-element single structure load to all lanes.
2686  void ld4r(const VRegister& vt, const VRegister& vt2, const VRegister& vt3,
2687  const VRegister& vt4, const MemOperand& src);
2688 
2689  // Count leading sign bits.
2690  void cls(const VRegister& vd, const VRegister& vn);
2691 
2692  // Count leading zero bits (vector).
2693  void clz(const VRegister& vd, const VRegister& vn);
2694 
2695  // Population count per byte.
2696  void cnt(const VRegister& vd, const VRegister& vn);
2697 
2698  // Reverse bit order.
2699  void rbit(const VRegister& vd, const VRegister& vn);
2700 
2701  // Reverse elements in 16-bit halfwords.
2702  void rev16(const VRegister& vd, const VRegister& vn);
2703 
2704  // Reverse elements in 32-bit words.
2705  void rev32(const VRegister& vd, const VRegister& vn);
2706 
2707  // Reverse elements in 64-bit doublewords.
2708  void rev64(const VRegister& vd, const VRegister& vn);
2709 
2710  // Unsigned reciprocal square root estimate.
2711  void ursqrte(const VRegister& vd, const VRegister& vn);
2712 
2713  // Unsigned reciprocal estimate.
2714  void urecpe(const VRegister& vd, const VRegister& vn);
2715 
2716  // Signed pairwise long add and accumulate.
2717  void sadalp(const VRegister& vd, const VRegister& vn);
2718 
2719  // Signed pairwise long add.
2720  void saddlp(const VRegister& vd, const VRegister& vn);
2721 
2722  // Unsigned pairwise long add.
2723  void uaddlp(const VRegister& vd, const VRegister& vn);
2724 
2725  // Unsigned pairwise long add and accumulate.
2726  void uadalp(const VRegister& vd, const VRegister& vn);
2727 
2728  // Shift left by immediate.
2729  void shl(const VRegister& vd, const VRegister& vn, int shift);
2730 
2731  // Signed saturating shift left by immediate.
2732  void sqshl(const VRegister& vd, const VRegister& vn, int shift);
2733 
2734  // Signed saturating shift left unsigned by immediate.
2735  void sqshlu(const VRegister& vd, const VRegister& vn, int shift);
2736 
2737  // Unsigned saturating shift left by immediate.
2738  void uqshl(const VRegister& vd, const VRegister& vn, int shift);
2739 
2740  // Signed shift left long by immediate.
2741  void sshll(const VRegister& vd, const VRegister& vn, int shift);
2742 
2743  // Signed shift left long by immediate (second part).
2744  void sshll2(const VRegister& vd, const VRegister& vn, int shift);
2745 
2746  // Signed extend long.
2747  void sxtl(const VRegister& vd, const VRegister& vn);
2748 
2749  // Signed extend long (second part).
2750  void sxtl2(const VRegister& vd, const VRegister& vn);
2751 
2752  // Unsigned shift left long by immediate.
2753  void ushll(const VRegister& vd, const VRegister& vn, int shift);
2754 
2755  // Unsigned shift left long by immediate (second part).
2756  void ushll2(const VRegister& vd, const VRegister& vn, int shift);
2757 
2758  // Shift left long by element size.
2759  void shll(const VRegister& vd, const VRegister& vn, int shift);
2760 
2761  // Shift left long by element size (second part).
2762  void shll2(const VRegister& vd, const VRegister& vn, int shift);
2763 
2764  // Unsigned extend long.
2765  void uxtl(const VRegister& vd, const VRegister& vn);
2766 
2767  // Unsigned extend long (second part).
2768  void uxtl2(const VRegister& vd, const VRegister& vn);
2769 
2770  // Signed rounding halving add.
2771  void srhadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2772 
2773  // Unsigned halving sub.
2774  void uhsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2775 
2776  // Signed halving sub.
2777  void shsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2778 
2779  // Unsigned saturating add.
2780  void uqadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2781 
2782  // Signed saturating add.
2783  void sqadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2784 
2785  // Unsigned saturating subtract.
2786  void uqsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2787 
2788  // Signed saturating subtract.
2789  void sqsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2790 
2791  // Add pairwise.
2792  void addp(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2793 
2794  // Add pair of elements scalar.
2795  void addp(const VRegister& vd, const VRegister& vn);
2796 
2797  // Multiply-add to accumulator.
2798  void mla(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2799 
2800  // Multiply-subtract to accumulator.
2801  void mls(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2802 
2803  // Multiply.
2804  void mul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2805 
2806  // Table lookup from one register.
2807  void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2808 
2809  // Table lookup from two registers.
2810  void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
2811  const VRegister& vm);
2812 
2813  // Table lookup from three registers.
2814  void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
2815  const VRegister& vn3, const VRegister& vm);
2816 
2817  // Table lookup from four registers.
2818  void tbl(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
2819  const VRegister& vn3, const VRegister& vn4, const VRegister& vm);
2820 
2821  // Table lookup extension from one register.
2822  void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vm);
2823 
2824  // Table lookup extension from two registers.
2825  void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
2826  const VRegister& vm);
2827 
2828  // Table lookup extension from three registers.
2829  void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
2830  const VRegister& vn3, const VRegister& vm);
2831 
2832  // Table lookup extension from four registers.
2833  void tbx(const VRegister& vd, const VRegister& vn, const VRegister& vn2,
2834  const VRegister& vn3, const VRegister& vn4, const VRegister& vm);
2835 
2836  // Instruction functions used only for test, debug, and patching.
2837  // Emit raw instructions in the instruction stream.
2838  void dci(Instr raw_inst) { Emit(raw_inst); }
2839 
2840  // Emit 8 bits of data in the instruction stream.
2841  void dc8(uint8_t data) { EmitData(&data, sizeof(data)); }
2842 
2843  // Emit 32 bits of data in the instruction stream.
2844  void dc32(uint32_t data) { EmitData(&data, sizeof(data)); }
2845 
2846  // Emit 64 bits of data in the instruction stream.
2847  void dc64(uint64_t data) { EmitData(&data, sizeof(data)); }
2848 
2849  // Emit an address in the instruction stream.
2850  void dcptr(Label* label);
2851 
2852  // Copy a string into the instruction stream, including the terminating
2853  // nullptr character. The instruction pointer (pc_) is then aligned correctly
2854  // for subsequent instructions.
2855  void EmitStringData(const char* string);
2856 
2857  // Pseudo-instructions ------------------------------------------------------
2858 
2859  // Parameters are described in arm64/instructions-arm64.h.
2860  void debug(const char* message, uint32_t code, Instr params = BREAK);
2861 
2862  // Required by V8.
2863  void dd(uint32_t data) { dc32(data); }
2864  void db(uint8_t data) { dc8(data); }
2865  void dq(uint64_t data) { dc64(data); }
2866  void dp(uintptr_t data) { dc64(data); }
2867 
2868  // Code generation helpers --------------------------------------------------
2869 
2870  bool IsConstPoolEmpty() const { return constpool_.IsEmpty(); }
2871 
2872  Instruction* pc() const { return Instruction::Cast(pc_); }
2873 
2874  Instruction* InstructionAt(ptrdiff_t offset) const {
2875  return reinterpret_cast<Instruction*>(buffer_ + offset);
2876  }
2877 
2878  ptrdiff_t InstructionOffset(Instruction* instr) const {
2879  return reinterpret_cast<byte*>(instr) - buffer_;
2880  }
2881 
2882  static const char* GetSpecialRegisterName(int code) {
2883  return (code == kSPRegInternalCode) ? "sp" : "UNKNOWN";
2884  }
2885 
2886  // Register encoding.
2887  static Instr Rd(CPURegister rd) {
2888  DCHECK_NE(rd.code(), kSPRegInternalCode);
2889  return rd.code() << Rd_offset;
2890  }
2891 
2892  static Instr Rn(CPURegister rn) {
2893  DCHECK_NE(rn.code(), kSPRegInternalCode);
2894  return rn.code() << Rn_offset;
2895  }
2896 
2897  static Instr Rm(CPURegister rm) {
2898  DCHECK_NE(rm.code(), kSPRegInternalCode);
2899  return rm.code() << Rm_offset;
2900  }
2901 
2902  static Instr RmNot31(CPURegister rm) {
2903  DCHECK_NE(rm.code(), kSPRegInternalCode);
2904  DCHECK(!rm.IsZero());
2905  return Rm(rm);
2906  }
2907 
2908  static Instr Ra(CPURegister ra) {
2909  DCHECK_NE(ra.code(), kSPRegInternalCode);
2910  return ra.code() << Ra_offset;
2911  }
2912 
2913  static Instr Rt(CPURegister rt) {
2914  DCHECK_NE(rt.code(), kSPRegInternalCode);
2915  return rt.code() << Rt_offset;
2916  }
2917 
2918  static Instr Rt2(CPURegister rt2) {
2919  DCHECK_NE(rt2.code(), kSPRegInternalCode);
2920  return rt2.code() << Rt2_offset;
2921  }
2922 
2923  static Instr Rs(CPURegister rs) {
2924  DCHECK_NE(rs.code(), kSPRegInternalCode);
2925  return rs.code() << Rs_offset;
2926  }
2927 
2928  // These encoding functions allow the stack pointer to be encoded, and
2929  // disallow the zero register.
2930  static Instr RdSP(Register rd) {
2931  DCHECK(!rd.IsZero());
2932  return (rd.code() & kRegCodeMask) << Rd_offset;
2933  }
2934 
2935  static Instr RnSP(Register rn) {
2936  DCHECK(!rn.IsZero());
2937  return (rn.code() & kRegCodeMask) << Rn_offset;
2938  }
2939 
2940  // Flags encoding.
2941  inline static Instr Flags(FlagsUpdate S);
2942  inline static Instr Cond(Condition cond);
2943 
2944  // PC-relative address encoding.
2945  inline static Instr ImmPCRelAddress(int imm21);
2946 
2947  // Branch encoding.
2948  inline static Instr ImmUncondBranch(int imm26);
2949  inline static Instr ImmCondBranch(int imm19);
2950  inline static Instr ImmCmpBranch(int imm19);
2951  inline static Instr ImmTestBranch(int imm14);
2952  inline static Instr ImmTestBranchBit(unsigned bit_pos);
2953 
2954  // Data Processing encoding.
2955  inline static Instr SF(Register rd);
2956  inline static Instr ImmAddSub(int imm);
2957  inline static Instr ImmS(unsigned imms, unsigned reg_size);
2958  inline static Instr ImmR(unsigned immr, unsigned reg_size);
2959  inline static Instr ImmSetBits(unsigned imms, unsigned reg_size);
2960  inline static Instr ImmRotate(unsigned immr, unsigned reg_size);
2961  inline static Instr ImmLLiteral(int imm19);
2962  inline static Instr BitN(unsigned bitn, unsigned reg_size);
2963  inline static Instr ShiftDP(Shift shift);
2964  inline static Instr ImmDPShift(unsigned amount);
2965  inline static Instr ExtendMode(Extend extend);
2966  inline static Instr ImmExtendShift(unsigned left_shift);
2967  inline static Instr ImmCondCmp(unsigned imm);
2968  inline static Instr Nzcv(StatusFlags nzcv);
2969 
2970  static bool IsImmAddSub(int64_t immediate);
2971  static bool IsImmLogical(uint64_t value,
2972  unsigned width,
2973  unsigned* n,
2974  unsigned* imm_s,
2975  unsigned* imm_r);
2976 
2977  // MemOperand offset encoding.
2978  inline static Instr ImmLSUnsigned(int imm12);
2979  inline static Instr ImmLS(int imm9);
2980  inline static Instr ImmLSPair(int imm7, unsigned size);
2981  inline static Instr ImmShiftLS(unsigned shift_amount);
2982  inline static Instr ImmException(int imm16);
2983  inline static Instr ImmSystemRegister(int imm15);
2984  inline static Instr ImmHint(int imm7);
2985  inline static Instr ImmBarrierDomain(int imm2);
2986  inline static Instr ImmBarrierType(int imm2);
2987  inline static unsigned CalcLSDataSize(LoadStoreOp op);
2988 
2989  // Instruction bits for vector format in data processing operations.
2990  static Instr VFormat(VRegister vd) {
2991  if (vd.Is64Bits()) {
2992  switch (vd.LaneCount()) {
2993  case 2:
2994  return NEON_2S;
2995  case 4:
2996  return NEON_4H;
2997  case 8:
2998  return NEON_8B;
2999  default:
3000  UNREACHABLE();
3001  }
3002  } else {
3003  DCHECK(vd.Is128Bits());
3004  switch (vd.LaneCount()) {
3005  case 2:
3006  return NEON_2D;
3007  case 4:
3008  return NEON_4S;
3009  case 8:
3010  return NEON_8H;
3011  case 16:
3012  return NEON_16B;
3013  default:
3014  UNREACHABLE();
3015  }
3016  }
3017  }
3018 
3019  // Instruction bits for vector format in floating point data processing
3020  // operations.
3021  static Instr FPFormat(VRegister vd) {
3022  if (vd.LaneCount() == 1) {
3023  // Floating point scalar formats.
3024  DCHECK(vd.Is32Bits() || vd.Is64Bits());
3025  return vd.Is64Bits() ? FP64 : FP32;
3026  }
3027 
3028  // Two lane floating point vector formats.
3029  if (vd.LaneCount() == 2) {
3030  DCHECK(vd.Is64Bits() || vd.Is128Bits());
3031  return vd.Is128Bits() ? NEON_FP_2D : NEON_FP_2S;
3032  }
3033 
3034  // Four lane floating point vector format.
3035  DCHECK((vd.LaneCount() == 4) && vd.Is128Bits());
3036  return NEON_FP_4S;
3037  }
3038 
3039  // Instruction bits for vector format in load and store operations.
3040  static Instr LSVFormat(VRegister vd) {
3041  if (vd.Is64Bits()) {
3042  switch (vd.LaneCount()) {
3043  case 1:
3044  return LS_NEON_1D;
3045  case 2:
3046  return LS_NEON_2S;
3047  case 4:
3048  return LS_NEON_4H;
3049  case 8:
3050  return LS_NEON_8B;
3051  default:
3052  UNREACHABLE();
3053  }
3054  } else {
3055  DCHECK(vd.Is128Bits());
3056  switch (vd.LaneCount()) {
3057  case 2:
3058  return LS_NEON_2D;
3059  case 4:
3060  return LS_NEON_4S;
3061  case 8:
3062  return LS_NEON_8H;
3063  case 16:
3064  return LS_NEON_16B;
3065  default:
3066  UNREACHABLE();
3067  }
3068  }
3069  }
3070 
3071  // Instruction bits for scalar format in data processing operations.
3072  static Instr SFormat(VRegister vd) {
3073  DCHECK(vd.IsScalar());
3074  switch (vd.SizeInBytes()) {
3075  case 1:
3076  return NEON_B;
3077  case 2:
3078  return NEON_H;
3079  case 4:
3080  return NEON_S;
3081  case 8:
3082  return NEON_D;
3083  default:
3084  UNREACHABLE();
3085  }
3086  }
3087 
3088  static Instr ImmNEONHLM(int index, int num_bits) {
3089  int h, l, m;
3090  if (num_bits == 3) {
3091  DCHECK(is_uint3(index));
3092  h = (index >> 2) & 1;
3093  l = (index >> 1) & 1;
3094  m = (index >> 0) & 1;
3095  } else if (num_bits == 2) {
3096  DCHECK(is_uint2(index));
3097  h = (index >> 1) & 1;
3098  l = (index >> 0) & 1;
3099  m = 0;
3100  } else {
3101  DCHECK(is_uint1(index) && (num_bits == 1));
3102  h = (index >> 0) & 1;
3103  l = 0;
3104  m = 0;
3105  }
3106  return (h << NEONH_offset) | (l << NEONL_offset) | (m << NEONM_offset);
3107  }
3108 
3109  static Instr ImmNEONExt(int imm4) {
3110  DCHECK(is_uint4(imm4));
3111  return imm4 << ImmNEONExt_offset;
3112  }
3113 
3114  static Instr ImmNEON5(Instr format, int index) {
3115  DCHECK(is_uint4(index));
3116  int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
3117  int imm5 = (index << (s + 1)) | (1 << s);
3118  return imm5 << ImmNEON5_offset;
3119  }
3120 
3121  static Instr ImmNEON4(Instr format, int index) {
3122  DCHECK(is_uint4(index));
3123  int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
3124  int imm4 = index << s;
3125  return imm4 << ImmNEON4_offset;
3126  }
3127 
3128  static Instr ImmNEONabcdefgh(int imm8) {
3129  DCHECK(is_uint8(imm8));
3130  Instr instr;
3131  instr = ((imm8 >> 5) & 7) << ImmNEONabc_offset;
3132  instr |= (imm8 & 0x1f) << ImmNEONdefgh_offset;
3133  return instr;
3134  }
3135 
3136  static Instr NEONCmode(int cmode) {
3137  DCHECK(is_uint4(cmode));
3138  return cmode << NEONCmode_offset;
3139  }
3140 
3141  static Instr NEONModImmOp(int op) {
3142  DCHECK(is_uint1(op));
3143  return op << NEONModImmOp_offset;
3144  }
3145 
3146  static bool IsImmLSUnscaled(int64_t offset);
3147  static bool IsImmLSScaled(int64_t offset, unsigned size);
3148  static bool IsImmLLiteral(int64_t offset);
3149 
3150  // Move immediates encoding.
3151  inline static Instr ImmMoveWide(int imm);
3152  inline static Instr ShiftMoveWide(int shift);
3153 
3154  // FP Immediates.
3155  static Instr ImmFP(double imm);
3156  static Instr ImmNEONFP(double imm);
3157  inline static Instr FPScale(unsigned scale);
3158 
3159  // FP register type.
3160  inline static Instr FPType(VRegister fd);
3161 
3162  // Class for scoping postponing the constant pool generation.
3163  class BlockConstPoolScope {
3164  public:
3165  explicit BlockConstPoolScope(Assembler* assem) : assem_(assem) {
3166  assem_->StartBlockConstPool();
3167  }
3168  ~BlockConstPoolScope() {
3169  assem_->EndBlockConstPool();
3170  }
3171 
3172  private:
3173  Assembler* assem_;
3174 
3175  DISALLOW_IMPLICIT_CONSTRUCTORS(BlockConstPoolScope);
3176  };
3177 
3178  // Check if is time to emit a constant pool.
3179  void CheckConstPool(bool force_emit, bool require_jump);
3180 
3181  // Returns true if we should emit a veneer as soon as possible for a branch
3182  // which can at most reach to specified pc.
3183  bool ShouldEmitVeneer(int max_reachable_pc,
3184  int margin = kVeneerDistanceMargin);
3185  bool ShouldEmitVeneers(int margin = kVeneerDistanceMargin) {
3186  return ShouldEmitVeneer(unresolved_branches_first_limit(), margin);
3187  }
3188 
3189  // The maximum code size generated for a veneer. Currently one branch
3190  // instruction. This is for code size checking purposes, and can be extended
3191  // in the future for example if we decide to add nops between the veneers.
3192  static constexpr int kMaxVeneerCodeSize = 1 * kInstrSize;
3193 
3194  void RecordVeneerPool(int location_offset, int size);
3195  // Emits veneers for branches that are approaching their maximum range.
3196  // If need_protection is true, the veneers are protected by a branch jumping
3197  // over the code.
3198  void EmitVeneers(bool force_emit, bool need_protection,
3199  int margin = kVeneerDistanceMargin);
3200  void EmitVeneersGuard() { EmitPoolGuard(); }
3201  // Checks whether veneers need to be emitted at this point.
3202  // If force_emit is set, a veneer is generated for *all* unresolved branches.
3203  void CheckVeneerPool(bool force_emit, bool require_jump,
3204  int margin = kVeneerDistanceMargin);
3205 
3207  public:
3208  explicit BlockPoolsScope(Assembler* assem) : assem_(assem) {
3209  assem_->StartBlockPools();
3210  }
3211  ~BlockPoolsScope() {
3212  assem_->EndBlockPools();
3213  }
3214 
3215  private:
3216  Assembler* assem_;
3217 
3218  DISALLOW_IMPLICIT_CONSTRUCTORS(BlockPoolsScope);
3219  };
3220 
3221  protected:
3222  inline const Register& AppropriateZeroRegFor(const CPURegister& reg) const;
3223 
3224  void LoadStore(const CPURegister& rt,
3225  const MemOperand& addr,
3226  LoadStoreOp op);
3227  void LoadStorePair(const CPURegister& rt, const CPURegister& rt2,
3228  const MemOperand& addr, LoadStorePairOp op);
3229  void LoadStoreStruct(const VRegister& vt, const MemOperand& addr,
3230  NEONLoadStoreMultiStructOp op);
3231  void LoadStoreStruct1(const VRegister& vt, int reg_count,
3232  const MemOperand& addr);
3233  void LoadStoreStructSingle(const VRegister& vt, uint32_t lane,
3234  const MemOperand& addr,
3235  NEONLoadStoreSingleStructOp op);
3236  void LoadStoreStructSingleAllLanes(const VRegister& vt,
3237  const MemOperand& addr,
3238  NEONLoadStoreSingleStructOp op);
3239  void LoadStoreStructVerify(const VRegister& vt, const MemOperand& addr,
3240  Instr op);
3241 
3242  static bool IsImmLSPair(int64_t offset, unsigned size);
3243 
3244  void Logical(const Register& rd,
3245  const Register& rn,
3246  const Operand& operand,
3247  LogicalOp op);
3248  void LogicalImmediate(const Register& rd,
3249  const Register& rn,
3250  unsigned n,
3251  unsigned imm_s,
3252  unsigned imm_r,
3253  LogicalOp op);
3254 
3255  void ConditionalCompare(const Register& rn,
3256  const Operand& operand,
3257  StatusFlags nzcv,
3258  Condition cond,
3259  ConditionalCompareOp op);
3260  static bool IsImmConditionalCompare(int64_t immediate);
3261 
3262  void AddSubWithCarry(const Register& rd,
3263  const Register& rn,
3264  const Operand& operand,
3265  FlagsUpdate S,
3266  AddSubWithCarryOp op);
3267 
3268  // Functions for emulating operands not directly supported by the instruction
3269  // set.
3270  void EmitShift(const Register& rd,
3271  const Register& rn,
3272  Shift shift,
3273  unsigned amount);
3274  void EmitExtendShift(const Register& rd,
3275  const Register& rn,
3276  Extend extend,
3277  unsigned left_shift);
3278 
3279  void AddSub(const Register& rd,
3280  const Register& rn,
3281  const Operand& operand,
3282  FlagsUpdate S,
3283  AddSubOp op);
3284 
3285  static bool IsImmFP32(float imm);
3286  static bool IsImmFP64(double imm);
3287 
3288  // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
3289  // registers. Only simple loads are supported; sign- and zero-extension (such
3290  // as in LDPSW_x or LDRB_w) are not supported.
3291  static inline LoadStoreOp LoadOpFor(const CPURegister& rt);
3292  static inline LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
3293  const CPURegister& rt2);
3294  static inline LoadStoreOp StoreOpFor(const CPURegister& rt);
3295  static inline LoadStorePairOp StorePairOpFor(const CPURegister& rt,
3296  const CPURegister& rt2);
3297  static inline LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
3298 
3299  // Remove the specified branch from the unbound label link chain.
3300  // If available, a veneer for this label can be used for other branches in the
3301  // chain if the link chain cannot be fixed up without this branch.
3302  void RemoveBranchFromLabelLinkChain(Instruction* branch, Label* label,
3303  Instruction* label_veneer = nullptr);
3304 
3305  private:
3306  static uint32_t FPToImm8(double imm);
3307 
3308  // Instruction helpers.
3309  void MoveWide(const Register& rd,
3310  uint64_t imm,
3311  int shift,
3312  MoveWideImmediateOp mov_op);
3313  void DataProcShiftedRegister(const Register& rd,
3314  const Register& rn,
3315  const Operand& operand,
3316  FlagsUpdate S,
3317  Instr op);
3318  void DataProcExtendedRegister(const Register& rd,
3319  const Register& rn,
3320  const Operand& operand,
3321  FlagsUpdate S,
3322  Instr op);
3323  void ConditionalSelect(const Register& rd,
3324  const Register& rn,
3325  const Register& rm,
3326  Condition cond,
3327  ConditionalSelectOp op);
3328  void DataProcessing1Source(const Register& rd,
3329  const Register& rn,
3330  DataProcessing1SourceOp op);
3331  void DataProcessing3Source(const Register& rd,
3332  const Register& rn,
3333  const Register& rm,
3334  const Register& ra,
3335  DataProcessing3SourceOp op);
3336  void FPDataProcessing1Source(const VRegister& fd, const VRegister& fn,
3337  FPDataProcessing1SourceOp op);
3338  void FPDataProcessing2Source(const VRegister& fd, const VRegister& fn,
3339  const VRegister& fm,
3340  FPDataProcessing2SourceOp op);
3341  void FPDataProcessing3Source(const VRegister& fd, const VRegister& fn,
3342  const VRegister& fm, const VRegister& fa,
3343  FPDataProcessing3SourceOp op);
3344  void NEONAcrossLanesL(const VRegister& vd, const VRegister& vn,
3345  NEONAcrossLanesOp op);
3346  void NEONAcrossLanes(const VRegister& vd, const VRegister& vn,
3347  NEONAcrossLanesOp op);
3348  void NEONModifiedImmShiftLsl(const VRegister& vd, const int imm8,
3349  const int left_shift,
3350  NEONModifiedImmediateOp op);
3351  void NEONModifiedImmShiftMsl(const VRegister& vd, const int imm8,
3352  const int shift_amount,
3353  NEONModifiedImmediateOp op);
3354  void NEON3Same(const VRegister& vd, const VRegister& vn, const VRegister& vm,
3355  NEON3SameOp vop);
3356  void NEONFP3Same(const VRegister& vd, const VRegister& vn,
3357  const VRegister& vm, Instr op);
3358  void NEON3DifferentL(const VRegister& vd, const VRegister& vn,
3359  const VRegister& vm, NEON3DifferentOp vop);
3360  void NEON3DifferentW(const VRegister& vd, const VRegister& vn,
3361  const VRegister& vm, NEON3DifferentOp vop);
3362  void NEON3DifferentHN(const VRegister& vd, const VRegister& vn,
3363  const VRegister& vm, NEON3DifferentOp vop);
3364  void NEONFP2RegMisc(const VRegister& vd, const VRegister& vn,
3365  NEON2RegMiscOp vop, double value = 0.0);
3366  void NEON2RegMisc(const VRegister& vd, const VRegister& vn,
3367  NEON2RegMiscOp vop, int value = 0);
3368  void NEONFP2RegMisc(const VRegister& vd, const VRegister& vn, Instr op);
3369  void NEONAddlp(const VRegister& vd, const VRegister& vn, NEON2RegMiscOp op);
3370  void NEONPerm(const VRegister& vd, const VRegister& vn, const VRegister& vm,
3371  NEONPermOp op);
3372  void NEONFPByElement(const VRegister& vd, const VRegister& vn,
3373  const VRegister& vm, int vm_index,
3374  NEONByIndexedElementOp op);
3375  void NEONByElement(const VRegister& vd, const VRegister& vn,
3376  const VRegister& vm, int vm_index,
3377  NEONByIndexedElementOp op);
3378  void NEONByElementL(const VRegister& vd, const VRegister& vn,
3379  const VRegister& vm, int vm_index,
3380  NEONByIndexedElementOp op);
3381  void NEONShiftImmediate(const VRegister& vd, const VRegister& vn,
3382  NEONShiftImmediateOp op, int immh_immb);
3383  void NEONShiftLeftImmediate(const VRegister& vd, const VRegister& vn,
3384  int shift, NEONShiftImmediateOp op);
3385  void NEONShiftRightImmediate(const VRegister& vd, const VRegister& vn,
3386  int shift, NEONShiftImmediateOp op);
3387  void NEONShiftImmediateL(const VRegister& vd, const VRegister& vn, int shift,
3388  NEONShiftImmediateOp op);
3389  void NEONShiftImmediateN(const VRegister& vd, const VRegister& vn, int shift,
3390  NEONShiftImmediateOp op);
3391  void NEONXtn(const VRegister& vd, const VRegister& vn, NEON2RegMiscOp vop);
3392  void NEONTable(const VRegister& vd, const VRegister& vn, const VRegister& vm,
3393  NEONTableOp op);
3394 
3395  Instr LoadStoreStructAddrModeField(const MemOperand& addr);
3396 
3397  // Label helpers.
3398 
3399  // Return an offset for a label-referencing instruction, typically a branch.
3400  int LinkAndGetByteOffsetTo(Label* label);
3401 
3402  // This is the same as LinkAndGetByteOffsetTo, but return an offset
3403  // suitable for fields that take instruction offsets.
3404  inline int LinkAndGetInstructionOffsetTo(Label* label);
3405 
3406  static constexpr int kStartOfLabelLinkChain = 0;
3407 
3408  // Verify that a label's link chain is intact.
3409  void CheckLabelLinkChain(Label const * label);
3410 
3411  // Postpone the generation of the constant pool for the specified number of
3412  // instructions.
3413  void BlockConstPoolFor(int instructions);
3414 
3415  // Set how far from current pc the next constant pool check will be.
3416  void SetNextConstPoolCheckIn(int instructions) {
3417  next_constant_pool_check_ = pc_offset() + instructions * kInstrSize;
3418  }
3419 
3420  // Emit the instruction at pc_.
3421  void Emit(Instr instruction) {
3422  STATIC_ASSERT(sizeof(*pc_) == 1);
3423  STATIC_ASSERT(sizeof(instruction) == kInstrSize);
3424  DCHECK((pc_ + sizeof(instruction)) <= (buffer_ + buffer_size_));
3425 
3426  memcpy(pc_, &instruction, sizeof(instruction));
3427  pc_ += sizeof(instruction);
3428  CheckBuffer();
3429  }
3430 
3431  // Emit data inline in the instruction stream.
3432  void EmitData(void const * data, unsigned size) {
3433  DCHECK_EQ(sizeof(*pc_), 1);
3434  DCHECK((pc_ + size) <= (buffer_ + buffer_size_));
3435 
3436  // TODO(all): Somehow register we have some data here. Then we can
3437  // disassemble it correctly.
3438  memcpy(pc_, data, size);
3439  pc_ += size;
3440  CheckBuffer();
3441  }
3442 
3443  void GrowBuffer();
3444  void CheckBufferSpace();
3445  void CheckBuffer();
3446 
3447  // Pc offset of the next constant pool check.
3448  int next_constant_pool_check_;
3449 
3450  // Constant pool generation
3451  // Pools are emitted in the instruction stream. They are emitted when:
3452  // * the distance to the first use is above a pre-defined distance or
3453  // * the numbers of entries in the pool is above a pre-defined size or
3454  // * code generation is finished
3455  // If a pool needs to be emitted before code generation is finished a branch
3456  // over the emitted pool will be inserted.
3457 
3458  // Constants in the pool may be addresses of functions that gets relocated;
3459  // if so, a relocation info entry is associated to the constant pool entry.
3460 
3461  // Repeated checking whether the constant pool should be emitted is rather
3462  // expensive. By default we only check again once a number of instructions
3463  // has been generated. That also means that the sizing of the buffers is not
3464  // an exact science, and that we rely on some slop to not overrun buffers.
3465  static constexpr int kCheckConstPoolInterval = 128;
3466 
3467  // Distance to first use after a which a pool will be emitted. Pool entries
3468  // are accessed with pc relative load therefore this cannot be more than
3469  // 1 * MB. Since constant pool emission checks are interval based this value
3470  // is an approximation.
3471  static constexpr int kApproxMaxDistToConstPool = 64 * KB;
3472 
3473  // Number of pool entries after which a pool will be emitted. Since constant
3474  // pool emission checks are interval based this value is an approximation.
3475  static constexpr int kApproxMaxPoolEntryCount = 512;
3476 
3477  // Emission of the constant pool may be blocked in some code sequences.
3478  int const_pool_blocked_nesting_; // Block emission if this is not zero.
3479  int no_const_pool_before_; // Block emission before this pc offset.
3480 
3481  // Emission of the veneer pools may be blocked in some code sequences.
3482  int veneer_pool_blocked_nesting_; // Block emission if this is not zero.
3483 
3484  // Relocation info generation
3485  // Each relocation is encoded as a variable size value
3486  static constexpr int kMaxRelocSize = RelocInfoWriter::kMaxSize;
3487  RelocInfoWriter reloc_info_writer;
3488 
3489  // Internal reference positions, required for (potential) patching in
3490  // GrowBuffer(); contains only those internal references whose labels
3491  // are already bound.
3492  std::deque<int> internal_reference_positions_;
3493 
3494  // Relocation info records are also used during code generation as temporary
3495  // containers for constants and code target addresses until they are emitted
3496  // to the constant pool. These pending relocation info records are temporarily
3497  // stored in a separate buffer until a constant pool is emitted.
3498  // If every instruction in a long sequence is accessing the pool, we need one
3499  // pending relocation entry per instruction.
3500 
3501  // The pending constant pool.
3502  ConstPool constpool_;
3503 
3504  protected:
3505  // Code generation
3506  // The relocation writer's position is at least kGap bytes below the end of
3507  // the generated instructions. This is so that multi-instruction sequences do
3508  // not have to check for overflow. The same is true for writes of large
3509  // relocation info entries, and debug strings encoded in the instruction
3510  // stream.
3511  static constexpr int kGap = 128;
3512 
3513  public:
3514 #ifdef DEBUG
3515  // Functions used for testing.
3516  int GetConstantPoolEntriesSizeForTesting() const {
3517  // Do not include branch over the pool.
3518  return constpool_.EntryCount() * kPointerSize;
3519  }
3520 
3521  static constexpr int GetCheckConstPoolIntervalForTesting() {
3522  return kCheckConstPoolInterval;
3523  }
3524 
3525  static constexpr int GetApproxMaxDistToConstPoolForTesting() {
3526  return kApproxMaxDistToConstPool;
3527  }
3528 #endif
3529 
3531  public:
3532  FarBranchInfo(int offset, Label* label)
3533  : pc_offset_(offset), label_(label) {}
3534  // Offset of the branch in the code generation buffer.
3535  int pc_offset_;
3536  // The label branched to.
3537  Label* label_;
3538  };
3539 
3540  protected:
3541  // Information about unresolved (forward) branches.
3542  // The Assembler is only allowed to delete out-of-date information from here
3543  // after a label is bound. The MacroAssembler uses this information to
3544  // generate veneers.
3545  //
3546  // The second member gives information about the unresolved branch. The first
3547  // member of the pair is the maximum offset that the branch can reach in the
3548  // buffer. The map is sorted according to this reachable offset, allowing to
3549  // easily check when veneers need to be emitted.
3550  // Note that the maximum reachable offset (first member of the pairs) should
3551  // always be positive but has the same type as the return value for
3552  // pc_offset() for convenience.
3553  std::multimap<int, FarBranchInfo> unresolved_branches_;
3554 
3555  // We generate a veneer for a branch if we reach within this distance of the
3556  // limit of the range.
3557  static constexpr int kVeneerDistanceMargin = 1 * KB;
3558  // The factor of 2 is a finger in the air guess. With a default margin of
3559  // 1KB, that leaves us an addional 256 instructions to avoid generating a
3560  // protective branch.
3561  static constexpr int kVeneerNoProtectionFactor = 2;
3562  static constexpr int kVeneerDistanceCheckMargin =
3563  kVeneerNoProtectionFactor * kVeneerDistanceMargin;
3564  int unresolved_branches_first_limit() const {
3565  DCHECK(!unresolved_branches_.empty());
3566  return unresolved_branches_.begin()->first;
3567  }
3568  // This is similar to next_constant_pool_check_ and helps reduce the overhead
3569  // of checking for veneer pools.
3570  // It is maintained to the closest unresolved branch limit minus the maximum
3571  // veneer margin (or kMaxInt if there are no unresolved branches).
3572  int next_veneer_pool_check_;
3573 
3574  private:
3575  // Avoid overflows for displacements etc.
3576  static const int kMaximalBufferSize = 512 * MB;
3577 
3578  // If a veneer is emitted for a branch instruction, that instruction must be
3579  // removed from the associated label's link chain so that the assembler does
3580  // not later attempt (likely unsuccessfully) to patch it to branch directly to
3581  // the label.
3582  void DeleteUnresolvedBranchInfoForLabel(Label* label);
3583  // This function deletes the information related to the label by traversing
3584  // the label chain, and for each PC-relative instruction in the chain checking
3585  // if pending unresolved information exists. Its complexity is proportional to
3586  // the length of the label chain.
3587  void DeleteUnresolvedBranchInfoForLabelTraverse(Label* label);
3588 
3589  void AllocateAndInstallRequestedHeapObjects(Isolate* isolate);
3590 
3591  friend class EnsureSpace;
3592  friend class ConstPool;
3593 };
3594 
3595 class PatchingAssembler : public Assembler {
3596  public:
3597  // Create an Assembler with a buffer starting at 'start'.
3598  // The buffer size is
3599  // size of instructions to patch + kGap
3600  // Where kGap is the distance from which the Assembler tries to grow the
3601  // buffer.
3602  // If more or fewer instructions than expected are generated or if some
3603  // relocation information takes space in the buffer, the PatchingAssembler
3604  // will crash trying to grow the buffer.
3605  // Note that the instruction cache will not be flushed.
3606  PatchingAssembler(const AssemblerOptions& options, byte* start,
3607  unsigned count)
3608  : Assembler(options, start, count * kInstrSize + kGap) {
3609  // Block constant pool emission.
3610  StartBlockPools();
3611  }
3612 
3613  ~PatchingAssembler() {
3614  // Const pool should still be blocked.
3615  DCHECK(is_const_pool_blocked());
3616  EndBlockPools();
3617  // Verify we have generated the number of instruction we expected.
3618  DCHECK((pc_offset() + kGap) == buffer_size_);
3619  // Verify no relocation information has been emitted.
3620  DCHECK(IsConstPoolEmpty());
3621  }
3622 
3623  // See definition of PatchAdrFar() for details.
3624  static constexpr int kAdrFarPatchableNNops = 2;
3625  static constexpr int kAdrFarPatchableNInstrs = kAdrFarPatchableNNops + 2;
3626  void PatchAdrFar(int64_t target_offset);
3627  void PatchSubSp(uint32_t immediate);
3628 };
3629 
3630 class EnsureSpace {
3631  public:
3632  explicit EnsureSpace(Assembler* assembler) {
3633  assembler->CheckBufferSpace();
3634  }
3635 };
3636 
3637 // Define a {RegisterName} method for {CPURegister}.
3638 DEFINE_REGISTER_NAMES(CPURegister, GENERAL_REGISTERS);
3639 
3640 } // namespace internal
3641 } // namespace v8
3642 
3643 #endif // V8_ARM64_ASSEMBLER_ARM64_H_
Definition: libplatform.h:13