V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
instruction-selector-ia32.cc
1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/base/adapters.h"
6 #include "src/compiler/backend/instruction-selector-impl.h"
7 #include "src/compiler/node-matchers.h"
8 #include "src/compiler/node-properties.h"
9 
10 namespace v8 {
11 namespace internal {
12 namespace compiler {
13 
14 // Adds IA32-specific methods for generating operands.
15 class IA32OperandGenerator final : public OperandGenerator {
16  public:
17  explicit IA32OperandGenerator(InstructionSelector* selector)
18  : OperandGenerator(selector) {}
19 
20  InstructionOperand UseByteRegister(Node* node) {
21  // TODO(titzer): encode byte register use constraints.
22  return UseFixed(node, edx);
23  }
24 
25  InstructionOperand DefineAsByteRegister(Node* node) {
26  // TODO(titzer): encode byte register def constraints.
27  return DefineAsRegister(node);
28  }
29 
30  bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
31  int effect_level) {
32  if (input->opcode() != IrOpcode::kLoad ||
33  !selector()->CanCover(node, input)) {
34  return false;
35  }
36  if (effect_level != selector()->GetEffectLevel(input)) {
37  return false;
38  }
39  MachineRepresentation rep =
40  LoadRepresentationOf(input->op()).representation();
41  switch (opcode) {
42  case kIA32And:
43  case kIA32Or:
44  case kIA32Xor:
45  case kIA32Add:
46  case kIA32Sub:
47  case kIA32Cmp:
48  case kIA32Test:
49  return rep == MachineRepresentation::kWord32 || IsAnyTagged(rep);
50  case kIA32Cmp16:
51  case kIA32Test16:
52  return rep == MachineRepresentation::kWord16;
53  case kIA32Cmp8:
54  case kIA32Test8:
55  return rep == MachineRepresentation::kWord8;
56  default:
57  break;
58  }
59  return false;
60  }
61 
62  bool CanBeImmediate(Node* node) {
63  switch (node->opcode()) {
64  case IrOpcode::kInt32Constant:
65  case IrOpcode::kNumberConstant:
66  case IrOpcode::kExternalConstant:
67  case IrOpcode::kRelocatableInt32Constant:
68  case IrOpcode::kRelocatableInt64Constant:
69  return true;
70  case IrOpcode::kHeapConstant: {
71 // TODO(bmeurer): We must not dereference handles concurrently. If we
72 // really have to this here, then we need to find a way to put this
73 // information on the HeapConstant node already.
74 #if 0
75  // Constants in new space cannot be used as immediates in V8 because
76  // the GC does not scan code objects when collecting the new generation.
77  Handle<HeapObject> value = HeapConstantOf(node->op());
78  return !Heap::InNewSpace(*value);
79 #else
80  return false;
81 #endif
82  }
83  default:
84  return false;
85  }
86  }
87 
88  AddressingMode GenerateMemoryOperandInputs(Node* index, int scale, Node* base,
89  Node* displacement_node,
90  DisplacementMode displacement_mode,
91  InstructionOperand inputs[],
92  size_t* input_count) {
93  AddressingMode mode = kMode_MRI;
94  int32_t displacement = (displacement_node == nullptr)
95  ? 0
96  : OpParameter<int32_t>(displacement_node->op());
97  if (displacement_mode == kNegativeDisplacement) {
98  displacement = -displacement;
99  }
100  if (base != nullptr) {
101  if (base->opcode() == IrOpcode::kInt32Constant) {
102  displacement += OpParameter<int32_t>(base->op());
103  base = nullptr;
104  }
105  }
106  if (base != nullptr) {
107  inputs[(*input_count)++] = UseRegister(base);
108  if (index != nullptr) {
109  DCHECK(scale >= 0 && scale <= 3);
110  inputs[(*input_count)++] = UseRegister(index);
111  if (displacement != 0) {
112  inputs[(*input_count)++] = TempImmediate(displacement);
113  static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
114  kMode_MR4I, kMode_MR8I};
115  mode = kMRnI_modes[scale];
116  } else {
117  static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
118  kMode_MR4, kMode_MR8};
119  mode = kMRn_modes[scale];
120  }
121  } else {
122  if (displacement == 0) {
123  mode = kMode_MR;
124  } else {
125  inputs[(*input_count)++] = TempImmediate(displacement);
126  mode = kMode_MRI;
127  }
128  }
129  } else {
130  DCHECK(scale >= 0 && scale <= 3);
131  if (index != nullptr) {
132  inputs[(*input_count)++] = UseRegister(index);
133  if (displacement != 0) {
134  inputs[(*input_count)++] = TempImmediate(displacement);
135  static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
136  kMode_M4I, kMode_M8I};
137  mode = kMnI_modes[scale];
138  } else {
139  static const AddressingMode kMn_modes[] = {kMode_MR, kMode_M2,
140  kMode_M4, kMode_M8};
141  mode = kMn_modes[scale];
142  }
143  } else {
144  inputs[(*input_count)++] = TempImmediate(displacement);
145  return kMode_MI;
146  }
147  }
148  return mode;
149  }
150 
151  AddressingMode GetEffectiveAddressMemoryOperand(Node* node,
152  InstructionOperand inputs[],
153  size_t* input_count) {
154  BaseWithIndexAndDisplacement32Matcher m(node, AddressOption::kAllowAll);
155  DCHECK(m.matches());
156  if ((m.displacement() == nullptr || CanBeImmediate(m.displacement()))) {
157  return GenerateMemoryOperandInputs(
158  m.index(), m.scale(), m.base(), m.displacement(),
159  m.displacement_mode(), inputs, input_count);
160  } else {
161  inputs[(*input_count)++] = UseRegister(node->InputAt(0));
162  inputs[(*input_count)++] = UseRegister(node->InputAt(1));
163  return kMode_MR1;
164  }
165  }
166 
167  InstructionOperand GetEffectiveIndexOperand(Node* index,
168  AddressingMode* mode) {
169  if (CanBeImmediate(index)) {
170  *mode = kMode_MRI;
171  return UseImmediate(index);
172  } else {
173  *mode = kMode_MR1;
174  return UseUniqueRegister(index);
175  }
176  }
177 
178  bool CanBeBetterLeftOperand(Node* node) const {
179  return !selector()->IsLive(node);
180  }
181 };
182 
183 namespace {
184 
185 void VisitRO(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
186  IA32OperandGenerator g(selector);
187  InstructionOperand temps[] = {g.TempRegister()};
188  selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
189  arraysize(temps), temps);
190 }
191 
192 void VisitRR(InstructionSelector* selector, Node* node,
193  InstructionCode opcode) {
194  IA32OperandGenerator g(selector);
195  selector->Emit(opcode, g.DefineAsRegister(node),
196  g.UseRegister(node->InputAt(0)));
197 }
198 
199 void VisitRROFloat(InstructionSelector* selector, Node* node,
200  ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
201  IA32OperandGenerator g(selector);
202  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
203  InstructionOperand operand1 = g.Use(node->InputAt(1));
204  if (selector->IsSupported(AVX)) {
205  selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
206  } else {
207  selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
208  }
209 }
210 
211 void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
212  ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
213  IA32OperandGenerator g(selector);
214  if (selector->IsSupported(AVX)) {
215  selector->Emit(avx_opcode, g.DefineAsRegister(node), g.Use(input));
216  } else {
217  selector->Emit(sse_opcode, g.DefineSameAsFirst(node), g.UseRegister(input));
218  }
219 }
220 
221 void VisitRRSimd(InstructionSelector* selector, Node* node,
222  ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
223  IA32OperandGenerator g(selector);
224  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
225  if (selector->IsSupported(AVX)) {
226  selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0);
227  } else {
228  selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0);
229  }
230 }
231 
232 void VisitRRISimd(InstructionSelector* selector, Node* node,
233  ArchOpcode opcode) {
234  IA32OperandGenerator g(selector);
235  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
236  InstructionOperand operand1 =
237  g.UseImmediate(OpParameter<int32_t>(node->op()));
238  // 8x16 uses movsx_b on dest to extract a byte, which only works
239  // if dest is a byte register.
240  InstructionOperand dest = opcode == kIA32I8x16ExtractLane
241  ? g.DefineAsFixed(node, eax)
242  : g.DefineAsRegister(node);
243  selector->Emit(opcode, dest, operand0, operand1);
244 }
245 
246 void VisitRRISimd(InstructionSelector* selector, Node* node,
247  ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
248  IA32OperandGenerator g(selector);
249  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
250  InstructionOperand operand1 =
251  g.UseImmediate(OpParameter<int32_t>(node->op()));
252  if (selector->IsSupported(AVX)) {
253  selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
254  } else {
255  selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
256  }
257 }
258 
259 } // namespace
260 
261 void InstructionSelector::VisitStackSlot(Node* node) {
262  StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
263  int slot = frame_->AllocateSpillSlot(rep.size());
264  OperandGenerator g(this);
265 
266  Emit(kArchStackSlot, g.DefineAsRegister(node),
267  sequence()->AddImmediate(Constant(slot)), 0, nullptr);
268 }
269 
270 void InstructionSelector::VisitDebugAbort(Node* node) {
271  IA32OperandGenerator g(this);
272  Emit(kArchDebugAbort, g.NoOutput(), g.UseFixed(node->InputAt(0), edx));
273 }
274 
275 void InstructionSelector::VisitSpeculationFence(Node* node) {
276  IA32OperandGenerator g(this);
277  Emit(kLFence, g.NoOutput());
278 }
279 
280 void InstructionSelector::VisitLoad(Node* node) {
281  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
282 
283  ArchOpcode opcode = kArchNop;
284  switch (load_rep.representation()) {
285  case MachineRepresentation::kFloat32:
286  opcode = kIA32Movss;
287  break;
288  case MachineRepresentation::kFloat64:
289  opcode = kIA32Movsd;
290  break;
291  case MachineRepresentation::kBit: // Fall through.
292  case MachineRepresentation::kWord8:
293  opcode = load_rep.IsSigned() ? kIA32Movsxbl : kIA32Movzxbl;
294  break;
295  case MachineRepresentation::kWord16:
296  opcode = load_rep.IsSigned() ? kIA32Movsxwl : kIA32Movzxwl;
297  break;
298  case MachineRepresentation::kTaggedSigned: // Fall through.
299  case MachineRepresentation::kTaggedPointer: // Fall through.
300  case MachineRepresentation::kTagged: // Fall through.
301  case MachineRepresentation::kWord32:
302  opcode = kIA32Movl;
303  break;
304  case MachineRepresentation::kSimd128:
305  opcode = kIA32Movdqu;
306  break;
307  case MachineRepresentation::kWord64: // Fall through.
308  case MachineRepresentation::kNone:
309  UNREACHABLE();
310  return;
311  }
312 
313  IA32OperandGenerator g(this);
314  InstructionOperand outputs[1];
315  outputs[0] = g.DefineAsRegister(node);
316  InstructionOperand inputs[3];
317  size_t input_count = 0;
318  AddressingMode mode =
319  g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
320  InstructionCode code = opcode | AddressingModeField::encode(mode);
321  if (node->opcode() == IrOpcode::kPoisonedLoad) {
322  CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
323  code |= MiscField::encode(kMemoryAccessPoisoned);
324  }
325  Emit(code, 1, outputs, input_count, inputs);
326 }
327 
328 void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }
329 
330 void InstructionSelector::VisitProtectedLoad(Node* node) {
331  // TODO(eholk)
332  UNIMPLEMENTED();
333 }
334 
335 void InstructionSelector::VisitStore(Node* node) {
336  IA32OperandGenerator g(this);
337  Node* base = node->InputAt(0);
338  Node* index = node->InputAt(1);
339  Node* value = node->InputAt(2);
340 
341  StoreRepresentation store_rep = StoreRepresentationOf(node->op());
342  WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
343  MachineRepresentation rep = store_rep.representation();
344 
345  if (write_barrier_kind != kNoWriteBarrier) {
346  DCHECK(CanBeTaggedPointer(rep));
347  AddressingMode addressing_mode;
348  InstructionOperand inputs[] = {
349  g.UseUniqueRegister(base),
350  g.GetEffectiveIndexOperand(index, &addressing_mode),
351  g.UseUniqueRegister(value)};
352  RecordWriteMode record_write_mode = RecordWriteMode::kValueIsAny;
353  switch (write_barrier_kind) {
354  case kNoWriteBarrier:
355  UNREACHABLE();
356  break;
357  case kMapWriteBarrier:
358  record_write_mode = RecordWriteMode::kValueIsMap;
359  break;
360  case kPointerWriteBarrier:
361  record_write_mode = RecordWriteMode::kValueIsPointer;
362  break;
363  case kFullWriteBarrier:
364  record_write_mode = RecordWriteMode::kValueIsAny;
365  break;
366  }
367  InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
368  size_t const temp_count = arraysize(temps);
369  InstructionCode code = kArchStoreWithWriteBarrier;
370  code |= AddressingModeField::encode(addressing_mode);
371  code |= MiscField::encode(static_cast<int>(record_write_mode));
372  Emit(code, 0, nullptr, arraysize(inputs), inputs, temp_count, temps);
373  } else {
374  ArchOpcode opcode = kArchNop;
375  switch (rep) {
376  case MachineRepresentation::kFloat32:
377  opcode = kIA32Movss;
378  break;
379  case MachineRepresentation::kFloat64:
380  opcode = kIA32Movsd;
381  break;
382  case MachineRepresentation::kBit: // Fall through.
383  case MachineRepresentation::kWord8:
384  opcode = kIA32Movb;
385  break;
386  case MachineRepresentation::kWord16:
387  opcode = kIA32Movw;
388  break;
389  case MachineRepresentation::kTaggedSigned: // Fall through.
390  case MachineRepresentation::kTaggedPointer: // Fall through.
391  case MachineRepresentation::kTagged: // Fall through.
392  case MachineRepresentation::kWord32:
393  opcode = kIA32Movl;
394  break;
395  case MachineRepresentation::kSimd128:
396  opcode = kIA32Movdqu;
397  break;
398  case MachineRepresentation::kWord64: // Fall through.
399  case MachineRepresentation::kNone:
400  UNREACHABLE();
401  return;
402  }
403 
404  InstructionOperand val;
405  if (g.CanBeImmediate(value)) {
406  val = g.UseImmediate(value);
407  } else if (rep == MachineRepresentation::kWord8 ||
408  rep == MachineRepresentation::kBit) {
409  val = g.UseByteRegister(value);
410  } else {
411  val = g.UseRegister(value);
412  }
413 
414  InstructionOperand inputs[4];
415  size_t input_count = 0;
416  AddressingMode addressing_mode =
417  g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
418  InstructionCode code =
419  opcode | AddressingModeField::encode(addressing_mode);
420  inputs[input_count++] = val;
421  Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count,
422  inputs);
423  }
424 }
425 
426 void InstructionSelector::VisitProtectedStore(Node* node) {
427  // TODO(eholk)
428  UNIMPLEMENTED();
429 }
430 
431 // Architecture supports unaligned access, therefore VisitLoad is used instead
432 void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
433 
434 // Architecture supports unaligned access, therefore VisitStore is used instead
435 void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
436 
437 namespace {
438 
439 // Shared routine for multiple binary operations.
440 void VisitBinop(InstructionSelector* selector, Node* node,
441  InstructionCode opcode, FlagsContinuation* cont) {
442  IA32OperandGenerator g(selector);
443  Int32BinopMatcher m(node);
444  Node* left = m.left().node();
445  Node* right = m.right().node();
446  InstructionOperand inputs[6];
447  size_t input_count = 0;
448  InstructionOperand outputs[1];
449  size_t output_count = 0;
450 
451  // TODO(turbofan): match complex addressing modes.
452  if (left == right) {
453  // If both inputs refer to the same operand, enforce allocating a register
454  // for both of them to ensure that we don't end up generating code like
455  // this:
456  //
457  // mov eax, [ebp-0x10]
458  // add eax, [ebp-0x10]
459  // jo label
460  InstructionOperand const input = g.UseRegister(left);
461  inputs[input_count++] = input;
462  inputs[input_count++] = input;
463  } else if (g.CanBeImmediate(right)) {
464  inputs[input_count++] = g.UseRegister(left);
465  inputs[input_count++] = g.UseImmediate(right);
466  } else {
467  int effect_level = selector->GetEffectLevel(node);
468  if (cont->IsBranch()) {
469  effect_level = selector->GetEffectLevel(
470  cont->true_block()->PredecessorAt(0)->control_input());
471  }
472  if (node->op()->HasProperty(Operator::kCommutative) &&
473  g.CanBeBetterLeftOperand(right) &&
474  (!g.CanBeBetterLeftOperand(left) ||
475  !g.CanBeMemoryOperand(opcode, node, right, effect_level))) {
476  std::swap(left, right);
477  }
478  if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) {
479  inputs[input_count++] = g.UseRegister(left);
480  AddressingMode addressing_mode =
481  g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
482  opcode |= AddressingModeField::encode(addressing_mode);
483  } else {
484  inputs[input_count++] = g.UseRegister(left);
485  inputs[input_count++] = g.Use(right);
486  }
487  }
488 
489  outputs[output_count++] = g.DefineSameAsFirst(node);
490 
491  DCHECK_NE(0u, input_count);
492  DCHECK_EQ(1u, output_count);
493  DCHECK_GE(arraysize(inputs), input_count);
494  DCHECK_GE(arraysize(outputs), output_count);
495 
496  selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
497  inputs, cont);
498 }
499 
500 // Shared routine for multiple binary operations.
501 void VisitBinop(InstructionSelector* selector, Node* node,
502  InstructionCode opcode) {
503  FlagsContinuation cont;
504  VisitBinop(selector, node, opcode, &cont);
505 }
506 
507 } // namespace
508 
509 void InstructionSelector::VisitWord32And(Node* node) {
510  VisitBinop(this, node, kIA32And);
511 }
512 
513 void InstructionSelector::VisitWord32Or(Node* node) {
514  VisitBinop(this, node, kIA32Or);
515 }
516 
517 void InstructionSelector::VisitWord32Xor(Node* node) {
518  IA32OperandGenerator g(this);
519  Int32BinopMatcher m(node);
520  if (m.right().Is(-1)) {
521  Emit(kIA32Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
522  } else {
523  VisitBinop(this, node, kIA32Xor);
524  }
525 }
526 
527 // Shared routine for multiple shift operations.
528 static inline void VisitShift(InstructionSelector* selector, Node* node,
529  ArchOpcode opcode) {
530  IA32OperandGenerator g(selector);
531  Node* left = node->InputAt(0);
532  Node* right = node->InputAt(1);
533 
534  if (g.CanBeImmediate(right)) {
535  selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
536  g.UseImmediate(right));
537  } else {
538  selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
539  g.UseFixed(right, ecx));
540  }
541 }
542 
543 namespace {
544 
545 void VisitMulHigh(InstructionSelector* selector, Node* node,
546  ArchOpcode opcode) {
547  IA32OperandGenerator g(selector);
548  InstructionOperand temps[] = {g.TempRegister(eax)};
549  selector->Emit(
550  opcode, g.DefineAsFixed(node, edx), g.UseFixed(node->InputAt(0), eax),
551  g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
552 }
553 
554 void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
555  IA32OperandGenerator g(selector);
556  InstructionOperand temps[] = {g.TempRegister(edx)};
557  selector->Emit(opcode, g.DefineAsFixed(node, eax),
558  g.UseFixed(node->InputAt(0), eax),
559  g.UseUnique(node->InputAt(1)), arraysize(temps), temps);
560 }
561 
562 void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
563  IA32OperandGenerator g(selector);
564  InstructionOperand temps[] = {g.TempRegister(eax)};
565  selector->Emit(opcode, g.DefineAsFixed(node, edx),
566  g.UseFixed(node->InputAt(0), eax),
567  g.UseUnique(node->InputAt(1)), arraysize(temps), temps);
568 }
569 
570 void EmitLea(InstructionSelector* selector, Node* result, Node* index,
571  int scale, Node* base, Node* displacement,
572  DisplacementMode displacement_mode) {
573  IA32OperandGenerator g(selector);
574  InstructionOperand inputs[4];
575  size_t input_count = 0;
576  AddressingMode mode =
577  g.GenerateMemoryOperandInputs(index, scale, base, displacement,
578  displacement_mode, inputs, &input_count);
579 
580  DCHECK_NE(0u, input_count);
581  DCHECK_GE(arraysize(inputs), input_count);
582 
583  InstructionOperand outputs[1];
584  outputs[0] = g.DefineAsRegister(result);
585 
586  InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea;
587 
588  selector->Emit(opcode, 1, outputs, input_count, inputs);
589 }
590 
591 } // namespace
592 
593 void InstructionSelector::VisitWord32Shl(Node* node) {
594  Int32ScaleMatcher m(node, true);
595  if (m.matches()) {
596  Node* index = node->InputAt(0);
597  Node* base = m.power_of_two_plus_one() ? index : nullptr;
598  EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement);
599  return;
600  }
601  VisitShift(this, node, kIA32Shl);
602 }
603 
604 void InstructionSelector::VisitWord32Shr(Node* node) {
605  VisitShift(this, node, kIA32Shr);
606 }
607 
608 void InstructionSelector::VisitWord32Sar(Node* node) {
609  VisitShift(this, node, kIA32Sar);
610 }
611 
612 void InstructionSelector::VisitInt32PairAdd(Node* node) {
613  IA32OperandGenerator g(this);
614 
615  Node* projection1 = NodeProperties::FindProjection(node, 1);
616  if (projection1) {
617  // We use UseUniqueRegister here to avoid register sharing with the temp
618  // register.
619  InstructionOperand inputs[] = {
620  g.UseRegister(node->InputAt(0)),
621  g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
622  g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))};
623 
624  InstructionOperand outputs[] = {g.DefineSameAsFirst(node),
625  g.DefineAsRegister(projection1)};
626 
627  InstructionOperand temps[] = {g.TempRegister()};
628 
629  Emit(kIA32AddPair, 2, outputs, 4, inputs, 1, temps);
630  } else {
631  // The high word of the result is not used, so we emit the standard 32 bit
632  // instruction.
633  Emit(kIA32Add, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
634  g.Use(node->InputAt(2)));
635  }
636 }
637 
638 void InstructionSelector::VisitInt32PairSub(Node* node) {
639  IA32OperandGenerator g(this);
640 
641  Node* projection1 = NodeProperties::FindProjection(node, 1);
642  if (projection1) {
643  // We use UseUniqueRegister here to avoid register sharing with the temp
644  // register.
645  InstructionOperand inputs[] = {
646  g.UseRegister(node->InputAt(0)),
647  g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
648  g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))};
649 
650  InstructionOperand outputs[] = {g.DefineSameAsFirst(node),
651  g.DefineAsRegister(projection1)};
652 
653  InstructionOperand temps[] = {g.TempRegister()};
654 
655  Emit(kIA32SubPair, 2, outputs, 4, inputs, 1, temps);
656  } else {
657  // The high word of the result is not used, so we emit the standard 32 bit
658  // instruction.
659  Emit(kIA32Sub, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
660  g.Use(node->InputAt(2)));
661  }
662 }
663 
664 void InstructionSelector::VisitInt32PairMul(Node* node) {
665  IA32OperandGenerator g(this);
666 
667  Node* projection1 = NodeProperties::FindProjection(node, 1);
668  if (projection1) {
669  // InputAt(3) explicitly shares ecx with OutputRegister(1) to save one
670  // register and one mov instruction.
671  InstructionOperand inputs[] = {
672  g.UseUnique(node->InputAt(0)),
673  g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
674  g.UseUniqueRegister(node->InputAt(2)),
675  g.UseFixed(node->InputAt(3), ecx)};
676 
677  InstructionOperand outputs[] = {
678  g.DefineAsFixed(node, eax),
679  g.DefineAsFixed(NodeProperties::FindProjection(node, 1), ecx)};
680 
681  InstructionOperand temps[] = {g.TempRegister(edx)};
682 
683  Emit(kIA32MulPair, 2, outputs, 4, inputs, 1, temps);
684  } else {
685  // The high word of the result is not used, so we emit the standard 32 bit
686  // instruction.
687  Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
688  g.Use(node->InputAt(2)));
689  }
690 }
691 
692 void VisitWord32PairShift(InstructionSelector* selector, InstructionCode opcode,
693  Node* node) {
694  IA32OperandGenerator g(selector);
695 
696  Node* shift = node->InputAt(2);
697  InstructionOperand shift_operand;
698  if (g.CanBeImmediate(shift)) {
699  shift_operand = g.UseImmediate(shift);
700  } else {
701  shift_operand = g.UseFixed(shift, ecx);
702  }
703  InstructionOperand inputs[] = {g.UseFixed(node->InputAt(0), eax),
704  g.UseFixed(node->InputAt(1), edx),
705  shift_operand};
706 
707  InstructionOperand outputs[2];
708  InstructionOperand temps[1];
709  int32_t output_count = 0;
710  int32_t temp_count = 0;
711  outputs[output_count++] = g.DefineAsFixed(node, eax);
712  Node* projection1 = NodeProperties::FindProjection(node, 1);
713  if (projection1) {
714  outputs[output_count++] = g.DefineAsFixed(projection1, edx);
715  } else {
716  temps[temp_count++] = g.TempRegister(edx);
717  }
718 
719  selector->Emit(opcode, output_count, outputs, 3, inputs, temp_count, temps);
720 }
721 
722 void InstructionSelector::VisitWord32PairShl(Node* node) {
723  VisitWord32PairShift(this, kIA32ShlPair, node);
724 }
725 
726 void InstructionSelector::VisitWord32PairShr(Node* node) {
727  VisitWord32PairShift(this, kIA32ShrPair, node);
728 }
729 
730 void InstructionSelector::VisitWord32PairSar(Node* node) {
731  VisitWord32PairShift(this, kIA32SarPair, node);
732 }
733 
734 void InstructionSelector::VisitWord32Ror(Node* node) {
735  VisitShift(this, node, kIA32Ror);
736 }
737 
738 #define RO_OP_LIST(V) \
739  V(Word32Clz, kIA32Lzcnt) \
740  V(Word32Ctz, kIA32Tzcnt) \
741  V(Word32Popcnt, kIA32Popcnt) \
742  V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64) \
743  V(RoundInt32ToFloat32, kSSEInt32ToFloat32) \
744  V(ChangeInt32ToFloat64, kSSEInt32ToFloat64) \
745  V(ChangeUint32ToFloat64, kSSEUint32ToFloat64) \
746  V(TruncateFloat32ToInt32, kSSEFloat32ToInt32) \
747  V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \
748  V(ChangeFloat64ToInt32, kSSEFloat64ToInt32) \
749  V(ChangeFloat64ToUint32, kSSEFloat64ToUint32) \
750  V(TruncateFloat64ToUint32, kSSEFloat64ToUint32) \
751  V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32) \
752  V(RoundFloat64ToInt32, kSSEFloat64ToInt32) \
753  V(BitcastFloat32ToInt32, kIA32BitcastFI) \
754  V(BitcastInt32ToFloat32, kIA32BitcastIF) \
755  V(Float32Sqrt, kSSEFloat32Sqrt) \
756  V(Float64Sqrt, kSSEFloat64Sqrt) \
757  V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32) \
758  V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \
759  V(SignExtendWord8ToInt32, kIA32Movsxbl) \
760  V(SignExtendWord16ToInt32, kIA32Movsxwl)
761 
762 #define RR_OP_LIST(V) \
763  V(TruncateFloat64ToWord32, kArchTruncateDoubleToI) \
764  V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown)) \
765  V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown)) \
766  V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp)) \
767  V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp)) \
768  V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \
769  V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \
770  V(Float32RoundTiesEven, \
771  kSSEFloat32Round | MiscField::encode(kRoundToNearest)) \
772  V(Float64RoundTiesEven, kSSEFloat64Round | MiscField::encode(kRoundToNearest))
773 
774 #define RRO_FLOAT_OP_LIST(V) \
775  V(Float32Add, kAVXFloat32Add, kSSEFloat32Add) \
776  V(Float64Add, kAVXFloat64Add, kSSEFloat64Add) \
777  V(Float32Sub, kAVXFloat32Sub, kSSEFloat32Sub) \
778  V(Float64Sub, kAVXFloat64Sub, kSSEFloat64Sub) \
779  V(Float32Mul, kAVXFloat32Mul, kSSEFloat32Mul) \
780  V(Float64Mul, kAVXFloat64Mul, kSSEFloat64Mul) \
781  V(Float32Div, kAVXFloat32Div, kSSEFloat32Div) \
782  V(Float64Div, kAVXFloat64Div, kSSEFloat64Div)
783 
784 #define FLOAT_UNOP_LIST(V) \
785  V(Float32Abs, kAVXFloat32Abs, kSSEFloat32Abs) \
786  V(Float64Abs, kAVXFloat64Abs, kSSEFloat64Abs) \
787  V(Float32Neg, kAVXFloat32Neg, kSSEFloat32Neg) \
788  V(Float64Neg, kAVXFloat64Neg, kSSEFloat64Neg)
789 
790 #define RO_VISITOR(Name, opcode) \
791  void InstructionSelector::Visit##Name(Node* node) { \
792  VisitRO(this, node, opcode); \
793  }
794 RO_OP_LIST(RO_VISITOR)
795 #undef RO_VISITOR
796 #undef RO_OP_LIST
797 
798 #define RR_VISITOR(Name, opcode) \
799  void InstructionSelector::Visit##Name(Node* node) { \
800  VisitRR(this, node, opcode); \
801  }
802 RR_OP_LIST(RR_VISITOR)
803 #undef RR_VISITOR
804 #undef RR_OP_LIST
805 
806 #define RRO_FLOAT_VISITOR(Name, avx, sse) \
807  void InstructionSelector::Visit##Name(Node* node) { \
808  VisitRROFloat(this, node, avx, sse); \
809  }
810 RRO_FLOAT_OP_LIST(RRO_FLOAT_VISITOR)
811 #undef RRO_FLOAT_VISITOR
812 #undef RRO_FLOAT_OP_LIST
813 
814 #define FLOAT_UNOP_VISITOR(Name, avx, sse) \
815  void InstructionSelector::Visit##Name(Node* node) { \
816  VisitFloatUnop(this, node, node->InputAt(0), avx, sse); \
817  }
818 FLOAT_UNOP_LIST(FLOAT_UNOP_VISITOR)
819 #undef FLOAT_UNOP_VISITOR
820 #undef FLOAT_UNOP_LIST
821 
822 void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
823 
824 void InstructionSelector::VisitWord64ReverseBytes(Node* node) { UNREACHABLE(); }
825 
826 void InstructionSelector::VisitWord32ReverseBytes(Node* node) {
827  IA32OperandGenerator g(this);
828  Emit(kIA32Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
829 }
830 
831 void InstructionSelector::VisitInt32Add(Node* node) {
832  IA32OperandGenerator g(this);
833 
834  // Try to match the Add to a lea pattern
835  BaseWithIndexAndDisplacement32Matcher m(node);
836  if (m.matches() &&
837  (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
838  InstructionOperand inputs[4];
839  size_t input_count = 0;
840  AddressingMode mode = g.GenerateMemoryOperandInputs(
841  m.index(), m.scale(), m.base(), m.displacement(), m.displacement_mode(),
842  inputs, &input_count);
843 
844  DCHECK_NE(0u, input_count);
845  DCHECK_GE(arraysize(inputs), input_count);
846 
847  InstructionOperand outputs[1];
848  outputs[0] = g.DefineAsRegister(node);
849 
850  InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea;
851  Emit(opcode, 1, outputs, input_count, inputs);
852  return;
853  }
854 
855  // No lea pattern match, use add
856  VisitBinop(this, node, kIA32Add);
857 }
858 
859 void InstructionSelector::VisitInt32Sub(Node* node) {
860  IA32OperandGenerator g(this);
861  Int32BinopMatcher m(node);
862  if (m.left().Is(0)) {
863  Emit(kIA32Neg, g.DefineSameAsFirst(node), g.Use(m.right().node()));
864  } else {
865  VisitBinop(this, node, kIA32Sub);
866  }
867 }
868 
869 void InstructionSelector::VisitInt32Mul(Node* node) {
870  Int32ScaleMatcher m(node, true);
871  if (m.matches()) {
872  Node* index = node->InputAt(0);
873  Node* base = m.power_of_two_plus_one() ? index : nullptr;
874  EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement);
875  return;
876  }
877  IA32OperandGenerator g(this);
878  Node* left = node->InputAt(0);
879  Node* right = node->InputAt(1);
880  if (g.CanBeImmediate(right)) {
881  Emit(kIA32Imul, g.DefineAsRegister(node), g.Use(left),
882  g.UseImmediate(right));
883  } else {
884  if (g.CanBeBetterLeftOperand(right)) {
885  std::swap(left, right);
886  }
887  Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(left),
888  g.Use(right));
889  }
890 }
891 
892 void InstructionSelector::VisitInt32MulHigh(Node* node) {
893  VisitMulHigh(this, node, kIA32ImulHigh);
894 }
895 
896 void InstructionSelector::VisitUint32MulHigh(Node* node) {
897  VisitMulHigh(this, node, kIA32UmulHigh);
898 }
899 
900 void InstructionSelector::VisitInt32Div(Node* node) {
901  VisitDiv(this, node, kIA32Idiv);
902 }
903 
904 void InstructionSelector::VisitUint32Div(Node* node) {
905  VisitDiv(this, node, kIA32Udiv);
906 }
907 
908 void InstructionSelector::VisitInt32Mod(Node* node) {
909  VisitMod(this, node, kIA32Idiv);
910 }
911 
912 void InstructionSelector::VisitUint32Mod(Node* node) {
913  VisitMod(this, node, kIA32Udiv);
914 }
915 
916 void InstructionSelector::VisitRoundUint32ToFloat32(Node* node) {
917  IA32OperandGenerator g(this);
918  InstructionOperand temps[] = {g.TempRegister()};
919  Emit(kSSEUint32ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
920  arraysize(temps), temps);
921 }
922 
923 void InstructionSelector::VisitFloat64Mod(Node* node) {
924  IA32OperandGenerator g(this);
925  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister()};
926  Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
927  g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
928  arraysize(temps), temps);
929 }
930 
931 void InstructionSelector::VisitFloat32Max(Node* node) {
932  IA32OperandGenerator g(this);
933  InstructionOperand temps[] = {g.TempRegister()};
934  Emit(kSSEFloat32Max, g.DefineSameAsFirst(node),
935  g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
936  arraysize(temps), temps);
937 }
938 
939 void InstructionSelector::VisitFloat64Max(Node* node) {
940  IA32OperandGenerator g(this);
941  InstructionOperand temps[] = {g.TempRegister()};
942  Emit(kSSEFloat64Max, g.DefineSameAsFirst(node),
943  g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
944  arraysize(temps), temps);
945 }
946 
947 void InstructionSelector::VisitFloat32Min(Node* node) {
948  IA32OperandGenerator g(this);
949  InstructionOperand temps[] = {g.TempRegister()};
950  Emit(kSSEFloat32Min, g.DefineSameAsFirst(node),
951  g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
952  arraysize(temps), temps);
953 }
954 
955 void InstructionSelector::VisitFloat64Min(Node* node) {
956  IA32OperandGenerator g(this);
957  InstructionOperand temps[] = {g.TempRegister()};
958  Emit(kSSEFloat64Min, g.DefineSameAsFirst(node),
959  g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
960  arraysize(temps), temps);
961 }
962 
963 void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
964  UNREACHABLE();
965 }
966 
967 void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
968  InstructionCode opcode) {
969  IA32OperandGenerator g(this);
970  Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
971  g.UseRegister(node->InputAt(1)))
972  ->MarkAsCall();
973 }
974 
975 void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
976  InstructionCode opcode) {
977  IA32OperandGenerator g(this);
978  Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)))
979  ->MarkAsCall();
980 }
981 
982 void InstructionSelector::EmitPrepareArguments(
983  ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
984  Node* node) {
985  IA32OperandGenerator g(this);
986 
987  // Prepare for C function call.
988  if (call_descriptor->IsCFunctionCall()) {
989  InstructionOperand temps[] = {g.TempRegister()};
990  size_t const temp_count = arraysize(temps);
991  Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>(
992  call_descriptor->ParameterCount())),
993  0, nullptr, 0, nullptr, temp_count, temps);
994 
995  // Poke any stack arguments.
996  for (size_t n = 0; n < arguments->size(); ++n) {
997  PushParameter input = (*arguments)[n];
998  if (input.node) {
999  int const slot = static_cast<int>(n);
1000  InstructionOperand value = g.CanBeImmediate(node)
1001  ? g.UseImmediate(input.node)
1002  : g.UseRegister(input.node);
1003  Emit(kIA32Poke | MiscField::encode(slot), g.NoOutput(), value);
1004  }
1005  }
1006  } else {
1007  // Push any stack arguments.
1008  int effect_level = GetEffectLevel(node);
1009  for (PushParameter input : base::Reversed(*arguments)) {
1010  // Skip any alignment holes in pushed nodes.
1011  if (input.node == nullptr) continue;
1012  if (g.CanBeMemoryOperand(kIA32Push, node, input.node, effect_level)) {
1013  InstructionOperand outputs[1];
1014  InstructionOperand inputs[4];
1015  size_t input_count = 0;
1016  InstructionCode opcode = kIA32Push;
1017  AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1018  input.node, inputs, &input_count);
1019  opcode |= AddressingModeField::encode(mode);
1020  Emit(opcode, 0, outputs, input_count, inputs);
1021  } else {
1022  InstructionOperand value =
1023  g.CanBeImmediate(input.node)
1024  ? g.UseImmediate(input.node)
1025  : IsSupported(ATOM) ||
1026  sequence()->IsFP(GetVirtualRegister(input.node))
1027  ? g.UseRegister(input.node)
1028  : g.Use(input.node);
1029  if (input.location.GetType() == MachineType::Float32()) {
1030  Emit(kIA32PushFloat32, g.NoOutput(), value);
1031  } else if (input.location.GetType() == MachineType::Float64()) {
1032  Emit(kIA32PushFloat64, g.NoOutput(), value);
1033  } else if (input.location.GetType() == MachineType::Simd128()) {
1034  Emit(kIA32PushSimd128, g.NoOutput(), value);
1035  } else {
1036  Emit(kIA32Push, g.NoOutput(), value);
1037  }
1038  }
1039  }
1040  }
1041 }
1042 
1043 void InstructionSelector::EmitPrepareResults(
1044  ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
1045  Node* node) {
1046  IA32OperandGenerator g(this);
1047 
1048  int reverse_slot = 0;
1049  for (PushParameter output : *results) {
1050  if (!output.location.IsCallerFrameSlot()) continue;
1051  // Skip any alignment holes in nodes.
1052  if (output.node != nullptr) {
1053  DCHECK(!call_descriptor->IsCFunctionCall());
1054  if (output.location.GetType() == MachineType::Float32()) {
1055  MarkAsFloat32(output.node);
1056  } else if (output.location.GetType() == MachineType::Float64()) {
1057  MarkAsFloat64(output.node);
1058  }
1059  Emit(kIA32Peek, g.DefineAsRegister(output.node),
1060  g.UseImmediate(reverse_slot));
1061  }
1062  reverse_slot += output.location.GetSizeInPointers();
1063  }
1064 }
1065 
1066 bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
1067 
1068 int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 0; }
1069 
1070 namespace {
1071 
1072 void VisitCompareWithMemoryOperand(InstructionSelector* selector,
1073  InstructionCode opcode, Node* left,
1074  InstructionOperand right,
1075  FlagsContinuation* cont) {
1076  DCHECK_EQ(IrOpcode::kLoad, left->opcode());
1077  IA32OperandGenerator g(selector);
1078  size_t input_count = 0;
1079  InstructionOperand inputs[4];
1080  AddressingMode addressing_mode =
1081  g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
1082  opcode |= AddressingModeField::encode(addressing_mode);
1083  inputs[input_count++] = right;
1084 
1085  selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
1086 }
1087 
1088 // Shared routine for multiple compare operations.
1089 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1090  InstructionOperand left, InstructionOperand right,
1091  FlagsContinuation* cont) {
1092  selector->EmitWithContinuation(opcode, left, right, cont);
1093 }
1094 
1095 // Shared routine for multiple compare operations.
1096 void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1097  Node* left, Node* right, FlagsContinuation* cont,
1098  bool commutative) {
1099  IA32OperandGenerator g(selector);
1100  if (commutative && g.CanBeBetterLeftOperand(right)) {
1101  std::swap(left, right);
1102  }
1103  VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
1104 }
1105 
1106 MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
1107  if (hint_node->opcode() == IrOpcode::kLoad) {
1108  MachineType hint = LoadRepresentationOf(hint_node->op());
1109  if (node->opcode() == IrOpcode::kInt32Constant ||
1110  node->opcode() == IrOpcode::kInt64Constant) {
1111  int64_t constant = node->opcode() == IrOpcode::kInt32Constant
1112  ? OpParameter<int32_t>(node->op())
1113  : OpParameter<int64_t>(node->op());
1114  if (hint == MachineType::Int8()) {
1115  if (constant >= std::numeric_limits<int8_t>::min() &&
1116  constant <= std::numeric_limits<int8_t>::max()) {
1117  return hint;
1118  }
1119  } else if (hint == MachineType::Uint8()) {
1120  if (constant >= std::numeric_limits<uint8_t>::min() &&
1121  constant <= std::numeric_limits<uint8_t>::max()) {
1122  return hint;
1123  }
1124  } else if (hint == MachineType::Int16()) {
1125  if (constant >= std::numeric_limits<int16_t>::min() &&
1126  constant <= std::numeric_limits<int16_t>::max()) {
1127  return hint;
1128  }
1129  } else if (hint == MachineType::Uint16()) {
1130  if (constant >= std::numeric_limits<uint16_t>::min() &&
1131  constant <= std::numeric_limits<uint16_t>::max()) {
1132  return hint;
1133  }
1134  } else if (hint == MachineType::Int32()) {
1135  return hint;
1136  } else if (hint == MachineType::Uint32()) {
1137  if (constant >= 0) return hint;
1138  }
1139  }
1140  }
1141  return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op())
1142  : MachineType::None();
1143 }
1144 
1145 // Tries to match the size of the given opcode to that of the operands, if
1146 // possible.
1147 InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
1148  Node* right, FlagsContinuation* cont) {
1149  // TODO(epertoso): we can probably get some size information out of phi nodes.
1150  // If the load representations don't match, both operands will be
1151  // zero/sign-extended to 32bit.
1152  MachineType left_type = MachineTypeForNarrow(left, right);
1153  MachineType right_type = MachineTypeForNarrow(right, left);
1154  if (left_type == right_type) {
1155  switch (left_type.representation()) {
1156  case MachineRepresentation::kBit:
1157  case MachineRepresentation::kWord8: {
1158  if (opcode == kIA32Test) return kIA32Test8;
1159  if (opcode == kIA32Cmp) {
1160  if (left_type.semantic() == MachineSemantic::kUint32) {
1161  cont->OverwriteUnsignedIfSigned();
1162  } else {
1163  CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1164  }
1165  return kIA32Cmp8;
1166  }
1167  break;
1168  }
1169  case MachineRepresentation::kWord16:
1170  if (opcode == kIA32Test) return kIA32Test16;
1171  if (opcode == kIA32Cmp) {
1172  if (left_type.semantic() == MachineSemantic::kUint32) {
1173  cont->OverwriteUnsignedIfSigned();
1174  } else {
1175  CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1176  }
1177  return kIA32Cmp16;
1178  }
1179  break;
1180  default:
1181  break;
1182  }
1183  }
1184  return opcode;
1185 }
1186 
1187 // Shared routine for multiple float32 compare operations (inputs commuted).
1188 void VisitFloat32Compare(InstructionSelector* selector, Node* node,
1189  FlagsContinuation* cont) {
1190  Node* const left = node->InputAt(0);
1191  Node* const right = node->InputAt(1);
1192  VisitCompare(selector, kSSEFloat32Cmp, right, left, cont, false);
1193 }
1194 
1195 // Shared routine for multiple float64 compare operations (inputs commuted).
1196 void VisitFloat64Compare(InstructionSelector* selector, Node* node,
1197  FlagsContinuation* cont) {
1198  Node* const left = node->InputAt(0);
1199  Node* const right = node->InputAt(1);
1200  VisitCompare(selector, kSSEFloat64Cmp, right, left, cont, false);
1201 }
1202 
1203 // Shared routine for multiple word compare operations.
1204 void VisitWordCompare(InstructionSelector* selector, Node* node,
1205  InstructionCode opcode, FlagsContinuation* cont) {
1206  IA32OperandGenerator g(selector);
1207  Node* left = node->InputAt(0);
1208  Node* right = node->InputAt(1);
1209 
1210  InstructionCode narrowed_opcode =
1211  TryNarrowOpcodeSize(opcode, left, right, cont);
1212 
1213  int effect_level = selector->GetEffectLevel(node);
1214  if (cont->IsBranch()) {
1215  effect_level = selector->GetEffectLevel(
1216  cont->true_block()->PredecessorAt(0)->control_input());
1217  }
1218 
1219  // If one of the two inputs is an immediate, make sure it's on the right, or
1220  // if one of the two inputs is a memory operand, make sure it's on the left.
1221  if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
1222  (g.CanBeMemoryOperand(narrowed_opcode, node, right, effect_level) &&
1223  !g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level))) {
1224  if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1225  std::swap(left, right);
1226  }
1227 
1228  // Match immediates on right side of comparison.
1229  if (g.CanBeImmediate(right)) {
1230  if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
1231  return VisitCompareWithMemoryOperand(selector, narrowed_opcode, left,
1232  g.UseImmediate(right), cont);
1233  }
1234  return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
1235  cont);
1236  }
1237 
1238  // Match memory operands on left side of comparison.
1239  if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
1240  bool needs_byte_register =
1241  narrowed_opcode == kIA32Test8 || narrowed_opcode == kIA32Cmp8;
1242  return VisitCompareWithMemoryOperand(
1243  selector, narrowed_opcode, left,
1244  needs_byte_register ? g.UseByteRegister(right) : g.UseRegister(right),
1245  cont);
1246  }
1247 
1248  return VisitCompare(selector, opcode, left, right, cont,
1249  node->op()->HasProperty(Operator::kCommutative));
1250 }
1251 
1252 void VisitWordCompare(InstructionSelector* selector, Node* node,
1253  FlagsContinuation* cont) {
1254  StackCheckMatcher<Int32BinopMatcher, IrOpcode::kUint32LessThan> m(
1255  selector->isolate(), node);
1256  if (m.Matched()) {
1257  // Compare(Load(js_stack_limit), LoadStackPointer)
1258  if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1259  InstructionCode opcode = cont->Encode(kIA32StackCheck);
1260  CHECK(cont->IsBranch());
1261  selector->EmitWithContinuation(opcode, cont);
1262  return;
1263  }
1264  WasmStackCheckMatcher<Int32BinopMatcher, IrOpcode::kUint32LessThan> wasm_m(
1265  node);
1266  if (wasm_m.Matched()) {
1267  // This is a wasm stack check. By structure, we know that we can use the
1268  // stack pointer directly, as wasm code does not modify the stack at points
1269  // where stack checks are performed.
1270  Node* left = node->InputAt(0);
1271  LocationOperand esp(InstructionOperand::EXPLICIT, LocationOperand::REGISTER,
1272  InstructionSequence::DefaultRepresentation(),
1273  RegisterCode::kRegCode_esp);
1274  return VisitCompareWithMemoryOperand(selector, kIA32Cmp, left, esp, cont);
1275  }
1276  VisitWordCompare(selector, node, kIA32Cmp, cont);
1277 }
1278 
1279 void VisitAtomicExchange(InstructionSelector* selector, Node* node,
1280  ArchOpcode opcode, MachineRepresentation rep) {
1281  IA32OperandGenerator g(selector);
1282  Node* base = node->InputAt(0);
1283  Node* index = node->InputAt(1);
1284  Node* value = node->InputAt(2);
1285 
1286  AddressingMode addressing_mode;
1287  InstructionOperand value_operand = (rep == MachineRepresentation::kWord8)
1288  ? g.UseFixed(value, edx)
1289  : g.UseUniqueRegister(value);
1290  InstructionOperand inputs[] = {
1291  value_operand, g.UseUniqueRegister(base),
1292  g.GetEffectiveIndexOperand(index, &addressing_mode)};
1293  InstructionOperand outputs[] = {
1294  (rep == MachineRepresentation::kWord8)
1295  // Using DefineSameAsFirst requires the register to be unallocated.
1296  ? g.DefineAsFixed(node, edx)
1297  : g.DefineSameAsFirst(node)};
1298  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1299  selector->Emit(code, 1, outputs, arraysize(inputs), inputs);
1300 }
1301 
1302 void VisitAtomicBinOp(InstructionSelector* selector, Node* node,
1303  ArchOpcode opcode, MachineRepresentation rep) {
1304  AddressingMode addressing_mode;
1305  IA32OperandGenerator g(selector);
1306  Node* base = node->InputAt(0);
1307  Node* index = node->InputAt(1);
1308  Node* value = node->InputAt(2);
1309  InstructionOperand inputs[] = {
1310  g.UseUniqueRegister(value), g.UseUniqueRegister(base),
1311  g.GetEffectiveIndexOperand(index, &addressing_mode)};
1312  InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)};
1313  InstructionOperand temp[] = {(rep == MachineRepresentation::kWord8)
1314  ? g.UseByteRegister(node)
1315  : g.TempRegister()};
1316  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1317  selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1318  arraysize(temp), temp);
1319 }
1320 
1321 void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node,
1322  ArchOpcode opcode) {
1323  IA32OperandGenerator g(selector);
1324  Node* base = node->InputAt(0);
1325  Node* index = node->InputAt(1);
1326  Node* value = node->InputAt(2);
1327  // For Word64 operations, the value input is split into the a high node,
1328  // and a low node in the int64-lowering phase.
1329  Node* value_high = node->InputAt(3);
1330 
1331  // Wasm lives in 32-bit address space, so we do not need to worry about
1332  // base/index lowering. This will need to be fixed for Wasm64.
1333  AddressingMode addressing_mode;
1334  InstructionOperand inputs[] = {
1335  g.UseUniqueRegisterOrSlotOrConstant(value), g.UseFixed(value_high, ecx),
1336  g.UseUniqueRegister(base),
1337  g.GetEffectiveIndexOperand(index, &addressing_mode)};
1338  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1339  Node* projection0 = NodeProperties::FindProjection(node, 0);
1340  Node* projection1 = NodeProperties::FindProjection(node, 1);
1341  if (projection1) {
1342  InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax),
1343  g.DefineAsFixed(projection1, edx)};
1344  selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1345  0, {});
1346  } else if (projection0) {
1347  InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax)};
1348  InstructionOperand temps[] = {g.TempRegister(edx)};
1349  const int num_temps = arraysize(temps);
1350  selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1351  num_temps, temps);
1352  } else {
1353  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
1354  const int num_temps = arraysize(temps);
1355  selector->Emit(code, 0, nullptr, arraysize(inputs), inputs, num_temps,
1356  temps);
1357  }
1358 }
1359 
1360 } // namespace
1361 
1362 // Shared routine for word comparison with zero.
1363 void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
1364  FlagsContinuation* cont) {
1365  // Try to combine with comparisons against 0 by simply inverting the branch.
1366  while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
1367  Int32BinopMatcher m(value);
1368  if (!m.right().Is(0)) break;
1369 
1370  user = value;
1371  value = m.left().node();
1372  cont->Negate();
1373  }
1374 
1375  if (CanCover(user, value)) {
1376  switch (value->opcode()) {
1377  case IrOpcode::kWord32Equal:
1378  cont->OverwriteAndNegateIfEqual(kEqual);
1379  return VisitWordCompare(this, value, cont);
1380  case IrOpcode::kInt32LessThan:
1381  cont->OverwriteAndNegateIfEqual(kSignedLessThan);
1382  return VisitWordCompare(this, value, cont);
1383  case IrOpcode::kInt32LessThanOrEqual:
1384  cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
1385  return VisitWordCompare(this, value, cont);
1386  case IrOpcode::kUint32LessThan:
1387  cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
1388  return VisitWordCompare(this, value, cont);
1389  case IrOpcode::kUint32LessThanOrEqual:
1390  cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
1391  return VisitWordCompare(this, value, cont);
1392  case IrOpcode::kFloat32Equal:
1393  cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1394  return VisitFloat32Compare(this, value, cont);
1395  case IrOpcode::kFloat32LessThan:
1396  cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1397  return VisitFloat32Compare(this, value, cont);
1398  case IrOpcode::kFloat32LessThanOrEqual:
1399  cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1400  return VisitFloat32Compare(this, value, cont);
1401  case IrOpcode::kFloat64Equal:
1402  cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1403  return VisitFloat64Compare(this, value, cont);
1404  case IrOpcode::kFloat64LessThan:
1405  cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1406  return VisitFloat64Compare(this, value, cont);
1407  case IrOpcode::kFloat64LessThanOrEqual:
1408  cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1409  return VisitFloat64Compare(this, value, cont);
1410  case IrOpcode::kProjection:
1411  // Check if this is the overflow output projection of an
1412  // <Operation>WithOverflow node.
1413  if (ProjectionIndexOf(value->op()) == 1u) {
1414  // We cannot combine the <Operation>WithOverflow with this branch
1415  // unless the 0th projection (the use of the actual value of the
1416  // <Operation> is either nullptr, which means there's no use of the
1417  // actual value, or was already defined, which means it is scheduled
1418  // *AFTER* this branch).
1419  Node* const node = value->InputAt(0);
1420  Node* const result = NodeProperties::FindProjection(node, 0);
1421  if (result == nullptr || IsDefined(result)) {
1422  switch (node->opcode()) {
1423  case IrOpcode::kInt32AddWithOverflow:
1424  cont->OverwriteAndNegateIfEqual(kOverflow);
1425  return VisitBinop(this, node, kIA32Add, cont);
1426  case IrOpcode::kInt32SubWithOverflow:
1427  cont->OverwriteAndNegateIfEqual(kOverflow);
1428  return VisitBinop(this, node, kIA32Sub, cont);
1429  case IrOpcode::kInt32MulWithOverflow:
1430  cont->OverwriteAndNegateIfEqual(kOverflow);
1431  return VisitBinop(this, node, kIA32Imul, cont);
1432  default:
1433  break;
1434  }
1435  }
1436  }
1437  break;
1438  case IrOpcode::kInt32Sub:
1439  return VisitWordCompare(this, value, cont);
1440  case IrOpcode::kWord32And:
1441  return VisitWordCompare(this, value, kIA32Test, cont);
1442  default:
1443  break;
1444  }
1445  }
1446 
1447  // Continuation could not be combined with a compare, emit compare against 0.
1448  IA32OperandGenerator g(this);
1449  VisitCompare(this, kIA32Cmp, g.Use(value), g.TempImmediate(0), cont);
1450 }
1451 
1452 void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
1453  IA32OperandGenerator g(this);
1454  InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
1455 
1456  // Emit either ArchTableSwitch or ArchLookupSwitch.
1457  if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
1458  static const size_t kMaxTableSwitchValueRange = 2 << 16;
1459  size_t table_space_cost = 4 + sw.value_range();
1460  size_t table_time_cost = 3;
1461  size_t lookup_space_cost = 3 + 2 * sw.case_count();
1462  size_t lookup_time_cost = sw.case_count();
1463  if (sw.case_count() > 4 &&
1464  table_space_cost + 3 * table_time_cost <=
1465  lookup_space_cost + 3 * lookup_time_cost &&
1466  sw.min_value() > std::numeric_limits<int32_t>::min() &&
1467  sw.value_range() <= kMaxTableSwitchValueRange) {
1468  InstructionOperand index_operand = value_operand;
1469  if (sw.min_value()) {
1470  index_operand = g.TempRegister();
1471  Emit(kIA32Lea | AddressingModeField::encode(kMode_MRI), index_operand,
1472  value_operand, g.TempImmediate(-sw.min_value()));
1473  }
1474  // Generate a table lookup.
1475  return EmitTableSwitch(sw, index_operand);
1476  }
1477  }
1478 
1479  // Generate a tree of conditional jumps.
1480  return EmitBinarySearchSwitch(sw, value_operand);
1481 }
1482 
1483 void InstructionSelector::VisitWord32Equal(Node* const node) {
1484  FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
1485  Int32BinopMatcher m(node);
1486  if (m.right().Is(0)) {
1487  return VisitWordCompareZero(m.node(), m.left().node(), &cont);
1488  }
1489  VisitWordCompare(this, node, &cont);
1490 }
1491 
1492 void InstructionSelector::VisitInt32LessThan(Node* node) {
1493  FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
1494  VisitWordCompare(this, node, &cont);
1495 }
1496 
1497 void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
1498  FlagsContinuation cont =
1499  FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
1500  VisitWordCompare(this, node, &cont);
1501 }
1502 
1503 void InstructionSelector::VisitUint32LessThan(Node* node) {
1504  FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
1505  VisitWordCompare(this, node, &cont);
1506 }
1507 
1508 void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
1509  FlagsContinuation cont =
1510  FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
1511  VisitWordCompare(this, node, &cont);
1512 }
1513 
1514 void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
1515  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1516  FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1517  return VisitBinop(this, node, kIA32Add, &cont);
1518  }
1519  FlagsContinuation cont;
1520  VisitBinop(this, node, kIA32Add, &cont);
1521 }
1522 
1523 void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
1524  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1525  FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1526  return VisitBinop(this, node, kIA32Sub, &cont);
1527  }
1528  FlagsContinuation cont;
1529  VisitBinop(this, node, kIA32Sub, &cont);
1530 }
1531 
1532 void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
1533  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1534  FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1535  return VisitBinop(this, node, kIA32Imul, &cont);
1536  }
1537  FlagsContinuation cont;
1538  VisitBinop(this, node, kIA32Imul, &cont);
1539 }
1540 
1541 void InstructionSelector::VisitFloat32Equal(Node* node) {
1542  FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
1543  VisitFloat32Compare(this, node, &cont);
1544 }
1545 
1546 void InstructionSelector::VisitFloat32LessThan(Node* node) {
1547  FlagsContinuation cont =
1548  FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
1549  VisitFloat32Compare(this, node, &cont);
1550 }
1551 
1552 void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
1553  FlagsContinuation cont =
1554  FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
1555  VisitFloat32Compare(this, node, &cont);
1556 }
1557 
1558 void InstructionSelector::VisitFloat64Equal(Node* node) {
1559  FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
1560  VisitFloat64Compare(this, node, &cont);
1561 }
1562 
1563 void InstructionSelector::VisitFloat64LessThan(Node* node) {
1564  FlagsContinuation cont =
1565  FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
1566  VisitFloat64Compare(this, node, &cont);
1567 }
1568 
1569 void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
1570  FlagsContinuation cont =
1571  FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
1572  VisitFloat64Compare(this, node, &cont);
1573 }
1574 
1575 void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
1576  IA32OperandGenerator g(this);
1577  Node* left = node->InputAt(0);
1578  Node* right = node->InputAt(1);
1579  Float64Matcher mleft(left);
1580  if (mleft.HasValue() && (bit_cast<uint64_t>(mleft.Value()) >> 32) == 0u) {
1581  Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
1582  return;
1583  }
1584  Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
1585  g.UseRegister(left), g.Use(right));
1586 }
1587 
1588 void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
1589  IA32OperandGenerator g(this);
1590  Node* left = node->InputAt(0);
1591  Node* right = node->InputAt(1);
1592  Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
1593  g.UseRegister(left), g.Use(right));
1594 }
1595 
1596 void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
1597  IA32OperandGenerator g(this);
1598  Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
1599  g.UseRegister(node->InputAt(0)));
1600 }
1601 
1602 void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
1603  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
1604  DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
1605  load_rep.representation() == MachineRepresentation::kWord16 ||
1606  load_rep.representation() == MachineRepresentation::kWord32);
1607  USE(load_rep);
1608  VisitLoad(node);
1609 }
1610 
1611 void InstructionSelector::VisitWord32AtomicStore(Node* node) {
1612  IA32OperandGenerator g(this);
1613  MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
1614  ArchOpcode opcode = kArchNop;
1615  switch (rep) {
1616  case MachineRepresentation::kWord8:
1617  opcode = kWord32AtomicExchangeInt8;
1618  break;
1619  case MachineRepresentation::kWord16:
1620  opcode = kWord32AtomicExchangeInt16;
1621  break;
1622  case MachineRepresentation::kWord32:
1623  opcode = kWord32AtomicExchangeWord32;
1624  break;
1625  default:
1626  UNREACHABLE();
1627  break;
1628  }
1629  VisitAtomicExchange(this, node, opcode, rep);
1630 }
1631 
1632 void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
1633  IA32OperandGenerator g(this);
1634  MachineType type = AtomicOpType(node->op());
1635  ArchOpcode opcode = kArchNop;
1636  if (type == MachineType::Int8()) {
1637  opcode = kWord32AtomicExchangeInt8;
1638  } else if (type == MachineType::Uint8()) {
1639  opcode = kWord32AtomicExchangeUint8;
1640  } else if (type == MachineType::Int16()) {
1641  opcode = kWord32AtomicExchangeInt16;
1642  } else if (type == MachineType::Uint16()) {
1643  opcode = kWord32AtomicExchangeUint16;
1644  } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1645  opcode = kWord32AtomicExchangeWord32;
1646  } else {
1647  UNREACHABLE();
1648  return;
1649  }
1650  VisitAtomicExchange(this, node, opcode, type.representation());
1651 }
1652 
1653 void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
1654  IA32OperandGenerator g(this);
1655  Node* base = node->InputAt(0);
1656  Node* index = node->InputAt(1);
1657  Node* old_value = node->InputAt(2);
1658  Node* new_value = node->InputAt(3);
1659 
1660  MachineType type = AtomicOpType(node->op());
1661  ArchOpcode opcode = kArchNop;
1662  if (type == MachineType::Int8()) {
1663  opcode = kWord32AtomicCompareExchangeInt8;
1664  } else if (type == MachineType::Uint8()) {
1665  opcode = kWord32AtomicCompareExchangeUint8;
1666  } else if (type == MachineType::Int16()) {
1667  opcode = kWord32AtomicCompareExchangeInt16;
1668  } else if (type == MachineType::Uint16()) {
1669  opcode = kWord32AtomicCompareExchangeUint16;
1670  } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1671  opcode = kWord32AtomicCompareExchangeWord32;
1672  } else {
1673  UNREACHABLE();
1674  return;
1675  }
1676  AddressingMode addressing_mode;
1677  InstructionOperand new_val_operand =
1678  (type.representation() == MachineRepresentation::kWord8)
1679  ? g.UseByteRegister(new_value)
1680  : g.UseUniqueRegister(new_value);
1681  InstructionOperand inputs[] = {
1682  g.UseFixed(old_value, eax), new_val_operand, g.UseUniqueRegister(base),
1683  g.GetEffectiveIndexOperand(index, &addressing_mode)};
1684  InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)};
1685  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1686  Emit(code, 1, outputs, arraysize(inputs), inputs);
1687 }
1688 
1689 void InstructionSelector::VisitWord32AtomicBinaryOperation(
1690  Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
1691  ArchOpcode uint16_op, ArchOpcode word32_op) {
1692  MachineType type = AtomicOpType(node->op());
1693  ArchOpcode opcode = kArchNop;
1694  if (type == MachineType::Int8()) {
1695  opcode = int8_op;
1696  } else if (type == MachineType::Uint8()) {
1697  opcode = uint8_op;
1698  } else if (type == MachineType::Int16()) {
1699  opcode = int16_op;
1700  } else if (type == MachineType::Uint16()) {
1701  opcode = uint16_op;
1702  } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1703  opcode = word32_op;
1704  } else {
1705  UNREACHABLE();
1706  return;
1707  }
1708  VisitAtomicBinOp(this, node, opcode, type.representation());
1709 }
1710 
1711 #define VISIT_ATOMIC_BINOP(op) \
1712  void InstructionSelector::VisitWord32Atomic##op(Node* node) { \
1713  VisitWord32AtomicBinaryOperation( \
1714  node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \
1715  kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \
1716  kWord32Atomic##op##Word32); \
1717  }
1718 VISIT_ATOMIC_BINOP(Add)
1719 VISIT_ATOMIC_BINOP(Sub)
1720 VISIT_ATOMIC_BINOP(And)
1721 VISIT_ATOMIC_BINOP(Or)
1722 VISIT_ATOMIC_BINOP(Xor)
1723 #undef VISIT_ATOMIC_BINOP
1724 
1725 void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
1726  IA32OperandGenerator g(this);
1727  AddressingMode mode;
1728  Node* base = node->InputAt(0);
1729  Node* index = node->InputAt(1);
1730  InstructionOperand inputs[] = {g.UseUniqueRegister(base),
1731  g.GetEffectiveIndexOperand(index, &mode)};
1732  Node* projection0 = NodeProperties::FindProjection(node, 0);
1733  Node* projection1 = NodeProperties::FindProjection(node, 1);
1734  InstructionCode code =
1735  kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode);
1736 
1737  if (projection1) {
1738  InstructionOperand temps[] = {g.TempDoubleRegister()};
1739  InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
1740  g.DefineAsRegister(projection1)};
1741  Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1742  arraysize(temps), temps);
1743  } else if (projection0) {
1744  InstructionOperand temps[] = {g.TempDoubleRegister(), g.TempRegister()};
1745  InstructionOperand outputs[] = {g.DefineAsRegister(projection0)};
1746  Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1747  arraysize(temps), temps);
1748  } else {
1749  InstructionOperand temps[] = {g.TempDoubleRegister(), g.TempRegister(),
1750  g.TempRegister()};
1751  Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
1752  }
1753 }
1754 
1755 void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
1756  IA32OperandGenerator g(this);
1757  Node* base = node->InputAt(0);
1758  Node* index = node->InputAt(1);
1759  Node* value = node->InputAt(2);
1760  Node* value_high = node->InputAt(3);
1761 
1762  AddressingMode addressing_mode;
1763  InstructionOperand inputs[] = {
1764  g.UseUniqueRegisterOrSlotOrConstant(value), g.UseFixed(value_high, ecx),
1765  g.UseUniqueRegister(base),
1766  g.GetEffectiveIndexOperand(index, &addressing_mode)};
1767  // Allocating temp registers here as stores are performed using an atomic
1768  // exchange, the output of which is stored in edx:eax, which should be saved
1769  // and restored at the end of the instruction.
1770  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
1771  const int num_temps = arraysize(temps);
1772  InstructionCode code =
1773  kIA32Word32AtomicPairStore | AddressingModeField::encode(addressing_mode);
1774  Emit(code, 0, nullptr, arraysize(inputs), inputs, num_temps, temps);
1775 }
1776 
1777 void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
1778  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAdd);
1779 }
1780 
1781 void InstructionSelector::VisitWord32AtomicPairSub(Node* node) {
1782  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairSub);
1783 }
1784 
1785 void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) {
1786  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAnd);
1787 }
1788 
1789 void InstructionSelector::VisitWord32AtomicPairOr(Node* node) {
1790  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairOr);
1791 }
1792 
1793 void InstructionSelector::VisitWord32AtomicPairXor(Node* node) {
1794  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairXor);
1795 }
1796 
1797 void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
1798  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairExchange);
1799 }
1800 
1801 void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
1802  IA32OperandGenerator g(this);
1803  Node* index = node->InputAt(1);
1804  AddressingMode addressing_mode;
1805 
1806  InstructionOperand inputs[] = {
1807  // High, Low values of old value
1808  g.UseFixed(node->InputAt(2), eax), g.UseFixed(node->InputAt(3), edx),
1809  // High, Low values of new value
1810  g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(4)),
1811  g.UseFixed(node->InputAt(5), ecx),
1812  // InputAt(0) => base
1813  g.UseUniqueRegister(node->InputAt(0)),
1814  g.GetEffectiveIndexOperand(index, &addressing_mode)};
1815  Node* projection0 = NodeProperties::FindProjection(node, 0);
1816  Node* projection1 = NodeProperties::FindProjection(node, 1);
1817  InstructionCode code = kIA32Word32AtomicPairCompareExchange |
1818  AddressingModeField::encode(addressing_mode);
1819 
1820  if (projection1) {
1821  InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax),
1822  g.DefineAsFixed(projection1, edx)};
1823  Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, 0, {});
1824  } else if (projection0) {
1825  InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax)};
1826  InstructionOperand temps[] = {g.TempRegister(edx)};
1827  const int num_temps = arraysize(temps);
1828  Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1829  num_temps, temps);
1830  } else {
1831  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
1832  const int num_temps = arraysize(temps);
1833  Emit(code, 0, nullptr, arraysize(inputs), inputs, num_temps, temps);
1834  }
1835 }
1836 
1837 #define SIMD_INT_TYPES(V) \
1838  V(I32x4) \
1839  V(I16x8) \
1840  V(I8x16)
1841 
1842 #define SIMD_BINOP_LIST(V) \
1843  V(F32x4Add) \
1844  V(F32x4AddHoriz) \
1845  V(F32x4Sub) \
1846  V(F32x4Mul) \
1847  V(F32x4Min) \
1848  V(F32x4Max) \
1849  V(F32x4Eq) \
1850  V(F32x4Ne) \
1851  V(F32x4Lt) \
1852  V(F32x4Le) \
1853  V(I32x4Add) \
1854  V(I32x4AddHoriz) \
1855  V(I32x4Sub) \
1856  V(I32x4Mul) \
1857  V(I32x4MinS) \
1858  V(I32x4MaxS) \
1859  V(I32x4Eq) \
1860  V(I32x4Ne) \
1861  V(I32x4GtS) \
1862  V(I32x4GeS) \
1863  V(I32x4MinU) \
1864  V(I32x4MaxU) \
1865  V(I32x4GtU) \
1866  V(I32x4GeU) \
1867  V(I16x8SConvertI32x4) \
1868  V(I16x8Add) \
1869  V(I16x8AddSaturateS) \
1870  V(I16x8AddHoriz) \
1871  V(I16x8Sub) \
1872  V(I16x8SubSaturateS) \
1873  V(I16x8Mul) \
1874  V(I16x8MinS) \
1875  V(I16x8MaxS) \
1876  V(I16x8Eq) \
1877  V(I16x8Ne) \
1878  V(I16x8GtS) \
1879  V(I16x8GeS) \
1880  V(I16x8AddSaturateU) \
1881  V(I16x8SubSaturateU) \
1882  V(I16x8MinU) \
1883  V(I16x8MaxU) \
1884  V(I16x8GtU) \
1885  V(I16x8GeU) \
1886  V(I8x16SConvertI16x8) \
1887  V(I8x16Add) \
1888  V(I8x16AddSaturateS) \
1889  V(I8x16Sub) \
1890  V(I8x16SubSaturateS) \
1891  V(I8x16MinS) \
1892  V(I8x16MaxS) \
1893  V(I8x16Eq) \
1894  V(I8x16Ne) \
1895  V(I8x16GtS) \
1896  V(I8x16GeS) \
1897  V(I8x16AddSaturateU) \
1898  V(I8x16SubSaturateU) \
1899  V(I8x16MinU) \
1900  V(I8x16MaxU) \
1901  V(I8x16GtU) \
1902  V(I8x16GeU) \
1903  V(S128And) \
1904  V(S128Or) \
1905  V(S128Xor)
1906 
1907 #define SIMD_UNOP_LIST(V) \
1908  V(F32x4SConvertI32x4) \
1909  V(F32x4RecipApprox) \
1910  V(F32x4RecipSqrtApprox) \
1911  V(I32x4SConvertI16x8Low) \
1912  V(I32x4SConvertI16x8High) \
1913  V(I32x4Neg) \
1914  V(I32x4UConvertI16x8Low) \
1915  V(I32x4UConvertI16x8High) \
1916  V(I16x8SConvertI8x16Low) \
1917  V(I16x8SConvertI8x16High) \
1918  V(I16x8Neg) \
1919  V(I16x8UConvertI8x16Low) \
1920  V(I16x8UConvertI8x16High) \
1921  V(I8x16Neg)
1922 
1923 #define SIMD_UNOP_PREFIX_LIST(V) \
1924  V(F32x4Abs) \
1925  V(F32x4Neg) \
1926  V(S128Not)
1927 
1928 #define SIMD_ANYTRUE_LIST(V) \
1929  V(S1x4AnyTrue) \
1930  V(S1x8AnyTrue) \
1931  V(S1x16AnyTrue)
1932 
1933 #define SIMD_ALLTRUE_LIST(V) \
1934  V(S1x4AllTrue) \
1935  V(S1x8AllTrue) \
1936  V(S1x16AllTrue)
1937 
1938 #define SIMD_SHIFT_OPCODES(V) \
1939  V(I32x4Shl) \
1940  V(I32x4ShrS) \
1941  V(I32x4ShrU) \
1942  V(I16x8Shl) \
1943  V(I16x8ShrS) \
1944  V(I16x8ShrU) \
1945  V(I8x16Shl)
1946 
1947 #define SIMD_I8X16_RIGHT_SHIFT_OPCODES(V) \
1948  V(I8x16ShrS) \
1949  V(I8x16ShrU)
1950 
1951 void InstructionSelector::VisitF32x4Splat(Node* node) {
1952  VisitRRSimd(this, node, kAVXF32x4Splat, kSSEF32x4Splat);
1953 }
1954 
1955 void InstructionSelector::VisitF32x4ExtractLane(Node* node) {
1956  VisitRRISimd(this, node, kAVXF32x4ExtractLane, kSSEF32x4ExtractLane);
1957 }
1958 
1959 void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
1960  VisitRRSimd(this, node, kAVXF32x4UConvertI32x4, kSSEF32x4UConvertI32x4);
1961 }
1962 
1963 void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
1964  VisitRRSimd(this, node, kAVXI32x4SConvertF32x4, kSSEI32x4SConvertF32x4);
1965 }
1966 
1967 void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
1968  IA32OperandGenerator g(this);
1969  InstructionOperand temps[] = {g.TempSimd128Register()};
1970  InstructionCode opcode =
1971  IsSupported(AVX) ? kAVXI32x4UConvertF32x4 : kSSEI32x4UConvertF32x4;
1972  Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
1973  arraysize(temps), temps);
1974 }
1975 
1976 void InstructionSelector::VisitI8x16Mul(Node* node) {
1977  IA32OperandGenerator g(this);
1978  InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
1979  InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1));
1980  InstructionOperand temps[] = {g.TempSimd128Register()};
1981  if (IsSupported(AVX)) {
1982  Emit(kAVXI8x16Mul, g.DefineAsRegister(node), operand0, operand1,
1983  arraysize(temps), temps);
1984  } else {
1985  Emit(kSSEI8x16Mul, g.DefineSameAsFirst(node), operand0, operand1,
1986  arraysize(temps), temps);
1987  }
1988 }
1989 
1990 void InstructionSelector::VisitS128Zero(Node* node) {
1991  IA32OperandGenerator g(this);
1992  Emit(kIA32S128Zero, g.DefineAsRegister(node));
1993 }
1994 
1995 void InstructionSelector::VisitS128Select(Node* node) {
1996  IA32OperandGenerator g(this);
1997  InstructionOperand operand2 = g.UseRegister(node->InputAt(2));
1998  if (IsSupported(AVX)) {
1999  Emit(kAVXS128Select, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
2000  g.Use(node->InputAt(1)), operand2);
2001  } else {
2002  Emit(kSSES128Select, g.DefineSameAsFirst(node),
2003  g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
2004  operand2);
2005  }
2006 }
2007 
2008 #define VISIT_SIMD_SPLAT(Type) \
2009  void InstructionSelector::Visit##Type##Splat(Node* node) { \
2010  VisitRO(this, node, kIA32##Type##Splat); \
2011  }
2012 SIMD_INT_TYPES(VISIT_SIMD_SPLAT)
2013 #undef VISIT_SIMD_SPLAT
2014 
2015 #define VISIT_SIMD_EXTRACT_LANE(Type) \
2016  void InstructionSelector::Visit##Type##ExtractLane(Node* node) { \
2017  VisitRRISimd(this, node, kIA32##Type##ExtractLane); \
2018  }
2019 SIMD_INT_TYPES(VISIT_SIMD_EXTRACT_LANE)
2020 #undef VISIT_SIMD_EXTRACT_LANE
2021 
2022 #define VISIT_SIMD_REPLACE_LANE(Type) \
2023  void InstructionSelector::Visit##Type##ReplaceLane(Node* node) { \
2024  IA32OperandGenerator g(this); \
2025  InstructionOperand operand0 = g.UseRegister(node->InputAt(0)); \
2026  InstructionOperand operand1 = \
2027  g.UseImmediate(OpParameter<int32_t>(node->op())); \
2028  InstructionOperand operand2 = g.Use(node->InputAt(1)); \
2029  if (IsSupported(AVX)) { \
2030  Emit(kAVX##Type##ReplaceLane, g.DefineAsRegister(node), operand0, \
2031  operand1, operand2); \
2032  } else { \
2033  Emit(kSSE##Type##ReplaceLane, g.DefineSameAsFirst(node), operand0, \
2034  operand1, operand2); \
2035  } \
2036  }
2037 SIMD_INT_TYPES(VISIT_SIMD_REPLACE_LANE)
2038 VISIT_SIMD_REPLACE_LANE(F32x4)
2039 #undef VISIT_SIMD_REPLACE_LANE
2040 #undef SIMD_INT_TYPES
2041 
2042 #define VISIT_SIMD_SHIFT(Opcode) \
2043  void InstructionSelector::Visit##Opcode(Node* node) { \
2044  VisitRRISimd(this, node, kAVX##Opcode, kSSE##Opcode); \
2045  }
2046 SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT)
2047 #undef VISIT_SIMD_SHIFT
2048 #undef SIMD_SHIFT_OPCODES
2049 
2050 #define VISIT_SIMD_I8X16_RIGHT_SHIFT(Op) \
2051  void InstructionSelector::Visit##Op(Node* node) { \
2052  VisitRRISimd(this, node, kIA32##Op); \
2053  }
2054 
2055 SIMD_I8X16_RIGHT_SHIFT_OPCODES(VISIT_SIMD_I8X16_RIGHT_SHIFT)
2056 #undef SIMD_I8X16_RIGHT_SHIFT_OPCODES
2057 #undef VISIT_SIMD_I8X16_RIGHT_SHIFT
2058 
2059 #define VISIT_SIMD_UNOP(Opcode) \
2060  void InstructionSelector::Visit##Opcode(Node* node) { \
2061  IA32OperandGenerator g(this); \
2062  Emit(kIA32##Opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); \
2063  }
2064 SIMD_UNOP_LIST(VISIT_SIMD_UNOP)
2065 #undef VISIT_SIMD_UNOP
2066 #undef SIMD_UNOP_LIST
2067 
2068 #define VISIT_SIMD_UNOP_PREFIX(Opcode) \
2069  void InstructionSelector::Visit##Opcode(Node* node) { \
2070  IA32OperandGenerator g(this); \
2071  InstructionCode opcode = IsSupported(AVX) ? kAVX##Opcode : kSSE##Opcode; \
2072  Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0))); \
2073  }
2074 SIMD_UNOP_PREFIX_LIST(VISIT_SIMD_UNOP_PREFIX)
2075 #undef VISIT_SIMD_UNOP_PREFIX
2076 #undef SIMD_UNOP_PREFIX_LIST
2077 
2078 #define VISIT_SIMD_ANYTRUE(Opcode) \
2079  void InstructionSelector::Visit##Opcode(Node* node) { \
2080  IA32OperandGenerator g(this); \
2081  InstructionOperand temps[] = {g.TempRegister()}; \
2082  Emit(kIA32##Opcode, g.DefineAsRegister(node), \
2083  g.UseRegister(node->InputAt(0)), arraysize(temps), temps); \
2084  }
2085 SIMD_ANYTRUE_LIST(VISIT_SIMD_ANYTRUE)
2086 #undef VISIT_SIMD_ANYTRUE
2087 #undef SIMD_ANYTRUE_LIST
2088 
2089 #define VISIT_SIMD_ALLTRUE(Opcode) \
2090  void InstructionSelector::Visit##Opcode(Node* node) { \
2091  IA32OperandGenerator g(this); \
2092  InstructionOperand temps[] = {g.TempRegister()}; \
2093  Emit(kIA32##Opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)), \
2094  arraysize(temps), temps); \
2095  }
2096 SIMD_ALLTRUE_LIST(VISIT_SIMD_ALLTRUE)
2097 #undef VISIT_SIMD_ALLTRUE
2098 #undef SIMD_ALLTRUE_LIST
2099 
2100 #define VISIT_SIMD_BINOP(Opcode) \
2101  void InstructionSelector::Visit##Opcode(Node* node) { \
2102  VisitRROFloat(this, node, kAVX##Opcode, kSSE##Opcode); \
2103  }
2104 SIMD_BINOP_LIST(VISIT_SIMD_BINOP)
2105 #undef VISIT_SIMD_BINOP
2106 #undef SIMD_BINOP_LIST
2107 
2108 void VisitPack(InstructionSelector* selector, Node* node, ArchOpcode avx_opcode,
2109  ArchOpcode sse_opcode) {
2110  IA32OperandGenerator g(selector);
2111  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
2112  InstructionOperand operand1 = g.Use(node->InputAt(1));
2113  if (selector->IsSupported(AVX)) {
2114  selector->Emit(avx_opcode, g.DefineSameAsFirst(node), operand0, operand1);
2115  } else {
2116  selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
2117  }
2118 }
2119 
2120 void InstructionSelector::VisitI16x8UConvertI32x4(Node* node) {
2121  VisitPack(this, node, kAVXI16x8UConvertI32x4, kSSEI16x8UConvertI32x4);
2122 }
2123 
2124 void InstructionSelector::VisitI8x16UConvertI16x8(Node* node) {
2125  VisitPack(this, node, kAVXI8x16UConvertI16x8, kSSEI8x16UConvertI16x8);
2126 }
2127 
2128 void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
2129  UNREACHABLE();
2130 }
2131 
2132 void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
2133  UNREACHABLE();
2134 }
2135 
2136 namespace {
2137 
2138 // Packs a 4 lane shuffle into a single imm8 suitable for use by pshufd,
2139 // pshuflw, and pshufhw.
2140 uint8_t PackShuffle4(uint8_t* shuffle) {
2141  return (shuffle[0] & 3) | ((shuffle[1] & 3) << 2) | ((shuffle[2] & 3) << 4) |
2142  ((shuffle[3] & 3) << 6);
2143 }
2144 
2145 // Gets an 8 bit lane mask suitable for 16x8 pblendw.
2146 uint8_t PackBlend8(const uint8_t* shuffle16x8) {
2147  int8_t result = 0;
2148  for (int i = 0; i < 8; ++i) {
2149  result |= (shuffle16x8[i] >= 8 ? 1 : 0) << i;
2150  }
2151  return result;
2152 }
2153 
2154 // Gets an 8 bit lane mask suitable for 32x4 pblendw.
2155 uint8_t PackBlend4(const uint8_t* shuffle32x4) {
2156  int8_t result = 0;
2157  for (int i = 0; i < 4; ++i) {
2158  result |= (shuffle32x4[i] >= 4 ? 0x3 : 0) << (i * 2);
2159  }
2160  return result;
2161 }
2162 
2163 // Returns true if shuffle can be decomposed into two 16x4 half shuffles
2164 // followed by a 16x8 blend.
2165 // E.g. [3 2 1 0 15 14 13 12].
2166 bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) {
2167  *blend_mask = 0;
2168  for (int i = 0; i < 8; i++) {
2169  if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false;
2170  *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i;
2171  }
2172  return true;
2173 }
2174 
2175 struct ShuffleEntry {
2176  uint8_t shuffle[kSimd128Size];
2177  ArchOpcode opcode;
2178  ArchOpcode avx_opcode;
2179  bool src0_needs_reg;
2180  bool src1_needs_reg;
2181 };
2182 
2183 // Shuffles that map to architecture-specific instruction sequences. These are
2184 // matched very early, so we shouldn't include shuffles that match better in
2185 // later tests, like 32x4 and 16x8 shuffles. In general, these patterns should
2186 // map to either a single instruction, or be finer grained, such as zip/unzip or
2187 // transpose patterns.
2188 static const ShuffleEntry arch_shuffles[] = {
2189  {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23},
2190  kIA32S64x2UnpackLow,
2191  kIA32S64x2UnpackLow,
2192  true,
2193  false},
2194  {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31},
2195  kIA32S64x2UnpackHigh,
2196  kIA32S64x2UnpackHigh,
2197  true,
2198  false},
2199  {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
2200  kIA32S32x4UnpackLow,
2201  kIA32S32x4UnpackLow,
2202  true,
2203  false},
2204  {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
2205  kIA32S32x4UnpackHigh,
2206  kIA32S32x4UnpackHigh,
2207  true,
2208  false},
2209  {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
2210  kIA32S16x8UnpackLow,
2211  kIA32S16x8UnpackLow,
2212  true,
2213  false},
2214  {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
2215  kIA32S16x8UnpackHigh,
2216  kIA32S16x8UnpackHigh,
2217  true,
2218  false},
2219  {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
2220  kIA32S8x16UnpackLow,
2221  kIA32S8x16UnpackLow,
2222  true,
2223  false},
2224  {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
2225  kIA32S8x16UnpackHigh,
2226  kIA32S8x16UnpackHigh,
2227  true,
2228  false},
2229 
2230  {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
2231  kSSES16x8UnzipLow,
2232  kAVXS16x8UnzipLow,
2233  true,
2234  false},
2235  {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
2236  kSSES16x8UnzipHigh,
2237  kAVXS16x8UnzipHigh,
2238  true,
2239  true},
2240  {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
2241  kSSES8x16UnzipLow,
2242  kAVXS8x16UnzipLow,
2243  true,
2244  true},
2245  {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
2246  kSSES8x16UnzipHigh,
2247  kAVXS8x16UnzipHigh,
2248  true,
2249  true},
2250 
2251  {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
2252  kSSES8x16TransposeLow,
2253  kAVXS8x16TransposeLow,
2254  true,
2255  true},
2256  {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
2257  kSSES8x16TransposeHigh,
2258  kAVXS8x16TransposeHigh,
2259  true,
2260  true},
2261  {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8},
2262  kSSES8x8Reverse,
2263  kAVXS8x8Reverse,
2264  false,
2265  false},
2266  {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12},
2267  kSSES8x4Reverse,
2268  kAVXS8x4Reverse,
2269  false,
2270  false},
2271  {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
2272  kSSES8x2Reverse,
2273  kAVXS8x2Reverse,
2274  true,
2275  true}};
2276 
2277 bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
2278  size_t num_entries, bool is_swizzle,
2279  const ShuffleEntry** arch_shuffle) {
2280  uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1;
2281  for (size_t i = 0; i < num_entries; ++i) {
2282  const ShuffleEntry& entry = table[i];
2283  int j = 0;
2284  for (; j < kSimd128Size; ++j) {
2285  if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
2286  break;
2287  }
2288  }
2289  if (j == kSimd128Size) {
2290  *arch_shuffle = &entry;
2291  return true;
2292  }
2293  }
2294  return false;
2295 }
2296 
2297 } // namespace
2298 
2299 void InstructionSelector::VisitS8x16Shuffle(Node* node) {
2300  uint8_t shuffle[kSimd128Size];
2301  bool is_swizzle;
2302  CanonicalizeShuffle(node, shuffle, &is_swizzle);
2303 
2304  int imm_count = 0;
2305  static const int kMaxImms = 6;
2306  uint32_t imms[kMaxImms];
2307  int temp_count = 0;
2308  static const int kMaxTemps = 2;
2309  InstructionOperand temps[kMaxTemps];
2310 
2311  IA32OperandGenerator g(this);
2312  bool use_avx = CpuFeatures::IsSupported(AVX);
2313  // AVX and swizzles don't generally need DefineSameAsFirst to avoid a move.
2314  bool no_same_as_first = use_avx || is_swizzle;
2315  // We generally need UseRegister for input0, Use for input1.
2316  bool src0_needs_reg = true;
2317  bool src1_needs_reg = false;
2318  ArchOpcode opcode = kIA32S8x16Shuffle; // general shuffle is the default
2319 
2320  uint8_t offset;
2321  uint8_t shuffle32x4[4];
2322  uint8_t shuffle16x8[8];
2323  int index;
2324  const ShuffleEntry* arch_shuffle;
2325  if (TryMatchConcat(shuffle, &offset)) {
2326  // Swap inputs from the normal order for (v)palignr.
2327  SwapShuffleInputs(node);
2328  is_swizzle = false; // It's simpler to just handle the general case.
2329  no_same_as_first = use_avx; // SSE requires same-as-first.
2330  opcode = kIA32S8x16Alignr;
2331  // palignr takes a single imm8 offset.
2332  imms[imm_count++] = offset;
2333  } else if (TryMatchArchShuffle(shuffle, arch_shuffles,
2334  arraysize(arch_shuffles), is_swizzle,
2335  &arch_shuffle)) {
2336  opcode = use_avx ? arch_shuffle->avx_opcode : arch_shuffle->opcode;
2337  src0_needs_reg = !use_avx || arch_shuffle->src0_needs_reg;
2338  // SSE can't take advantage of both operands in registers and needs
2339  // same-as-first.
2340  src1_needs_reg = use_avx && arch_shuffle->src1_needs_reg;
2341  no_same_as_first = use_avx;
2342  } else if (TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
2343  uint8_t shuffle_mask = PackShuffle4(shuffle32x4);
2344  if (is_swizzle) {
2345  if (TryMatchIdentity(shuffle)) {
2346  // Bypass normal shuffle code generation in this case.
2347  EmitIdentity(node);
2348  return;
2349  } else {
2350  // pshufd takes a single imm8 shuffle mask.
2351  opcode = kIA32S32x4Swizzle;
2352  no_same_as_first = true;
2353  src0_needs_reg = false;
2354  imms[imm_count++] = shuffle_mask;
2355  }
2356  } else {
2357  // 2 operand shuffle
2358  // A blend is more efficient than a general 32x4 shuffle; try it first.
2359  if (TryMatchBlend(shuffle)) {
2360  opcode = kIA32S16x8Blend;
2361  uint8_t blend_mask = PackBlend4(shuffle32x4);
2362  imms[imm_count++] = blend_mask;
2363  } else {
2364  opcode = kIA32S32x4Shuffle;
2365  no_same_as_first = true;
2366  src0_needs_reg = false;
2367  imms[imm_count++] = shuffle_mask;
2368  int8_t blend_mask = PackBlend4(shuffle32x4);
2369  imms[imm_count++] = blend_mask;
2370  }
2371  }
2372  } else if (TryMatch16x8Shuffle(shuffle, shuffle16x8)) {
2373  uint8_t blend_mask;
2374  if (TryMatchBlend(shuffle)) {
2375  opcode = kIA32S16x8Blend;
2376  blend_mask = PackBlend8(shuffle16x8);
2377  imms[imm_count++] = blend_mask;
2378  } else if (TryMatchDup<8>(shuffle, &index)) {
2379  opcode = kIA32S16x8Dup;
2380  src0_needs_reg = false;
2381  imms[imm_count++] = index;
2382  } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) {
2383  opcode = is_swizzle ? kIA32S16x8HalfShuffle1 : kIA32S16x8HalfShuffle2;
2384  // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0).
2385  no_same_as_first = true;
2386  src0_needs_reg = false;
2387  uint8_t mask_lo = PackShuffle4(shuffle16x8);
2388  uint8_t mask_hi = PackShuffle4(shuffle16x8 + 4);
2389  imms[imm_count++] = mask_lo;
2390  imms[imm_count++] = mask_hi;
2391  if (!is_swizzle) imms[imm_count++] = blend_mask;
2392  }
2393  } else if (TryMatchDup<16>(shuffle, &index)) {
2394  opcode = kIA32S8x16Dup;
2395  no_same_as_first = use_avx;
2396  src0_needs_reg = true;
2397  imms[imm_count++] = index;
2398  }
2399  if (opcode == kIA32S8x16Shuffle) {
2400  // Use same-as-first for general swizzle, but not shuffle.
2401  no_same_as_first = !is_swizzle;
2402  src0_needs_reg = !no_same_as_first;
2403  imms[imm_count++] = Pack4Lanes(shuffle);
2404  imms[imm_count++] = Pack4Lanes(shuffle + 4);
2405  imms[imm_count++] = Pack4Lanes(shuffle + 8);
2406  imms[imm_count++] = Pack4Lanes(shuffle + 12);
2407  temps[temp_count++] = g.TempRegister();
2408  }
2409 
2410  // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra
2411  // move instruction in the CodeGenerator.
2412  Node* input0 = node->InputAt(0);
2413  InstructionOperand dst =
2414  no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
2415  InstructionOperand src0 =
2416  src0_needs_reg ? g.UseRegister(input0) : g.Use(input0);
2417 
2418  int input_count = 0;
2419  InstructionOperand inputs[2 + kMaxImms + kMaxTemps];
2420  inputs[input_count++] = src0;
2421  if (!is_swizzle) {
2422  Node* input1 = node->InputAt(1);
2423  inputs[input_count++] =
2424  src1_needs_reg ? g.UseRegister(input1) : g.Use(input1);
2425  }
2426  for (int i = 0; i < imm_count; ++i) {
2427  inputs[input_count++] = g.UseImmediate(imms[i]);
2428  }
2429  Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps);
2430 }
2431 
2432 // static
2433 MachineOperatorBuilder::Flags
2434 InstructionSelector::SupportedMachineOperatorFlags() {
2435  MachineOperatorBuilder::Flags flags =
2436  MachineOperatorBuilder::kWord32ShiftIsSafe |
2437  MachineOperatorBuilder::kWord32Ctz |
2438  MachineOperatorBuilder::kSpeculationFence;
2439  if (CpuFeatures::IsSupported(POPCNT)) {
2440  flags |= MachineOperatorBuilder::kWord32Popcnt;
2441  }
2442  if (CpuFeatures::IsSupported(SSE4_1)) {
2443  flags |= MachineOperatorBuilder::kFloat32RoundDown |
2444  MachineOperatorBuilder::kFloat64RoundDown |
2445  MachineOperatorBuilder::kFloat32RoundUp |
2446  MachineOperatorBuilder::kFloat64RoundUp |
2447  MachineOperatorBuilder::kFloat32RoundTruncate |
2448  MachineOperatorBuilder::kFloat64RoundTruncate |
2449  MachineOperatorBuilder::kFloat32RoundTiesEven |
2450  MachineOperatorBuilder::kFloat64RoundTiesEven;
2451  }
2452  return flags;
2453 }
2454 
2455 // static
2456 MachineOperatorBuilder::AlignmentRequirements
2457 InstructionSelector::AlignmentRequirements() {
2458  return MachineOperatorBuilder::AlignmentRequirements::
2459  FullUnalignedAccessSupport();
2460 }
2461 
2462 } // namespace compiler
2463 } // namespace internal
2464 } // namespace v8
int64_t
v8
Definition: libplatform.h:13
uint32_t