V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
simd-scalar-lowering.cc
1 // Copyright 2016 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/simd-scalar-lowering.h"
6 
7 #include "src/compiler/diamond.h"
8 #include "src/compiler/linkage.h"
9 #include "src/compiler/node-matchers.h"
10 #include "src/compiler/node-properties.h"
11 #include "src/compiler/node.h"
12 #include "src/compiler/wasm-compiler.h"
13 
14 namespace v8 {
15 namespace internal {
16 namespace compiler {
17 
18 namespace {
19 static const int kNumLanes32 = 4;
20 static const int kNumLanes16 = 8;
21 static const int kNumLanes8 = 16;
22 static const int32_t kMask16 = 0xFFFF;
23 static const int32_t kMask8 = 0xFF;
24 static const int32_t kShift16 = 16;
25 static const int32_t kShift8 = 24;
26 } // anonymous
27 
28 SimdScalarLowering::SimdScalarLowering(
29  MachineGraph* mcgraph, Signature<MachineRepresentation>* signature)
30  : mcgraph_(mcgraph),
31  state_(mcgraph->graph(), 3),
32  stack_(mcgraph_->zone()),
33  replacements_(nullptr),
34  signature_(signature),
35  placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
36  graph()->start())),
37  parameter_count_after_lowering_(-1) {
38  DCHECK_NOT_NULL(graph());
39  DCHECK_NOT_NULL(graph()->end());
40  replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
41  memset(static_cast<void*>(replacements_), 0,
42  sizeof(Replacement) * graph()->NodeCount());
43 }
44 
45 void SimdScalarLowering::LowerGraph() {
46  stack_.push_back({graph()->end(), 0});
47  state_.Set(graph()->end(), State::kOnStack);
48  replacements_[graph()->end()->id()].type = SimdType::kInt32x4;
49 
50  while (!stack_.empty()) {
51  NodeState& top = stack_.back();
52  if (top.input_index == top.node->InputCount()) {
53  // All inputs of top have already been lowered, now lower top.
54  stack_.pop_back();
55  state_.Set(top.node, State::kVisited);
56  LowerNode(top.node);
57  } else {
58  // Push the next input onto the stack.
59  Node* input = top.node->InputAt(top.input_index++);
60  if (state_.Get(input) == State::kUnvisited) {
61  SetLoweredType(input, top.node);
62  if (input->opcode() == IrOpcode::kPhi) {
63  // To break cycles with phi nodes we push phis on a separate stack so
64  // that they are processed after all other nodes.
65  PreparePhiReplacement(input);
66  stack_.push_front({input, 0});
67  } else if (input->opcode() == IrOpcode::kEffectPhi ||
68  input->opcode() == IrOpcode::kLoop) {
69  stack_.push_front({input, 0});
70  } else {
71  stack_.push_back({input, 0});
72  }
73  state_.Set(input, State::kOnStack);
74  }
75  }
76  }
77 }
78 
79 #define FOREACH_INT32X4_OPCODE(V) \
80  V(I32x4Splat) \
81  V(I32x4ExtractLane) \
82  V(I32x4ReplaceLane) \
83  V(I32x4SConvertF32x4) \
84  V(I32x4UConvertF32x4) \
85  V(I32x4SConvertI16x8Low) \
86  V(I32x4SConvertI16x8High) \
87  V(I32x4Neg) \
88  V(I32x4Shl) \
89  V(I32x4ShrS) \
90  V(I32x4Add) \
91  V(I32x4AddHoriz) \
92  V(I32x4Sub) \
93  V(I32x4Mul) \
94  V(I32x4MinS) \
95  V(I32x4MaxS) \
96  V(I32x4ShrU) \
97  V(I32x4MinU) \
98  V(I32x4MaxU) \
99  V(I32x4Eq) \
100  V(I32x4Ne) \
101  V(I32x4LtS) \
102  V(I32x4LeS) \
103  V(I32x4GtS) \
104  V(I32x4GeS) \
105  V(I32x4UConvertI16x8Low) \
106  V(I32x4UConvertI16x8High) \
107  V(I32x4LtU) \
108  V(I32x4LeU) \
109  V(I32x4GtU) \
110  V(I32x4GeU) \
111  V(S128And) \
112  V(S128Or) \
113  V(S128Xor) \
114  V(S128Not) \
115  V(S1x4AnyTrue) \
116  V(S1x4AllTrue) \
117  V(S1x8AnyTrue) \
118  V(S1x8AllTrue) \
119  V(S1x16AnyTrue) \
120  V(S1x16AllTrue)
121 
122 #define FOREACH_FLOAT32X4_OPCODE(V) \
123  V(F32x4Splat) \
124  V(F32x4ExtractLane) \
125  V(F32x4ReplaceLane) \
126  V(F32x4SConvertI32x4) \
127  V(F32x4UConvertI32x4) \
128  V(F32x4Abs) \
129  V(F32x4Neg) \
130  V(F32x4RecipApprox) \
131  V(F32x4RecipSqrtApprox) \
132  V(F32x4Add) \
133  V(F32x4AddHoriz) \
134  V(F32x4Sub) \
135  V(F32x4Mul) \
136  V(F32x4Min) \
137  V(F32x4Max)
138 
139 #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \
140  V(F32x4Eq) \
141  V(F32x4Ne) \
142  V(F32x4Lt) \
143  V(F32x4Le) \
144  V(F32x4Gt) \
145  V(F32x4Ge)
146 
147 #define FOREACH_INT16X8_OPCODE(V) \
148  V(I16x8Splat) \
149  V(I16x8ExtractLane) \
150  V(I16x8ReplaceLane) \
151  V(I16x8SConvertI8x16Low) \
152  V(I16x8SConvertI8x16High) \
153  V(I16x8Neg) \
154  V(I16x8Shl) \
155  V(I16x8ShrS) \
156  V(I16x8SConvertI32x4) \
157  V(I16x8Add) \
158  V(I16x8AddSaturateS) \
159  V(I16x8AddHoriz) \
160  V(I16x8Sub) \
161  V(I16x8SubSaturateS) \
162  V(I16x8Mul) \
163  V(I16x8MinS) \
164  V(I16x8MaxS) \
165  V(I16x8UConvertI8x16Low) \
166  V(I16x8UConvertI8x16High) \
167  V(I16x8ShrU) \
168  V(I16x8UConvertI32x4) \
169  V(I16x8AddSaturateU) \
170  V(I16x8SubSaturateU) \
171  V(I16x8MinU) \
172  V(I16x8MaxU) \
173  V(I16x8Eq) \
174  V(I16x8Ne) \
175  V(I16x8LtS) \
176  V(I16x8LeS) \
177  V(I16x8LtU) \
178  V(I16x8LeU)
179 
180 #define FOREACH_INT8X16_OPCODE(V) \
181  V(I8x16Splat) \
182  V(I8x16ExtractLane) \
183  V(I8x16ReplaceLane) \
184  V(I8x16SConvertI16x8) \
185  V(I8x16Neg) \
186  V(I8x16Shl) \
187  V(I8x16ShrS) \
188  V(I8x16Add) \
189  V(I8x16AddSaturateS) \
190  V(I8x16Sub) \
191  V(I8x16SubSaturateS) \
192  V(I8x16Mul) \
193  V(I8x16MinS) \
194  V(I8x16MaxS) \
195  V(I8x16ShrU) \
196  V(I8x16UConvertI16x8) \
197  V(I8x16AddSaturateU) \
198  V(I8x16SubSaturateU) \
199  V(I8x16MinU) \
200  V(I8x16MaxU) \
201  V(I8x16Eq) \
202  V(I8x16Ne) \
203  V(I8x16LtS) \
204  V(I8x16LeS) \
205  V(I8x16LtU) \
206  V(I8x16LeU) \
207  V(S8x16Shuffle)
208 
209 MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) {
210  switch (simdType) {
211  case SimdType::kFloat32x4:
212  return MachineType::Float32();
213  case SimdType::kInt32x4:
214  return MachineType::Int32();
215  case SimdType::kInt16x8:
216  return MachineType::Int16();
217  case SimdType::kInt8x16:
218  return MachineType::Int8();
219  }
220  return MachineType::None();
221 }
222 
223 void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
224  switch (node->opcode()) {
225 #define CASE_STMT(name) case IrOpcode::k##name:
226  FOREACH_INT32X4_OPCODE(CASE_STMT)
227  case IrOpcode::kReturn:
228  case IrOpcode::kParameter:
229  case IrOpcode::kCall: {
230  replacements_[node->id()].type = SimdType::kInt32x4;
231  break;
232  }
233  FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
234  replacements_[node->id()].type = SimdType::kFloat32x4;
235  break;
236  }
237  FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) {
238  replacements_[node->id()].type = SimdType::kInt32x4;
239  break;
240  }
241  FOREACH_INT16X8_OPCODE(CASE_STMT) {
242  replacements_[node->id()].type = SimdType::kInt16x8;
243  break;
244  }
245  FOREACH_INT8X16_OPCODE(CASE_STMT) {
246  replacements_[node->id()].type = SimdType::kInt8x16;
247  break;
248  }
249  default: {
250  switch (output->opcode()) {
251  case IrOpcode::kF32x4SConvertI32x4:
252  case IrOpcode::kF32x4UConvertI32x4:
253  case IrOpcode::kI16x8SConvertI32x4:
254  case IrOpcode::kI16x8UConvertI32x4: {
255  replacements_[node->id()].type = SimdType::kInt32x4;
256  break;
257  }
258  case IrOpcode::kI8x16SConvertI16x8:
259  case IrOpcode::kI8x16UConvertI16x8:
260  case IrOpcode::kI32x4SConvertI16x8Low:
261  case IrOpcode::kI32x4SConvertI16x8High:
262  case IrOpcode::kI32x4UConvertI16x8Low:
263  case IrOpcode::kI32x4UConvertI16x8High: {
264  replacements_[node->id()].type = SimdType::kInt16x8;
265  break;
266  }
267  case IrOpcode::kI16x8SConvertI8x16Low:
268  case IrOpcode::kI16x8SConvertI8x16High:
269  case IrOpcode::kI16x8UConvertI8x16Low:
270  case IrOpcode::kI16x8UConvertI8x16High: {
271  replacements_[node->id()].type = SimdType::kInt8x16;
272  break;
273  }
274  FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT)
275  case IrOpcode::kI32x4SConvertF32x4:
276  case IrOpcode::kI32x4UConvertF32x4: {
277  replacements_[node->id()].type = SimdType::kFloat32x4;
278  break;
279  }
280  case IrOpcode::kS128Select: {
281  replacements_[node->id()].type = SimdType::kInt32x4;
282  break;
283  }
284  default: {
285  replacements_[node->id()].type = replacements_[output->id()].type;
286  }
287  }
288  }
289 #undef CASE_STMT
290  }
291 }
292 
293 static int GetParameterIndexAfterLoweringSimd128(
294  Signature<MachineRepresentation>* signature, int old_index) {
295  // In function calls, the simd128 types are passed as 4 Int32 types. The
296  // parameters are typecast to the types as needed for various operations.
297  int result = old_index;
298  for (int i = 0; i < old_index; ++i) {
299  if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
300  result += 3;
301  }
302  }
303  return result;
304 }
305 
306 int SimdScalarLowering::GetParameterCountAfterLowering() {
307  if (parameter_count_after_lowering_ == -1) {
308  // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the
309  // parameter count after lowering.
310  parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128(
311  signature(), static_cast<int>(signature()->parameter_count()));
312  }
313  return parameter_count_after_lowering_;
314 }
315 
316 static int GetReturnCountAfterLoweringSimd128(
317  Signature<MachineRepresentation>* signature) {
318  int result = static_cast<int>(signature->return_count());
319  for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
320  if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
321  result += 3;
322  }
323  }
324  return result;
325 }
326 
327 int SimdScalarLowering::NumLanes(SimdType type) {
328  int num_lanes = 0;
329  if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) {
330  num_lanes = kNumLanes32;
331  } else if (type == SimdType::kInt16x8) {
332  num_lanes = kNumLanes16;
333  } else if (type == SimdType::kInt8x16) {
334  num_lanes = kNumLanes8;
335  } else {
336  UNREACHABLE();
337  }
338  return num_lanes;
339 }
340 
341 constexpr int SimdScalarLowering::kLaneOffsets[];
342 
343 void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices,
344  SimdType type) {
345  int num_lanes = NumLanes(type);
346  int lane_width = kSimd128Size / num_lanes;
347  int laneIndex = kLaneOffsets[0] / lane_width;
348  new_indices[laneIndex] = index;
349  for (int i = 1; i < num_lanes; ++i) {
350  laneIndex = kLaneOffsets[i * lane_width] / lane_width;
351  new_indices[laneIndex] = graph()->NewNode(
352  machine()->Int32Add(), index,
353  graph()->NewNode(
354  common()->Int32Constant(static_cast<int>(i) * lane_width)));
355  }
356 }
357 
358 void SimdScalarLowering::LowerLoadOp(Node* node, SimdType type) {
359  MachineRepresentation rep = LoadRepresentationOf(node->op()).representation();
360  const Operator* load_op;
361  switch (node->opcode()) {
362  case IrOpcode::kLoad:
363  load_op = machine()->Load(MachineTypeFrom(type));
364  break;
365  case IrOpcode::kUnalignedLoad:
366  load_op = machine()->UnalignedLoad(MachineTypeFrom(type));
367  break;
368  case IrOpcode::kProtectedLoad:
369  load_op = machine()->ProtectedLoad(MachineTypeFrom(type));
370  break;
371  default:
372  UNREACHABLE();
373  }
374  if (rep == MachineRepresentation::kSimd128) {
375  Node* base = node->InputAt(0);
376  Node* index = node->InputAt(1);
377  int num_lanes = NumLanes(type);
378  Node** indices = zone()->NewArray<Node*>(num_lanes);
379  GetIndexNodes(index, indices, type);
380  Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
381  rep_nodes[0] = node;
382  rep_nodes[0]->ReplaceInput(1, indices[0]);
383  NodeProperties::ChangeOp(rep_nodes[0], load_op);
384  if (node->InputCount() > 2) {
385  DCHECK_LT(3, node->InputCount());
386  Node* effect_input = node->InputAt(2);
387  Node* control_input = node->InputAt(3);
388  for (int i = num_lanes - 1; i > 0; --i) {
389  rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
390  control_input);
391  effect_input = rep_nodes[i];
392  }
393  rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
394  } else {
395  for (int i = 1; i < num_lanes; ++i) {
396  rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
397  }
398  }
399  ReplaceNode(node, rep_nodes, num_lanes);
400  } else {
401  DefaultLowering(node);
402  }
403 }
404 
405 void SimdScalarLowering::LowerStoreOp(Node* node) {
406  // For store operation, use replacement type of its input instead of the
407  // one of its effected node.
408  DCHECK_LT(2, node->InputCount());
409  SimdType rep_type = ReplacementType(node->InputAt(2));
410  replacements_[node->id()].type = rep_type;
411  const Operator* store_op;
412  MachineRepresentation rep;
413  switch (node->opcode()) {
414  case IrOpcode::kStore: {
415  rep = StoreRepresentationOf(node->op()).representation();
416  WriteBarrierKind write_barrier_kind =
417  StoreRepresentationOf(node->op()).write_barrier_kind();
418  store_op = machine()->Store(StoreRepresentation(
419  MachineTypeFrom(rep_type).representation(), write_barrier_kind));
420  break;
421  }
422  case IrOpcode::kUnalignedStore: {
423  rep = UnalignedStoreRepresentationOf(node->op());
424  store_op =
425  machine()->UnalignedStore(MachineTypeFrom(rep_type).representation());
426  break;
427  }
428  case IrOpcode::kProtectedStore: {
429  rep = StoreRepresentationOf(node->op()).representation();
430  store_op =
431  machine()->ProtectedStore(MachineTypeFrom(rep_type).representation());
432  break;
433  }
434  default:
435  UNREACHABLE();
436  }
437  if (rep == MachineRepresentation::kSimd128) {
438  Node* base = node->InputAt(0);
439  Node* index = node->InputAt(1);
440  int num_lanes = NumLanes(rep_type);
441  Node** indices = zone()->NewArray<Node*>(num_lanes);
442  GetIndexNodes(index, indices, rep_type);
443  Node* value = node->InputAt(2);
444  DCHECK(HasReplacement(1, value));
445  Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
446  rep_nodes[0] = node;
447  Node** rep_inputs = GetReplacementsWithType(value, rep_type);
448  rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
449  rep_nodes[0]->ReplaceInput(1, indices[0]);
450  NodeProperties::ChangeOp(node, store_op);
451  if (node->InputCount() > 3) {
452  DCHECK_LT(4, node->InputCount());
453  Node* effect_input = node->InputAt(3);
454  Node* control_input = node->InputAt(4);
455  for (int i = num_lanes - 1; i > 0; --i) {
456  rep_nodes[i] =
457  graph()->NewNode(store_op, base, indices[i], rep_inputs[i],
458  effect_input, control_input);
459  effect_input = rep_nodes[i];
460  }
461  rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
462  } else {
463  for (int i = 1; i < num_lanes; ++i) {
464  rep_nodes[i] =
465  graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
466  }
467  }
468  ReplaceNode(node, rep_nodes, num_lanes);
469  } else {
470  DefaultLowering(node);
471  }
472 }
473 
474 void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
475  const Operator* op,
476  bool not_horizontal) {
477  DCHECK_EQ(2, node->InputCount());
478  Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
479  Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
480  int num_lanes = NumLanes(input_rep_type);
481  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
482  if (not_horizontal) {
483  for (int i = 0; i < num_lanes; ++i) {
484  rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
485  }
486  } else {
487  for (int i = 0; i < num_lanes / 2; ++i) {
488  rep_node[i] = graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]);
489  rep_node[i + num_lanes / 2] =
490  graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]);
491  }
492  }
493  ReplaceNode(node, rep_node, num_lanes);
494 }
495 
496 void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type,
497  const Operator* op,
498  bool invert_inputs) {
499  DCHECK_EQ(2, node->InputCount());
500  Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
501  Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
502  int num_lanes = NumLanes(input_rep_type);
503  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
504  for (int i = 0; i < num_lanes; ++i) {
505  Node* cmp_result = nullptr;
506  if (invert_inputs) {
507  cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]);
508  } else {
509  cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]);
510  }
511  Diamond d_cmp(graph(), common(),
512  graph()->NewNode(machine()->Word32Equal(), cmp_result,
513  mcgraph_->Int32Constant(0)));
514  MachineRepresentation rep =
515  (input_rep_type == SimdType::kFloat32x4)
516  ? MachineRepresentation::kWord32
517  : MachineTypeFrom(input_rep_type).representation();
518  rep_node[i] =
519  d_cmp.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
520  }
521  ReplaceNode(node, rep_node, num_lanes);
522 }
523 
524 Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) {
525  return graph()->NewNode(machine()->Word32Sar(),
526  graph()->NewNode(machine()->Word32Shl(), input,
527  mcgraph_->Int32Constant(shift)),
528  mcgraph_->Int32Constant(shift));
529 }
530 
531 void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node,
532  SimdType input_rep_type,
533  const Operator* op,
534  bool not_horizontal) {
535  DCHECK_EQ(2, node->InputCount());
536  DCHECK(input_rep_type == SimdType::kInt16x8 ||
537  input_rep_type == SimdType::kInt8x16);
538  Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
539  Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
540  int num_lanes = NumLanes(input_rep_type);
541  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
542  int32_t shift_val =
543  (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8;
544  if (not_horizontal) {
545  for (int i = 0; i < num_lanes; ++i) {
546  rep_node[i] = FixUpperBits(
547  graph()->NewNode(op, rep_left[i], rep_right[i]), shift_val);
548  }
549  } else {
550  for (int i = 0; i < num_lanes / 2; ++i) {
551  rep_node[i] = FixUpperBits(
552  graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]),
553  shift_val);
554  rep_node[i + num_lanes / 2] = FixUpperBits(
555  graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]),
556  shift_val);
557  }
558  }
559  ReplaceNode(node, rep_node, num_lanes);
560 }
561 
562 Node* SimdScalarLowering::Mask(Node* input, int32_t mask) {
563  return graph()->NewNode(machine()->Word32And(), input,
564  mcgraph_->Int32Constant(mask));
565 }
566 
567 void SimdScalarLowering::LowerSaturateBinaryOp(Node* node,
568  SimdType input_rep_type,
569  const Operator* op,
570  bool is_signed) {
571  DCHECK_EQ(2, node->InputCount());
572  DCHECK(input_rep_type == SimdType::kInt16x8 ||
573  input_rep_type == SimdType::kInt8x16);
574  Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
575  Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
576  int32_t min = 0;
577  int32_t max = 0;
578  int32_t mask = 0;
579  int32_t shift_val = 0;
580  MachineRepresentation phi_rep;
581  if (input_rep_type == SimdType::kInt16x8) {
582  if (is_signed) {
583  min = std::numeric_limits<int16_t>::min();
584  max = std::numeric_limits<int16_t>::max();
585  } else {
586  min = std::numeric_limits<uint16_t>::min();
587  max = std::numeric_limits<uint16_t>::max();
588  }
589  mask = kMask16;
590  shift_val = kShift16;
591  phi_rep = MachineRepresentation::kWord16;
592  } else {
593  if (is_signed) {
594  min = std::numeric_limits<int8_t>::min();
595  max = std::numeric_limits<int8_t>::max();
596  } else {
597  min = std::numeric_limits<uint8_t>::min();
598  max = std::numeric_limits<uint8_t>::max();
599  }
600  mask = kMask8;
601  shift_val = kShift8;
602  phi_rep = MachineRepresentation::kWord8;
603  }
604  int num_lanes = NumLanes(input_rep_type);
605  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
606  for (int i = 0; i < num_lanes; ++i) {
607  Node* op_result = nullptr;
608  Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask);
609  Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask);
610  op_result = graph()->NewNode(op, left, right);
611  Diamond d_min(graph(), common(),
612  graph()->NewNode(machine()->Int32LessThan(), op_result,
613  mcgraph_->Int32Constant(min)));
614  rep_node[i] = d_min.Phi(phi_rep, mcgraph_->Int32Constant(min), op_result);
615  Diamond d_max(graph(), common(),
616  graph()->NewNode(machine()->Int32LessThan(),
617  mcgraph_->Int32Constant(max), rep_node[i]));
618  rep_node[i] = d_max.Phi(phi_rep, mcgraph_->Int32Constant(max), rep_node[i]);
619  rep_node[i] =
620  is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
621  }
622  ReplaceNode(node, rep_node, num_lanes);
623 }
624 
625 void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
626  const Operator* op) {
627  DCHECK_EQ(1, node->InputCount());
628  Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
629  int num_lanes = NumLanes(input_rep_type);
630  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
631  for (int i = 0; i < num_lanes; ++i) {
632  rep_node[i] = graph()->NewNode(op, rep[i]);
633  }
634  ReplaceNode(node, rep_node, num_lanes);
635 }
636 
637 void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
638  bool is_max, SimdType type) {
639  DCHECK_EQ(2, node->InputCount());
640  Node** rep_left = GetReplacementsWithType(node->InputAt(0), type);
641  Node** rep_right = GetReplacementsWithType(node->InputAt(1), type);
642  int num_lanes = NumLanes(type);
643  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
644  MachineRepresentation rep = MachineRepresentation::kNone;
645  if (type == SimdType::kInt32x4) {
646  rep = MachineRepresentation::kWord32;
647  } else if (type == SimdType::kInt16x8) {
648  rep = MachineRepresentation::kWord16;
649  } else if (type == SimdType::kInt8x16) {
650  rep = MachineRepresentation::kWord8;
651  } else {
652  UNREACHABLE();
653  }
654  for (int i = 0; i < num_lanes; ++i) {
655  Diamond d(graph(), common(),
656  graph()->NewNode(op, rep_left[i], rep_right[i]));
657  if (is_max) {
658  rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]);
659  } else {
660  rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]);
661  }
662  }
663  ReplaceNode(node, rep_node, num_lanes);
664 }
665 
666 Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
667  if (machine()->Float64RoundTruncate().IsSupported()) {
668  return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
669  } else {
670  ExternalReference ref = ExternalReference::wasm_f64_trunc();
671  Node* stack_slot =
672  graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
673  const Operator* store_op = machine()->Store(
674  StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
675  Node* effect =
676  graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(0),
677  input, graph()->start(), graph()->start());
678  Node* function = graph()->NewNode(common()->ExternalConstant(ref));
679  Node** args = zone()->NewArray<Node*>(4);
680  args[0] = function;
681  args[1] = stack_slot;
682  args[2] = effect;
683  args[3] = graph()->start();
684  Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
685  sig_builder.AddParam(MachineType::Pointer());
686  auto call_descriptor =
687  Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
688  Node* call = graph()->NewNode(common()->Call(call_descriptor), 4, args);
689  return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
690  stack_slot, mcgraph_->Int32Constant(0), call,
691  graph()->start());
692  }
693 }
694 
695 void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
696  DCHECK_EQ(1, node->InputCount());
697  Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4);
698  Node* rep_node[kNumLanes32];
699  Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
700  Node* min = graph()->NewNode(
701  common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
702  Node* max = graph()->NewNode(common()->Float64Constant(
703  static_cast<double>(is_signed ? kMaxInt : 0xFFFFFFFFu)));
704  for (int i = 0; i < kNumLanes32; ++i) {
705  Node* double_rep =
706  graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
707  Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
708  double_rep, double_rep));
709  Node* temp =
710  nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
711  Diamond min_d(graph(), common(),
712  graph()->NewNode(machine()->Float64LessThan(), temp, min));
713  temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
714  Diamond max_d(graph(), common(),
715  graph()->NewNode(machine()->Float64LessThan(), max, temp));
716  temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
717  Node* trunc = BuildF64Trunc(temp);
718  if (is_signed) {
719  rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
720  } else {
721  rep_node[i] =
722  graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
723  }
724  }
725  ReplaceNode(node, rep_node, kNumLanes32);
726 }
727 
728 void SimdScalarLowering::LowerConvertFromInt(Node* node,
729  SimdType input_rep_type,
730  SimdType output_rep_type,
731  bool is_signed, int start_index) {
732  DCHECK_EQ(1, node->InputCount());
733  Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
734 
735  int32_t mask = 0;
736  if (input_rep_type == SimdType::kInt16x8) {
737  DCHECK_EQ(output_rep_type, SimdType::kInt32x4);
738  mask = kMask16;
739  } else {
740  DCHECK_EQ(output_rep_type, SimdType::kInt16x8);
741  DCHECK_EQ(input_rep_type, SimdType::kInt8x16);
742  mask = kMask8;
743  }
744 
745  int num_lanes = NumLanes(output_rep_type);
746  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
747  for (int i = 0; i < num_lanes; ++i) {
748  rep_node[i] =
749  is_signed ? rep[i + start_index] : Mask(rep[i + start_index], mask);
750  }
751 
752  ReplaceNode(node, rep_node, num_lanes);
753 }
754 
755 void SimdScalarLowering::LowerPack(Node* node, SimdType input_rep_type,
756  SimdType output_rep_type, bool is_signed) {
757  DCHECK_EQ(2, node->InputCount());
758  Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
759  Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
760  const Operator* less_op =
761  is_signed ? machine()->Int32LessThan() : machine()->Uint32LessThan();
762  Node* min = nullptr;
763  Node* max = nullptr;
764  int32_t shift_val = 0;
765  MachineRepresentation phi_rep;
766  if (output_rep_type == SimdType::kInt16x8) {
767  DCHECK(input_rep_type == SimdType::kInt32x4);
768  if (is_signed) {
769  min = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::min());
770  max = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::max());
771  } else {
772  max = mcgraph_->Uint32Constant(std::numeric_limits<uint16_t>::max());
773  shift_val = kShift16;
774  }
775  phi_rep = MachineRepresentation::kWord16;
776  } else {
777  DCHECK(output_rep_type == SimdType::kInt8x16 &&
778  input_rep_type == SimdType::kInt16x8);
779  if (is_signed) {
780  min = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::min());
781  max = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::max());
782  } else {
783  max = mcgraph_->Uint32Constant(std::numeric_limits<uint8_t>::max());
784  shift_val = kShift8;
785  }
786  phi_rep = MachineRepresentation::kWord8;
787  }
788  int num_lanes = NumLanes(output_rep_type);
789  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
790  for (int i = 0; i < num_lanes; ++i) {
791  Node* input = nullptr;
792  if (i < num_lanes / 2)
793  input = rep_left[i];
794  else
795  input = rep_right[i - num_lanes / 2];
796  if (is_signed) {
797  Diamond d_min(graph(), common(), graph()->NewNode(less_op, input, min));
798  input = d_min.Phi(phi_rep, min, input);
799  }
800  Diamond d_max(graph(), common(), graph()->NewNode(less_op, max, input));
801  rep_node[i] = d_max.Phi(phi_rep, max, input);
802  rep_node[i] =
803  is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
804  }
805  ReplaceNode(node, rep_node, num_lanes);
806 }
807 
808 void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) {
809  DCHECK_EQ(1, node->InputCount());
810  int32_t shift_amount = OpParameter<int32_t>(node->op());
811  Node* shift_node = graph()->NewNode(common()->Int32Constant(shift_amount));
812  Node** rep = GetReplacementsWithType(node->InputAt(0), type);
813  int num_lanes = NumLanes(type);
814  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
815  for (int i = 0; i < num_lanes; ++i) {
816  rep_node[i] = rep[i];
817  switch (node->opcode()) {
818  case IrOpcode::kI8x16ShrU:
819  rep_node[i] = Mask(rep_node[i], kMask8);
820  rep_node[i] =
821  graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
822  break;
823  case IrOpcode::kI16x8ShrU:
824  rep_node[i] = Mask(rep_node[i], kMask16);
825  V8_FALLTHROUGH;
826  case IrOpcode::kI32x4ShrU:
827  rep_node[i] =
828  graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
829  break;
830  case IrOpcode::kI32x4Shl:
831  rep_node[i] =
832  graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
833  break;
834  case IrOpcode::kI16x8Shl:
835  rep_node[i] =
836  graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
837  rep_node[i] = FixUpperBits(rep_node[i], kShift16);
838  break;
839  case IrOpcode::kI8x16Shl:
840  rep_node[i] =
841  graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
842  rep_node[i] = FixUpperBits(rep_node[i], kShift8);
843  break;
844  case IrOpcode::kI32x4ShrS:
845  case IrOpcode::kI16x8ShrS:
846  case IrOpcode::kI8x16ShrS:
847  rep_node[i] =
848  graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node);
849  break;
850  default:
851  UNREACHABLE();
852  }
853  }
854  ReplaceNode(node, rep_node, num_lanes);
855 }
856 
857 void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
858  const Operator* op) {
859  DCHECK_EQ(2, node->InputCount());
860  Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
861  Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
862  int num_lanes = NumLanes(input_rep_type);
863  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
864  for (int i = 0; i < num_lanes; ++i) {
865  Diamond d(graph(), common(),
866  graph()->NewNode(op, rep_left[i], rep_right[i]));
867  MachineRepresentation rep =
868  (input_rep_type == SimdType::kFloat32x4)
869  ? MachineRepresentation::kWord32
870  : MachineTypeFrom(input_rep_type).representation();
871  rep_node[i] =
872  d.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
873  }
874  ReplaceNode(node, rep_node, num_lanes);
875 }
876 
877 void SimdScalarLowering::LowerNode(Node* node) {
878  SimdType rep_type = ReplacementType(node);
879  int num_lanes = NumLanes(rep_type);
880  switch (node->opcode()) {
881  case IrOpcode::kStart: {
882  int parameter_count = GetParameterCountAfterLowering();
883  // Only exchange the node if the parameter count actually changed.
884  if (parameter_count != static_cast<int>(signature()->parameter_count())) {
885  int delta =
886  parameter_count - static_cast<int>(signature()->parameter_count());
887  int new_output_count = node->op()->ValueOutputCount() + delta;
888  NodeProperties::ChangeOp(node, common()->Start(new_output_count));
889  }
890  break;
891  }
892  case IrOpcode::kParameter: {
893  DCHECK_EQ(1, node->InputCount());
894  // Only exchange the node if the parameter count actually changed. We do
895  // not even have to do the default lowering because the the start node,
896  // the only input of a parameter node, only changes if the parameter count
897  // changes.
898  if (GetParameterCountAfterLowering() !=
899  static_cast<int>(signature()->parameter_count())) {
900  int old_index = ParameterIndexOf(node->op());
901  int new_index =
902  GetParameterIndexAfterLoweringSimd128(signature(), old_index);
903  if (old_index == new_index) {
904  NodeProperties::ChangeOp(node, common()->Parameter(new_index));
905 
906  Node* new_node[kNumLanes32];
907  for (int i = 0; i < kNumLanes32; ++i) {
908  new_node[i] = nullptr;
909  }
910  new_node[0] = node;
911  if (signature()->GetParam(old_index) ==
912  MachineRepresentation::kSimd128) {
913  for (int i = 1; i < kNumLanes32; ++i) {
914  new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
915  graph()->start());
916  }
917  }
918  ReplaceNode(node, new_node, kNumLanes32);
919  }
920  }
921  break;
922  }
923  case IrOpcode::kLoad:
924  case IrOpcode::kUnalignedLoad:
925  case IrOpcode::kProtectedLoad: {
926  LowerLoadOp(node, rep_type);
927  break;
928  }
929  case IrOpcode::kStore:
930  case IrOpcode::kUnalignedStore:
931  case IrOpcode::kProtectedStore: {
932  LowerStoreOp(node);
933  break;
934  }
935  case IrOpcode::kReturn: {
936  DefaultLowering(node);
937  int new_return_count = GetReturnCountAfterLoweringSimd128(signature());
938  if (static_cast<int>(signature()->return_count()) != new_return_count) {
939  NodeProperties::ChangeOp(node, common()->Return(new_return_count));
940  }
941  break;
942  }
943  case IrOpcode::kCall: {
944  // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor.
945  auto call_descriptor =
946  const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
947  if (DefaultLowering(node) ||
948  (call_descriptor->ReturnCount() == 1 &&
949  call_descriptor->GetReturnType(0) == MachineType::Simd128())) {
950  // We have to adjust the call descriptor.
951  const Operator* op = common()->Call(
952  GetI32WasmCallDescriptorForSimd(zone(), call_descriptor));
953  NodeProperties::ChangeOp(node, op);
954  }
955  if (call_descriptor->ReturnCount() == 1 &&
956  call_descriptor->GetReturnType(0) == MachineType::Simd128()) {
957  // We access the additional return values through projections.
958  Node* rep_node[kNumLanes32];
959  for (int i = 0; i < kNumLanes32; ++i) {
960  rep_node[i] =
961  graph()->NewNode(common()->Projection(i), node, graph()->start());
962  }
963  ReplaceNode(node, rep_node, kNumLanes32);
964  }
965  break;
966  }
967  case IrOpcode::kPhi: {
968  MachineRepresentation rep = PhiRepresentationOf(node->op());
969  if (rep == MachineRepresentation::kSimd128) {
970  // The replacement nodes have already been created, we only have to
971  // replace placeholder nodes.
972  Node** rep_node = GetReplacements(node);
973  for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
974  Node** rep_input =
975  GetReplacementsWithType(node->InputAt(i), rep_type);
976  for (int j = 0; j < num_lanes; j++) {
977  rep_node[j]->ReplaceInput(i, rep_input[j]);
978  }
979  }
980  } else {
981  DefaultLowering(node);
982  }
983  break;
984  }
985 #define I32X4_BINOP_CASE(opcode, instruction) \
986  case IrOpcode::opcode: { \
987  LowerBinaryOp(node, rep_type, machine()->instruction()); \
988  break; \
989  }
990  I32X4_BINOP_CASE(kI32x4Add, Int32Add)
991  I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
992  I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
993  I32X4_BINOP_CASE(kS128And, Word32And)
994  I32X4_BINOP_CASE(kS128Or, Word32Or)
995  I32X4_BINOP_CASE(kS128Xor, Word32Xor)
996 #undef I32X4_BINOP_CASE
997  case IrOpcode::kI32x4AddHoriz: {
998  LowerBinaryOp(node, rep_type, machine()->Int32Add(), false);
999  break;
1000  }
1001  case IrOpcode::kI16x8AddHoriz: {
1002  LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add(), false);
1003  break;
1004  }
1005  case IrOpcode::kI16x8Add:
1006  case IrOpcode::kI8x16Add: {
1007  LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add());
1008  break;
1009  }
1010  case IrOpcode::kI16x8Sub:
1011  case IrOpcode::kI8x16Sub: {
1012  LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub());
1013  break;
1014  }
1015  case IrOpcode::kI16x8Mul:
1016  case IrOpcode::kI8x16Mul: {
1017  LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul());
1018  break;
1019  }
1020  case IrOpcode::kI16x8AddSaturateS:
1021  case IrOpcode::kI8x16AddSaturateS: {
1022  LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true);
1023  break;
1024  }
1025  case IrOpcode::kI16x8SubSaturateS:
1026  case IrOpcode::kI8x16SubSaturateS: {
1027  LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true);
1028  break;
1029  }
1030  case IrOpcode::kI16x8AddSaturateU:
1031  case IrOpcode::kI8x16AddSaturateU: {
1032  LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false);
1033  break;
1034  }
1035  case IrOpcode::kI16x8SubSaturateU:
1036  case IrOpcode::kI8x16SubSaturateU: {
1037  LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false);
1038  break;
1039  }
1040  case IrOpcode::kI32x4MaxS:
1041  case IrOpcode::kI16x8MaxS:
1042  case IrOpcode::kI8x16MaxS: {
1043  LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type);
1044  break;
1045  }
1046  case IrOpcode::kI32x4MinS:
1047  case IrOpcode::kI16x8MinS:
1048  case IrOpcode::kI8x16MinS: {
1049  LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type);
1050  break;
1051  }
1052  case IrOpcode::kI32x4MaxU:
1053  case IrOpcode::kI16x8MaxU:
1054  case IrOpcode::kI8x16MaxU: {
1055  LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type);
1056  break;
1057  }
1058  case IrOpcode::kI32x4MinU:
1059  case IrOpcode::kI16x8MinU:
1060  case IrOpcode::kI8x16MinU: {
1061  LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type);
1062  break;
1063  }
1064  case IrOpcode::kI32x4Neg:
1065  case IrOpcode::kI16x8Neg:
1066  case IrOpcode::kI8x16Neg: {
1067  DCHECK_EQ(1, node->InputCount());
1068  Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1069  int num_lanes = NumLanes(rep_type);
1070  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1071  Node* zero = graph()->NewNode(common()->Int32Constant(0));
1072  for (int i = 0; i < num_lanes; ++i) {
1073  rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
1074  if (node->opcode() == IrOpcode::kI16x8Neg) {
1075  rep_node[i] = FixUpperBits(rep_node[i], kShift16);
1076  } else if (node->opcode() == IrOpcode::kI8x16Neg) {
1077  rep_node[i] = FixUpperBits(rep_node[i], kShift8);
1078  }
1079  }
1080  ReplaceNode(node, rep_node, num_lanes);
1081  break;
1082  }
1083  case IrOpcode::kS128Not: {
1084  DCHECK_EQ(1, node->InputCount());
1085  Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1086  Node* rep_node[kNumLanes32];
1087  Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF));
1088  for (int i = 0; i < kNumLanes32; ++i) {
1089  rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
1090  }
1091  ReplaceNode(node, rep_node, kNumLanes32);
1092  break;
1093  }
1094  case IrOpcode::kI32x4SConvertF32x4: {
1095  LowerConvertFromFloat(node, true);
1096  break;
1097  }
1098  case IrOpcode::kI32x4UConvertF32x4: {
1099  LowerConvertFromFloat(node, false);
1100  break;
1101  }
1102  case IrOpcode::kI32x4SConvertI16x8Low: {
1103  LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1104  0);
1105  break;
1106  }
1107  case IrOpcode::kI32x4SConvertI16x8High: {
1108  LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1109  4);
1110  break;
1111  }
1112  case IrOpcode::kI32x4UConvertI16x8Low: {
1113  LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1114  0);
1115  break;
1116  }
1117  case IrOpcode::kI32x4UConvertI16x8High: {
1118  LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1119  4);
1120  break;
1121  }
1122  case IrOpcode::kI16x8SConvertI8x16Low: {
1123  LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1124  0);
1125  break;
1126  }
1127  case IrOpcode::kI16x8SConvertI8x16High: {
1128  LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1129  8);
1130  break;
1131  }
1132  case IrOpcode::kI16x8UConvertI8x16Low: {
1133  LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1134  0);
1135  break;
1136  }
1137  case IrOpcode::kI16x8UConvertI8x16High: {
1138  LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1139  8);
1140  break;
1141  }
1142  case IrOpcode::kI16x8SConvertI32x4: {
1143  LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, true);
1144  break;
1145  }
1146  case IrOpcode::kI16x8UConvertI32x4: {
1147  LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, false);
1148  break;
1149  }
1150  case IrOpcode::kI8x16SConvertI16x8: {
1151  LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, true);
1152  break;
1153  }
1154  case IrOpcode::kI8x16UConvertI16x8: {
1155  LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, false);
1156  break;
1157  }
1158  case IrOpcode::kI32x4Shl:
1159  case IrOpcode::kI16x8Shl:
1160  case IrOpcode::kI8x16Shl:
1161  case IrOpcode::kI32x4ShrS:
1162  case IrOpcode::kI16x8ShrS:
1163  case IrOpcode::kI8x16ShrS:
1164  case IrOpcode::kI32x4ShrU:
1165  case IrOpcode::kI16x8ShrU:
1166  case IrOpcode::kI8x16ShrU: {
1167  LowerShiftOp(node, rep_type);
1168  break;
1169  }
1170  case IrOpcode::kF32x4AddHoriz: {
1171  LowerBinaryOp(node, rep_type, machine()->Float32Add(), false);
1172  break;
1173  }
1174 #define F32X4_BINOP_CASE(name) \
1175  case IrOpcode::kF32x4##name: { \
1176  LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
1177  break; \
1178  }
1179  F32X4_BINOP_CASE(Add)
1180  F32X4_BINOP_CASE(Sub)
1181  F32X4_BINOP_CASE(Mul)
1182  F32X4_BINOP_CASE(Min)
1183  F32X4_BINOP_CASE(Max)
1184 #undef F32X4_BINOP_CASE
1185 #define F32X4_UNOP_CASE(name) \
1186  case IrOpcode::kF32x4##name: { \
1187  LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
1188  break; \
1189  }
1190  F32X4_UNOP_CASE(Abs)
1191  F32X4_UNOP_CASE(Neg)
1192 #undef F32x4_UNOP_CASE
1193  case IrOpcode::kF32x4RecipApprox:
1194  case IrOpcode::kF32x4RecipSqrtApprox: {
1195  DCHECK_EQ(1, node->InputCount());
1196  Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1197  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1198  Node* float_one = graph()->NewNode(common()->Float32Constant(1.0));
1199  for (int i = 0; i < num_lanes; ++i) {
1200  Node* tmp = rep[i];
1201  if (node->opcode() == IrOpcode::kF32x4RecipSqrtApprox) {
1202  tmp = graph()->NewNode(machine()->Float32Sqrt(), rep[i]);
1203  }
1204  rep_node[i] = graph()->NewNode(machine()->Float32Div(), float_one, tmp);
1205  }
1206  ReplaceNode(node, rep_node, num_lanes);
1207  break;
1208  }
1209  case IrOpcode::kF32x4SConvertI32x4: {
1210  LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32());
1211  break;
1212  }
1213  case IrOpcode::kF32x4UConvertI32x4: {
1214  LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32());
1215  break;
1216  }
1217  case IrOpcode::kI32x4Splat:
1218  case IrOpcode::kF32x4Splat:
1219  case IrOpcode::kI16x8Splat:
1220  case IrOpcode::kI8x16Splat: {
1221  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1222  for (int i = 0; i < num_lanes; ++i) {
1223  if (HasReplacement(0, node->InputAt(0))) {
1224  rep_node[i] = GetReplacements(node->InputAt(0))[0];
1225  } else {
1226  rep_node[i] = node->InputAt(0);
1227  }
1228  }
1229  ReplaceNode(node, rep_node, num_lanes);
1230  break;
1231  }
1232  case IrOpcode::kI32x4ExtractLane:
1233  case IrOpcode::kF32x4ExtractLane:
1234  case IrOpcode::kI16x8ExtractLane:
1235  case IrOpcode::kI8x16ExtractLane: {
1236  int32_t lane = OpParameter<int32_t>(node->op());
1237  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1238  rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane];
1239  for (int i = 1; i < num_lanes; ++i) {
1240  rep_node[i] = nullptr;
1241  }
1242  ReplaceNode(node, rep_node, num_lanes);
1243  break;
1244  }
1245  case IrOpcode::kI32x4ReplaceLane:
1246  case IrOpcode::kF32x4ReplaceLane:
1247  case IrOpcode::kI16x8ReplaceLane:
1248  case IrOpcode::kI8x16ReplaceLane: {
1249  DCHECK_EQ(2, node->InputCount());
1250  Node* repNode = node->InputAt(1);
1251  int32_t lane = OpParameter<int32_t>(node->op());
1252  Node** old_rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
1253  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1254  for (int i = 0; i < num_lanes; ++i) {
1255  rep_node[i] = old_rep_node[i];
1256  }
1257  if (HasReplacement(0, repNode)) {
1258  rep_node[lane] = GetReplacements(repNode)[0];
1259  } else {
1260  rep_node[lane] = repNode;
1261  }
1262  ReplaceNode(node, rep_node, num_lanes);
1263  break;
1264  }
1265 #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \
1266  case IrOpcode::simd_op: { \
1267  LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
1268  break; \
1269  }
1270  COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false)
1271  COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false)
1272  COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false)
1273  COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true)
1274  COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true)
1275  COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false)
1276  COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false)
1277  COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false)
1278  COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true)
1279  COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true)
1280  COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false)
1281  COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false)
1282  COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true)
1283  COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true)
1284  COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false)
1285  COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false)
1286  COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false)
1287  COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true)
1288  COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true)
1289  COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false)
1290  COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false)
1291  COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true)
1292  COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true)
1293  COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false)
1294  COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false)
1295  COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false)
1296  COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true)
1297  COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true)
1298  COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false)
1299  COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false)
1300  COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true)
1301  COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true)
1302 #undef COMPARISON_CASE
1303  case IrOpcode::kF32x4Ne: {
1304  LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal());
1305  break;
1306  }
1307  case IrOpcode::kI32x4Ne: {
1308  LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal());
1309  break;
1310  }
1311  case IrOpcode::kI16x8Ne: {
1312  LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal());
1313  break;
1314  }
1315  case IrOpcode::kI8x16Ne: {
1316  LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal());
1317  break;
1318  }
1319  case IrOpcode::kS128Select: {
1320  DCHECK_EQ(3, node->InputCount());
1321  DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 ||
1322  ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 ||
1323  ReplacementType(node->InputAt(0)) == SimdType::kInt8x16);
1324  Node** boolean_input = GetReplacements(node->InputAt(0));
1325  Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
1326  Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
1327  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1328  for (int i = 0; i < num_lanes; ++i) {
1329  Node* tmp1 =
1330  graph()->NewNode(machine()->Word32Xor(), rep_left[i], rep_right[i]);
1331  Node* tmp2 =
1332  graph()->NewNode(machine()->Word32And(), boolean_input[i], tmp1);
1333  rep_node[i] =
1334  graph()->NewNode(machine()->Word32Xor(), rep_right[i], tmp2);
1335  }
1336  ReplaceNode(node, rep_node, num_lanes);
1337  break;
1338  }
1339  case IrOpcode::kS8x16Shuffle: {
1340  DCHECK_EQ(2, node->InputCount());
1341  const uint8_t* shuffle = OpParameter<uint8_t*>(node->op());
1342  Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
1343  Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
1344  Node** rep_node = zone()->NewArray<Node*>(16);
1345  for (int i = 0; i < 16; i++) {
1346  int lane = shuffle[i];
1347  rep_node[i] = lane < 16 ? rep_left[lane] : rep_right[lane - 16];
1348  }
1349  ReplaceNode(node, rep_node, 16);
1350  break;
1351  }
1352  case IrOpcode::kS1x4AnyTrue:
1353  case IrOpcode::kS1x4AllTrue:
1354  case IrOpcode::kS1x8AnyTrue:
1355  case IrOpcode::kS1x8AllTrue:
1356  case IrOpcode::kS1x16AnyTrue:
1357  case IrOpcode::kS1x16AllTrue: {
1358  DCHECK_EQ(1, node->InputCount());
1359  SimdType input_rep_type = ReplacementType(node->InputAt(0));
1360  int input_num_lanes = NumLanes(input_rep_type);
1361  Node** rep = GetReplacements(node->InputAt(0));
1362  Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1363  Node* true_node = mcgraph_->Int32Constant(-1);
1364  Node* false_node = mcgraph_->Int32Constant(0);
1365  Node* tmp_result = false_node;
1366  if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1367  node->opcode() == IrOpcode::kS1x8AllTrue ||
1368  node->opcode() == IrOpcode::kS1x16AllTrue) {
1369  tmp_result = true_node;
1370  }
1371  for (int i = 0; i < input_num_lanes; ++i) {
1372  Diamond is_false(
1373  graph(), common(),
1374  graph()->NewNode(machine()->Word32Equal(), rep[i], false_node));
1375  if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1376  node->opcode() == IrOpcode::kS1x8AllTrue ||
1377  node->opcode() == IrOpcode::kS1x16AllTrue) {
1378  tmp_result = is_false.Phi(MachineRepresentation::kWord32, false_node,
1379  tmp_result);
1380  } else {
1381  tmp_result = is_false.Phi(MachineRepresentation::kWord32, tmp_result,
1382  true_node);
1383  }
1384  }
1385  rep_node[0] = tmp_result;
1386  for (int i = 1; i < num_lanes; ++i) {
1387  rep_node[i] = nullptr;
1388  }
1389  ReplaceNode(node, rep_node, num_lanes);
1390  break;
1391  }
1392  default: { DefaultLowering(node); }
1393  }
1394 }
1395 
1396 bool SimdScalarLowering::DefaultLowering(Node* node) {
1397  bool something_changed = false;
1398  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1399  Node* input = node->InputAt(i);
1400  if (HasReplacement(0, input)) {
1401  something_changed = true;
1402  node->ReplaceInput(i, GetReplacements(input)[0]);
1403  }
1404  if (HasReplacement(1, input)) {
1405  something_changed = true;
1406  for (int j = 1; j < ReplacementCount(input); ++j) {
1407  node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
1408  }
1409  }
1410  }
1411  return something_changed;
1412 }
1413 
1414 void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) {
1415  replacements_[old->id()].node = zone()->NewArray<Node*>(count);
1416  for (int i = 0; i < count; ++i) {
1417  replacements_[old->id()].node[i] = new_nodes[i];
1418  }
1419  replacements_[old->id()].num_replacements = count;
1420 }
1421 
1422 bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
1423  return replacements_[node->id()].node != nullptr &&
1424  replacements_[node->id()].node[index] != nullptr;
1425 }
1426 
1427 SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
1428  return replacements_[node->id()].type;
1429 }
1430 
1431 Node** SimdScalarLowering::GetReplacements(Node* node) {
1432  Node** result = replacements_[node->id()].node;
1433  DCHECK(result);
1434  return result;
1435 }
1436 
1437 int SimdScalarLowering::ReplacementCount(Node* node) {
1438  return replacements_[node->id()].num_replacements;
1439 }
1440 
1441 void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) {
1442  for (int i = 0; i < kNumLanes32; ++i) {
1443  if (replacements[i] != nullptr) {
1444  result[i] =
1445  graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]);
1446  } else {
1447  result[i] = nullptr;
1448  }
1449  }
1450 }
1451 
1452 void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) {
1453  for (int i = 0; i < kNumLanes32; ++i) {
1454  if (replacements[i] != nullptr) {
1455  result[i] =
1456  graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]);
1457  } else {
1458  result[i] = nullptr;
1459  }
1460  }
1461 }
1462 
1463 template <typename T>
1464 void SimdScalarLowering::Int32ToSmallerInt(Node** replacements, Node** result) {
1465  const int num_ints = sizeof(int32_t) / sizeof(T);
1466  const int bit_size = sizeof(T) * 8;
1467  const Operator* sign_extend;
1468  switch (sizeof(T)) {
1469  case 1:
1470  sign_extend = machine()->SignExtendWord8ToInt32();
1471  break;
1472  case 2:
1473  sign_extend = machine()->SignExtendWord16ToInt32();
1474  break;
1475  default:
1476  UNREACHABLE();
1477  }
1478 
1479  for (int i = 0; i < kNumLanes32; i++) {
1480  if (replacements[i] != nullptr) {
1481  for (int j = 0; j < num_ints; j++) {
1482  result[num_ints * i + j] = graph()->NewNode(
1483  sign_extend,
1484  graph()->NewNode(machine()->Word32Sar(), replacements[i],
1485  mcgraph_->Int32Constant(j * bit_size)));
1486  }
1487  } else {
1488  for (int j = 0; j < num_ints; j++) {
1489  result[num_ints * i + j] = nullptr;
1490  }
1491  }
1492  }
1493 }
1494 
1495 template <typename T>
1496 void SimdScalarLowering::SmallerIntToInt32(Node** replacements, Node** result) {
1497  const int num_ints = sizeof(int32_t) / sizeof(T);
1498  const int bit_size = sizeof(T) * 8;
1499  const int bit_mask = (1 << bit_size) - 1;
1500 
1501  for (int i = 0; i < kNumLanes32; ++i) {
1502  result[i] = mcgraph_->Int32Constant(0);
1503  for (int j = 0; j < num_ints; j++) {
1504  if (replacements[num_ints * i + j] != nullptr) {
1505  Node* clean_bits = graph()->NewNode(machine()->Word32And(),
1506  replacements[num_ints * i + j],
1507  mcgraph_->Int32Constant(bit_mask));
1508  Node* shift = graph()->NewNode(machine()->Word32Shl(), clean_bits,
1509  mcgraph_->Int32Constant(j * bit_size));
1510  result[i] = graph()->NewNode(machine()->Word32Or(), result[i], shift);
1511  }
1512  }
1513  }
1514 }
1515 
1516 Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
1517  Node** replacements = GetReplacements(node);
1518  if (ReplacementType(node) == type) {
1519  return GetReplacements(node);
1520  }
1521  int num_lanes = NumLanes(type);
1522  Node** result = zone()->NewArray<Node*>(num_lanes);
1523  if (type == SimdType::kInt32x4) {
1524  if (ReplacementType(node) == SimdType::kFloat32x4) {
1525  Float32ToInt32(replacements, result);
1526  } else if (ReplacementType(node) == SimdType::kInt16x8) {
1527  SmallerIntToInt32<int16_t>(replacements, result);
1528  } else if (ReplacementType(node) == SimdType::kInt8x16) {
1529  SmallerIntToInt32<int8_t>(replacements, result);
1530  } else {
1531  UNREACHABLE();
1532  }
1533  } else if (type == SimdType::kFloat32x4) {
1534  if (ReplacementType(node) == SimdType::kInt32x4) {
1535  Int32ToFloat32(replacements, result);
1536  } else if (ReplacementType(node) == SimdType::kInt16x8) {
1537  UNIMPLEMENTED();
1538  } else {
1539  UNREACHABLE();
1540  }
1541  } else if (type == SimdType::kInt16x8) {
1542  if (ReplacementType(node) == SimdType::kInt32x4) {
1543  Int32ToSmallerInt<int16_t>(replacements, result);
1544  } else if (ReplacementType(node) == SimdType::kFloat32x4) {
1545  UNIMPLEMENTED();
1546  } else {
1547  UNREACHABLE();
1548  }
1549  } else if (type == SimdType::kInt8x16) {
1550  if (ReplacementType(node) == SimdType::kInt32x4) {
1551  Int32ToSmallerInt<int8_t>(replacements, result);
1552  } else {
1553  UNIMPLEMENTED();
1554  }
1555  } else {
1556  UNREACHABLE();
1557  }
1558  return result;
1559 }
1560 
1561 void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
1562  MachineRepresentation rep = PhiRepresentationOf(phi->op());
1563  if (rep == MachineRepresentation::kSimd128) {
1564  // We have to create the replacements for a phi node before we actually
1565  // lower the phi to break potential cycles in the graph. The replacements of
1566  // input nodes do not exist yet, so we use a placeholder node to pass the
1567  // graph verifier.
1568  int value_count = phi->op()->ValueInputCount();
1569  SimdType type = ReplacementType(phi);
1570  int num_lanes = NumLanes(type);
1571  Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes);
1572  for (int i = 0; i < num_lanes; ++i) {
1573  inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
1574  inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
1575  }
1576  for (int i = 0; i < value_count; ++i) {
1577  for (int j = 0; j < num_lanes; ++j) {
1578  inputs_rep[j][i] = placeholder_;
1579  }
1580  }
1581  Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
1582  for (int i = 0; i < num_lanes; ++i) {
1583  rep_nodes[i] = graph()->NewNode(
1584  common()->Phi(MachineTypeFrom(type).representation(), value_count),
1585  value_count + 1, inputs_rep[i], false);
1586  }
1587  ReplaceNode(phi, rep_nodes, num_lanes);
1588  }
1589 }
1590 } // namespace compiler
1591 } // namespace internal
1592 } // namespace v8
v8
Definition: libplatform.h:13