V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
int64-lowering.cc
1 // Copyright 2014 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/int64-lowering.h"
6 
7 #include "src/compiler/common-operator.h"
8 #include "src/compiler/diamond.h"
9 #include "src/compiler/graph.h"
10 #include "src/compiler/linkage.h"
11 #include "src/compiler/machine-operator.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/node-properties.h"
14 #include "src/compiler/node.h"
15 #include "src/compiler/wasm-compiler.h"
16 // TODO(wasm): Remove this include.
17 #include "src/wasm/wasm-linkage.h"
18 #include "src/zone/zone.h"
19 
20 namespace v8 {
21 namespace internal {
22 namespace compiler {
23 
24 Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
25  CommonOperatorBuilder* common, Zone* zone,
26  Signature<MachineRepresentation>* signature)
27  : zone_(zone),
28  graph_(graph),
29  machine_(machine),
30  common_(common),
31  state_(graph, 3),
32  stack_(zone),
33  replacements_(nullptr),
34  signature_(signature),
35  placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
36  graph->start())) {
37  DCHECK_NOT_NULL(graph);
38  DCHECK_NOT_NULL(graph->end());
39  replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
40  memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
41 }
42 
43 void Int64Lowering::LowerGraph() {
44  if (!machine()->Is32()) {
45  return;
46  }
47  stack_.push_back({graph()->end(), 0});
48  state_.Set(graph()->end(), State::kOnStack);
49 
50  while (!stack_.empty()) {
51  NodeState& top = stack_.back();
52  if (top.input_index == top.node->InputCount()) {
53  // All inputs of top have already been lowered, now lower top.
54  stack_.pop_back();
55  state_.Set(top.node, State::kVisited);
56  LowerNode(top.node);
57  } else {
58  // Push the next input onto the stack.
59  Node* input = top.node->InputAt(top.input_index++);
60  if (state_.Get(input) == State::kUnvisited) {
61  if (input->opcode() == IrOpcode::kPhi) {
62  // To break cycles with phi nodes we push phis on a separate stack so
63  // that they are processed after all other nodes.
64  PreparePhiReplacement(input);
65  stack_.push_front({input, 0});
66  } else if (input->opcode() == IrOpcode::kEffectPhi ||
67  input->opcode() == IrOpcode::kLoop) {
68  stack_.push_front({input, 0});
69  } else {
70  stack_.push_back({input, 0});
71  }
72  state_.Set(input, State::kOnStack);
73  }
74  }
75  }
76 }
77 
78 namespace {
79 
80 int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor,
81  int old_index) {
82  int result = old_index;
83  for (int i = 0; i < old_index; i++) {
84  if (call_descriptor->GetReturnType(i).representation() ==
85  MachineRepresentation::kWord64) {
86  result++;
87  }
88  }
89  return result;
90 }
91 
92 int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) {
93  return GetReturnIndexAfterLowering(
94  call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
95 }
96 
97 int GetParameterIndexAfterLowering(
98  Signature<MachineRepresentation>* signature, int old_index) {
99  int result = old_index;
100  for (int i = 0; i < old_index; i++) {
101  if (signature->GetParam(i) == MachineRepresentation::kWord64) {
102  result++;
103  }
104  }
105  return result;
106 }
107 
108 int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
109  int result = static_cast<int>(signature->return_count());
110  for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
111  if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
112  result++;
113  }
114  }
115  return result;
116 }
117 
118 } // namespace
119 
120 void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
121  DCHECK_EQ(5, node->InputCount());
122  Node* value = node->InputAt(2);
123  node->ReplaceInput(2, GetReplacementLow(value));
124  node->InsertInput(zone(), 3, GetReplacementHigh(value));
125  NodeProperties::ChangeOp(node, op);
126  ReplaceNodeWithProjections(node);
127 }
128 
129 void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
130  Node* value = node->InputAt(2);
131  node->ReplaceInput(2, GetReplacementLow(value));
132  NodeProperties::ChangeOp(node, op);
133  ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
134 }
135 
136 // static
137 int Int64Lowering::GetParameterCountAfterLowering(
138  Signature<MachineRepresentation>* signature) {
139  // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
140  // after lowering.
141  return GetParameterIndexAfterLowering(
142  signature, static_cast<int>(signature->parameter_count()));
143 }
144 
145 void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
146  Node*& index_high) {
147  if (HasReplacementLow(index)) {
148  index = GetReplacementLow(index);
149  }
150 #if defined(V8_TARGET_LITTLE_ENDIAN)
151  index_low = index;
152  index_high = graph()->NewNode(machine()->Int32Add(), index,
153  graph()->NewNode(common()->Int32Constant(4)));
154 #elif defined(V8_TARGET_BIG_ENDIAN)
155  index_low = graph()->NewNode(machine()->Int32Add(), index,
156  graph()->NewNode(common()->Int32Constant(4)));
157  index_high = index;
158 #endif
159 }
160 
161 void Int64Lowering::LowerNode(Node* node) {
162  switch (node->opcode()) {
163  case IrOpcode::kInt64Constant: {
164  int64_t value = OpParameter<int64_t>(node->op());
165  Node* low_node = graph()->NewNode(
166  common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
167  Node* high_node = graph()->NewNode(
168  common()->Int32Constant(static_cast<int32_t>(value >> 32)));
169  ReplaceNode(node, low_node, high_node);
170  break;
171  }
172  case IrOpcode::kLoad:
173  case IrOpcode::kUnalignedLoad: {
174  MachineRepresentation rep;
175  if (node->opcode() == IrOpcode::kLoad) {
176  rep = LoadRepresentationOf(node->op()).representation();
177  } else {
178  DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
179  rep = LoadRepresentationOf(node->op()).representation();
180  }
181 
182  if (rep == MachineRepresentation::kWord64) {
183  Node* base = node->InputAt(0);
184  Node* index = node->InputAt(1);
185  Node* index_low;
186  Node* index_high;
187  GetIndexNodes(index, index_low, index_high);
188  const Operator* load_op;
189 
190  if (node->opcode() == IrOpcode::kLoad) {
191  load_op = machine()->Load(MachineType::Int32());
192  } else {
193  DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
194  load_op = machine()->UnalignedLoad(MachineType::Int32());
195  }
196 
197  Node* high_node;
198  if (node->InputCount() > 2) {
199  Node* effect_high = node->InputAt(2);
200  Node* control_high = node->InputAt(3);
201  high_node = graph()->NewNode(load_op, base, index_high, effect_high,
202  control_high);
203  // change the effect change from old_node --> old_effect to
204  // old_node --> high_node --> old_effect.
205  node->ReplaceInput(2, high_node);
206  } else {
207  high_node = graph()->NewNode(load_op, base, index_high);
208  }
209  node->ReplaceInput(1, index_low);
210  NodeProperties::ChangeOp(node, load_op);
211  ReplaceNode(node, node, high_node);
212  } else {
213  DefaultLowering(node);
214  }
215  break;
216  }
217  case IrOpcode::kStore:
218  case IrOpcode::kUnalignedStore: {
219  MachineRepresentation rep;
220  if (node->opcode() == IrOpcode::kStore) {
221  rep = StoreRepresentationOf(node->op()).representation();
222  } else {
223  DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
224  rep = UnalignedStoreRepresentationOf(node->op());
225  }
226 
227  if (rep == MachineRepresentation::kWord64) {
228  // We change the original store node to store the low word, and create
229  // a new store node to store the high word. The effect and control edges
230  // are copied from the original store to the new store node, the effect
231  // edge of the original store is redirected to the new store.
232  Node* base = node->InputAt(0);
233  Node* index = node->InputAt(1);
234  Node* index_low;
235  Node* index_high;
236  GetIndexNodes(index, index_low, index_high);
237  Node* value = node->InputAt(2);
238  DCHECK(HasReplacementLow(value));
239  DCHECK(HasReplacementHigh(value));
240 
241  const Operator* store_op;
242  if (node->opcode() == IrOpcode::kStore) {
243  WriteBarrierKind write_barrier_kind =
244  StoreRepresentationOf(node->op()).write_barrier_kind();
245  store_op = machine()->Store(StoreRepresentation(
246  MachineRepresentation::kWord32, write_barrier_kind));
247  } else {
248  DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
249  store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
250  }
251 
252  Node* high_node;
253  if (node->InputCount() > 3) {
254  Node* effect_high = node->InputAt(3);
255  Node* control_high = node->InputAt(4);
256  high_node = graph()->NewNode(store_op, base, index_high,
257  GetReplacementHigh(value), effect_high,
258  control_high);
259  node->ReplaceInput(3, high_node);
260 
261  } else {
262  high_node = graph()->NewNode(store_op, base, index_high,
263  GetReplacementHigh(value));
264  }
265 
266  node->ReplaceInput(1, index_low);
267  node->ReplaceInput(2, GetReplacementLow(value));
268  NodeProperties::ChangeOp(node, store_op);
269  ReplaceNode(node, node, high_node);
270  } else {
271  DefaultLowering(node, true);
272  }
273  break;
274  }
275  case IrOpcode::kStart: {
276  int parameter_count = GetParameterCountAfterLowering(signature());
277  // Only exchange the node if the parameter count actually changed.
278  if (parameter_count != static_cast<int>(signature()->parameter_count())) {
279  int delta =
280  parameter_count - static_cast<int>(signature()->parameter_count());
281  int new_output_count = node->op()->ValueOutputCount() + delta;
282  NodeProperties::ChangeOp(node, common()->Start(new_output_count));
283  }
284  break;
285  }
286  case IrOpcode::kParameter: {
287  DCHECK_EQ(1, node->InputCount());
288  // Only exchange the node if the parameter count actually changed. We do
289  // not even have to do the default lowering because the the start node,
290  // the only input of a parameter node, only changes if the parameter count
291  // changes.
292  if (GetParameterCountAfterLowering(signature()) !=
293  static_cast<int>(signature()->parameter_count())) {
294  int old_index = ParameterIndexOf(node->op());
295  // TODO(wasm): Make this part not wasm specific.
296  // Prevent special lowering of the instance parameter.
297  if (old_index == wasm::kWasmInstanceParameterIndex) {
298  DefaultLowering(node);
299  break;
300  }
301  // Adjust old_index to be compliant with the signature.
302  --old_index;
303  int new_index = GetParameterIndexAfterLowering(signature(), old_index);
304  // Adjust new_index to consider the instance parameter.
305  ++new_index;
306  NodeProperties::ChangeOp(node, common()->Parameter(new_index));
307 
308  if (signature()->GetParam(old_index) ==
309  MachineRepresentation::kWord64) {
310  Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
311  graph()->start());
312  ReplaceNode(node, node, high_node);
313  }
314  }
315  break;
316  }
317  case IrOpcode::kReturn: {
318  int input_count = node->InputCount();
319  DefaultLowering(node);
320  if (input_count != node->InputCount()) {
321  int new_return_count = GetReturnCountAfterLowering(signature());
322  if (static_cast<int>(signature()->return_count()) != new_return_count) {
323  NodeProperties::ChangeOp(node, common()->Return(new_return_count));
324  }
325  }
326  break;
327  }
328  case IrOpcode::kTailCall: {
329  auto call_descriptor =
330  const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
331  bool returns_require_lowering =
332  GetReturnCountAfterLowering(call_descriptor) !=
333  static_cast<int>(call_descriptor->ReturnCount());
334  if (DefaultLowering(node) || returns_require_lowering) {
335  // Tail calls do not have return values, so adjusting the call
336  // descriptor is enough.
337  auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor);
338  NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
339  }
340  break;
341  }
342  case IrOpcode::kCall: {
343  auto call_descriptor =
344  const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
345  bool returns_require_lowering =
346  GetReturnCountAfterLowering(call_descriptor) !=
347  static_cast<int>(call_descriptor->ReturnCount());
348  if (DefaultLowering(node) || returns_require_lowering) {
349  // We have to adjust the call descriptor.
350  NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor(
351  zone(), call_descriptor)));
352  }
353  if (returns_require_lowering) {
354  size_t return_arity = call_descriptor->ReturnCount();
355  if (return_arity == 1) {
356  // We access the additional return values through projections.
357  ReplaceNodeWithProjections(node);
358  } else {
359  ZoneVector<Node*> projections(return_arity, zone());
360  NodeProperties::CollectValueProjections(node, projections.data(),
361  return_arity);
362  for (size_t old_index = 0, new_index = 0; old_index < return_arity;
363  ++old_index, ++new_index) {
364  Node* use_node = projections[old_index];
365  DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
366  DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
367  static_cast<int>(old_index)),
368  static_cast<int>(new_index));
369  if (new_index != old_index) {
370  NodeProperties::ChangeOp(
371  use_node, common()->Projection(new_index));
372  }
373  if (call_descriptor->GetReturnType(old_index).representation() ==
374  MachineRepresentation::kWord64) {
375  Node* high_node = graph()->NewNode(
376  common()->Projection(new_index + 1), node,
377  graph()->start());
378  ReplaceNode(use_node, use_node, high_node);
379  ++new_index;
380  }
381  }
382  }
383  }
384  break;
385  }
386  case IrOpcode::kWord64And: {
387  DCHECK_EQ(2, node->InputCount());
388  Node* left = node->InputAt(0);
389  Node* right = node->InputAt(1);
390 
391  Node* low_node =
392  graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
393  GetReplacementLow(right));
394  Node* high_node =
395  graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
396  GetReplacementHigh(right));
397  ReplaceNode(node, low_node, high_node);
398  break;
399  }
400  case IrOpcode::kTruncateInt64ToInt32: {
401  DCHECK_EQ(1, node->InputCount());
402  Node* input = node->InputAt(0);
403  ReplaceNode(node, GetReplacementLow(input), nullptr);
404  node->NullAllInputs();
405  break;
406  }
407  case IrOpcode::kInt64Add: {
408  DCHECK_EQ(2, node->InputCount());
409 
410  Node* right = node->InputAt(1);
411  node->ReplaceInput(1, GetReplacementLow(right));
412  node->AppendInput(zone(), GetReplacementHigh(right));
413 
414  Node* left = node->InputAt(0);
415  node->ReplaceInput(0, GetReplacementLow(left));
416  node->InsertInput(zone(), 1, GetReplacementHigh(left));
417 
418  NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
419  // We access the additional return values through projections.
420  ReplaceNodeWithProjections(node);
421  break;
422  }
423  case IrOpcode::kInt64Sub: {
424  DCHECK_EQ(2, node->InputCount());
425 
426  Node* right = node->InputAt(1);
427  node->ReplaceInput(1, GetReplacementLow(right));
428  node->AppendInput(zone(), GetReplacementHigh(right));
429 
430  Node* left = node->InputAt(0);
431  node->ReplaceInput(0, GetReplacementLow(left));
432  node->InsertInput(zone(), 1, GetReplacementHigh(left));
433 
434  NodeProperties::ChangeOp(node, machine()->Int32PairSub());
435  // We access the additional return values through projections.
436  ReplaceNodeWithProjections(node);
437  break;
438  }
439  case IrOpcode::kInt64Mul: {
440  DCHECK_EQ(2, node->InputCount());
441 
442  Node* right = node->InputAt(1);
443  node->ReplaceInput(1, GetReplacementLow(right));
444  node->AppendInput(zone(), GetReplacementHigh(right));
445 
446  Node* left = node->InputAt(0);
447  node->ReplaceInput(0, GetReplacementLow(left));
448  node->InsertInput(zone(), 1, GetReplacementHigh(left));
449 
450  NodeProperties::ChangeOp(node, machine()->Int32PairMul());
451  // We access the additional return values through projections.
452  ReplaceNodeWithProjections(node);
453  break;
454  }
455  case IrOpcode::kWord64Or: {
456  DCHECK_EQ(2, node->InputCount());
457  Node* left = node->InputAt(0);
458  Node* right = node->InputAt(1);
459 
460  Node* low_node =
461  graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
462  GetReplacementLow(right));
463  Node* high_node =
464  graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
465  GetReplacementHigh(right));
466  ReplaceNode(node, low_node, high_node);
467  break;
468  }
469  case IrOpcode::kWord64Xor: {
470  DCHECK_EQ(2, node->InputCount());
471  Node* left = node->InputAt(0);
472  Node* right = node->InputAt(1);
473 
474  Node* low_node =
475  graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
476  GetReplacementLow(right));
477  Node* high_node =
478  graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
479  GetReplacementHigh(right));
480  ReplaceNode(node, low_node, high_node);
481  break;
482  }
483  case IrOpcode::kWord64Shl: {
484  // TODO(turbofan): if the shift count >= 32, then we can set the low word
485  // of the output to 0 and just calculate the high word.
486  DCHECK_EQ(2, node->InputCount());
487  Node* shift = node->InputAt(1);
488  if (HasReplacementLow(shift)) {
489  // We do not have to care about the high word replacement, because
490  // the shift can only be between 0 and 63 anyways.
491  node->ReplaceInput(1, GetReplacementLow(shift));
492  }
493 
494  Node* value = node->InputAt(0);
495  node->ReplaceInput(0, GetReplacementLow(value));
496  node->InsertInput(zone(), 1, GetReplacementHigh(value));
497 
498  NodeProperties::ChangeOp(node, machine()->Word32PairShl());
499  // We access the additional return values through projections.
500  ReplaceNodeWithProjections(node);
501  break;
502  }
503  case IrOpcode::kWord64Shr: {
504  // TODO(turbofan): if the shift count >= 32, then we can set the low word
505  // of the output to 0 and just calculate the high word.
506  DCHECK_EQ(2, node->InputCount());
507  Node* shift = node->InputAt(1);
508  if (HasReplacementLow(shift)) {
509  // We do not have to care about the high word replacement, because
510  // the shift can only be between 0 and 63 anyways.
511  node->ReplaceInput(1, GetReplacementLow(shift));
512  }
513 
514  Node* value = node->InputAt(0);
515  node->ReplaceInput(0, GetReplacementLow(value));
516  node->InsertInput(zone(), 1, GetReplacementHigh(value));
517 
518  NodeProperties::ChangeOp(node, machine()->Word32PairShr());
519  // We access the additional return values through projections.
520  ReplaceNodeWithProjections(node);
521  break;
522  }
523  case IrOpcode::kWord64Sar: {
524  // TODO(turbofan): if the shift count >= 32, then we can set the low word
525  // of the output to 0 and just calculate the high word.
526  DCHECK_EQ(2, node->InputCount());
527  Node* shift = node->InputAt(1);
528  if (HasReplacementLow(shift)) {
529  // We do not have to care about the high word replacement, because
530  // the shift can only be between 0 and 63 anyways.
531  node->ReplaceInput(1, GetReplacementLow(shift));
532  }
533 
534  Node* value = node->InputAt(0);
535  node->ReplaceInput(0, GetReplacementLow(value));
536  node->InsertInput(zone(), 1, GetReplacementHigh(value));
537 
538  NodeProperties::ChangeOp(node, machine()->Word32PairSar());
539  // We access the additional return values through projections.
540  ReplaceNodeWithProjections(node);
541  break;
542  }
543  case IrOpcode::kWord64Equal: {
544  DCHECK_EQ(2, node->InputCount());
545  Node* left = node->InputAt(0);
546  Node* right = node->InputAt(1);
547 
548  // TODO(wasm): Use explicit comparisons and && here?
549  Node* replacement = graph()->NewNode(
550  machine()->Word32Equal(),
551  graph()->NewNode(
552  machine()->Word32Or(),
553  graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
554  GetReplacementLow(right)),
555  graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
556  GetReplacementHigh(right))),
557  graph()->NewNode(common()->Int32Constant(0)));
558 
559  ReplaceNode(node, replacement, nullptr);
560  break;
561  }
562  case IrOpcode::kInt64LessThan: {
563  LowerComparison(node, machine()->Int32LessThan(),
564  machine()->Uint32LessThan());
565  break;
566  }
567  case IrOpcode::kInt64LessThanOrEqual: {
568  LowerComparison(node, machine()->Int32LessThan(),
569  machine()->Uint32LessThanOrEqual());
570  break;
571  }
572  case IrOpcode::kUint64LessThan: {
573  LowerComparison(node, machine()->Uint32LessThan(),
574  machine()->Uint32LessThan());
575  break;
576  }
577  case IrOpcode::kUint64LessThanOrEqual: {
578  LowerComparison(node, machine()->Uint32LessThan(),
579  machine()->Uint32LessThanOrEqual());
580  break;
581  }
582  case IrOpcode::kSignExtendWord32ToInt64:
583  case IrOpcode::kChangeInt32ToInt64: {
584  DCHECK_EQ(1, node->InputCount());
585  Node* input = node->InputAt(0);
586  if (HasReplacementLow(input)) {
587  input = GetReplacementLow(input);
588  }
589  // We use SAR to preserve the sign in the high word.
590  ReplaceNode(
591  node, input,
592  graph()->NewNode(machine()->Word32Sar(), input,
593  graph()->NewNode(common()->Int32Constant(31))));
594  node->NullAllInputs();
595  break;
596  }
597  case IrOpcode::kChangeUint32ToUint64: {
598  DCHECK_EQ(1, node->InputCount());
599  Node* input = node->InputAt(0);
600  if (HasReplacementLow(input)) {
601  input = GetReplacementLow(input);
602  }
603  ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
604  node->NullAllInputs();
605  break;
606  }
607  case IrOpcode::kBitcastInt64ToFloat64: {
608  DCHECK_EQ(1, node->InputCount());
609  Node* input = node->InputAt(0);
610  Node* stack_slot = graph()->NewNode(
611  machine()->StackSlot(MachineRepresentation::kWord64));
612 
613  Node* store_high_word = graph()->NewNode(
614  machine()->Store(
615  StoreRepresentation(MachineRepresentation::kWord32,
616  WriteBarrierKind::kNoWriteBarrier)),
617  stack_slot,
618  graph()->NewNode(
619  common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
620  GetReplacementHigh(input), graph()->start(), graph()->start());
621 
622  Node* store_low_word = graph()->NewNode(
623  machine()->Store(
624  StoreRepresentation(MachineRepresentation::kWord32,
625  WriteBarrierKind::kNoWriteBarrier)),
626  stack_slot,
627  graph()->NewNode(
628  common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
629  GetReplacementLow(input), store_high_word, graph()->start());
630 
631  Node* load =
632  graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
633  graph()->NewNode(common()->Int32Constant(0)),
634  store_low_word, graph()->start());
635 
636  ReplaceNode(node, load, nullptr);
637  break;
638  }
639  case IrOpcode::kBitcastFloat64ToInt64: {
640  DCHECK_EQ(1, node->InputCount());
641  Node* input = node->InputAt(0);
642  if (HasReplacementLow(input)) {
643  input = GetReplacementLow(input);
644  }
645  Node* stack_slot = graph()->NewNode(
646  machine()->StackSlot(MachineRepresentation::kWord64));
647  Node* store = graph()->NewNode(
648  machine()->Store(
649  StoreRepresentation(MachineRepresentation::kFloat64,
650  WriteBarrierKind::kNoWriteBarrier)),
651  stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
652  graph()->start(), graph()->start());
653 
654  Node* high_node = graph()->NewNode(
655  machine()->Load(MachineType::Int32()), stack_slot,
656  graph()->NewNode(
657  common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
658  store, graph()->start());
659 
660  Node* low_node = graph()->NewNode(
661  machine()->Load(MachineType::Int32()), stack_slot,
662  graph()->NewNode(
663  common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
664  store, graph()->start());
665  ReplaceNode(node, low_node, high_node);
666  break;
667  }
668  case IrOpcode::kWord64Ror: {
669  DCHECK_EQ(2, node->InputCount());
670  Node* input = node->InputAt(0);
671  Node* shift = HasReplacementLow(node->InputAt(1))
672  ? GetReplacementLow(node->InputAt(1))
673  : node->InputAt(1);
674  Int32Matcher m(shift);
675  if (m.HasValue()) {
676  // Precondition: 0 <= shift < 64.
677  int32_t shift_value = m.Value() & 0x3F;
678  if (shift_value == 0) {
679  ReplaceNode(node, GetReplacementLow(input),
680  GetReplacementHigh(input));
681  } else if (shift_value == 32) {
682  ReplaceNode(node, GetReplacementHigh(input),
683  GetReplacementLow(input));
684  } else {
685  Node* low_input;
686  Node* high_input;
687  if (shift_value < 32) {
688  low_input = GetReplacementLow(input);
689  high_input = GetReplacementHigh(input);
690  } else {
691  low_input = GetReplacementHigh(input);
692  high_input = GetReplacementLow(input);
693  }
694  int32_t masked_shift_value = shift_value & 0x1F;
695  Node* masked_shift =
696  graph()->NewNode(common()->Int32Constant(masked_shift_value));
697  Node* inv_shift = graph()->NewNode(
698  common()->Int32Constant(32 - masked_shift_value));
699 
700  Node* low_node = graph()->NewNode(
701  machine()->Word32Or(),
702  graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
703  graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
704  Node* high_node = graph()->NewNode(
705  machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
706  high_input, masked_shift),
707  graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
708  ReplaceNode(node, low_node, high_node);
709  }
710  } else {
711  Node* safe_shift = shift;
712  if (!machine()->Word32ShiftIsSafe()) {
713  safe_shift =
714  graph()->NewNode(machine()->Word32And(), shift,
715  graph()->NewNode(common()->Int32Constant(0x1F)));
716  }
717 
718  // By creating this bit-mask with SAR and SHL we do not have to deal
719  // with shift == 0 as a special case.
720  Node* inv_mask = graph()->NewNode(
721  machine()->Word32Shl(),
722  graph()->NewNode(machine()->Word32Sar(),
723  graph()->NewNode(common()->Int32Constant(
724  std::numeric_limits<int32_t>::min())),
725  safe_shift),
726  graph()->NewNode(common()->Int32Constant(1)));
727 
728  Node* bit_mask =
729  graph()->NewNode(machine()->Word32Xor(), inv_mask,
730  graph()->NewNode(common()->Int32Constant(-1)));
731 
732  // We have to mask the shift value for this comparison. If
733  // !machine()->Word32ShiftIsSafe() then the masking should already be
734  // part of the graph.
735  Node* masked_shift6 = shift;
736  if (machine()->Word32ShiftIsSafe()) {
737  masked_shift6 =
738  graph()->NewNode(machine()->Word32And(), shift,
739  graph()->NewNode(common()->Int32Constant(0x3F)));
740  }
741 
742  Diamond lt32(
743  graph(), common(),
744  graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
745  graph()->NewNode(common()->Int32Constant(32))));
746 
747  // The low word and the high word can be swapped either at the input or
748  // at the output. We swap the inputs so that shift does not have to be
749  // kept for so long in a register.
750  Node* input_low =
751  lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
752  GetReplacementHigh(input));
753  Node* input_high =
754  lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
755  GetReplacementLow(input));
756 
757  Node* rotate_low =
758  graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
759  Node* rotate_high =
760  graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
761 
762  Node* low_node = graph()->NewNode(
763  machine()->Word32Or(),
764  graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
765  graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
766 
767  Node* high_node = graph()->NewNode(
768  machine()->Word32Or(),
769  graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
770  graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
771 
772  ReplaceNode(node, low_node, high_node);
773  }
774  break;
775  }
776  case IrOpcode::kWord64Clz: {
777  DCHECK_EQ(1, node->InputCount());
778  Node* input = node->InputAt(0);
779  Diamond d(
780  graph(), common(),
781  graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
782  graph()->NewNode(common()->Int32Constant(0))));
783 
784  Node* low_node = d.Phi(
785  MachineRepresentation::kWord32,
786  graph()->NewNode(machine()->Int32Add(),
787  graph()->NewNode(machine()->Word32Clz(),
788  GetReplacementLow(input)),
789  graph()->NewNode(common()->Int32Constant(32))),
790  graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
791  ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
792  break;
793  }
794  case IrOpcode::kWord64Ctz: {
795  DCHECK_EQ(1, node->InputCount());
796  DCHECK(machine()->Word32Ctz().IsSupported());
797  Node* input = node->InputAt(0);
798  Diamond d(
799  graph(), common(),
800  graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
801  graph()->NewNode(common()->Int32Constant(0))));
802  Node* low_node =
803  d.Phi(MachineRepresentation::kWord32,
804  graph()->NewNode(machine()->Int32Add(),
805  graph()->NewNode(machine()->Word32Ctz().op(),
806  GetReplacementHigh(input)),
807  graph()->NewNode(common()->Int32Constant(32))),
808  graph()->NewNode(machine()->Word32Ctz().op(),
809  GetReplacementLow(input)));
810  ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
811  break;
812  }
813  case IrOpcode::kWord64Popcnt: {
814  DCHECK_EQ(1, node->InputCount());
815  Node* input = node->InputAt(0);
816  // We assume that a Word64Popcnt node only has been created if
817  // Word32Popcnt is actually supported.
818  DCHECK(machine()->Word32Popcnt().IsSupported());
819  ReplaceNode(node, graph()->NewNode(
820  machine()->Int32Add(),
821  graph()->NewNode(machine()->Word32Popcnt().op(),
822  GetReplacementLow(input)),
823  graph()->NewNode(machine()->Word32Popcnt().op(),
824  GetReplacementHigh(input))),
825  graph()->NewNode(common()->Int32Constant(0)));
826  break;
827  }
828  case IrOpcode::kPhi: {
829  MachineRepresentation rep = PhiRepresentationOf(node->op());
830  if (rep == MachineRepresentation::kWord64) {
831  // The replacement nodes have already been created, we only have to
832  // replace placeholder nodes.
833  Node* low_node = GetReplacementLow(node);
834  Node* high_node = GetReplacementHigh(node);
835  for (int i = 0; i < node->op()->ValueInputCount(); i++) {
836  low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
837  high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
838  }
839  } else {
840  DefaultLowering(node);
841  }
842  break;
843  }
844  case IrOpcode::kWord64ReverseBytes: {
845  Node* input = node->InputAt(0);
846  ReplaceNode(node,
847  graph()->NewNode(machine()->Word32ReverseBytes(),
848  GetReplacementHigh(input)),
849  graph()->NewNode(machine()->Word32ReverseBytes(),
850  GetReplacementLow(input)));
851  break;
852  }
853  case IrOpcode::kSignExtendWord8ToInt64: {
854  DCHECK_EQ(1, node->InputCount());
855  Node* input = node->InputAt(0);
856  if (HasReplacementLow(input)) {
857  input = GetReplacementLow(input);
858  }
859  // Sign extend low node to Int32
860  input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
861 
862  // We use SAR to preserve the sign in the high word.
863  ReplaceNode(
864  node, input,
865  graph()->NewNode(machine()->Word32Sar(), input,
866  graph()->NewNode(common()->Int32Constant(31))));
867  node->NullAllInputs();
868  break;
869  }
870  case IrOpcode::kSignExtendWord16ToInt64: {
871  DCHECK_EQ(1, node->InputCount());
872  Node* input = node->InputAt(0);
873  if (HasReplacementLow(input)) {
874  input = GetReplacementLow(input);
875  }
876  // Sign extend low node to Int32
877  input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
878 
879  // We use SAR to preserve the sign in the high word.
880  ReplaceNode(
881  node, input,
882  graph()->NewNode(machine()->Word32Sar(), input,
883  graph()->NewNode(common()->Int32Constant(31))));
884  node->NullAllInputs();
885  break;
886  }
887  case IrOpcode::kWord64AtomicLoad: {
888  DCHECK_EQ(4, node->InputCount());
889  MachineType type = AtomicOpType(node->op());
890  if (type == MachineType::Uint64()) {
891  NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
892  ReplaceNodeWithProjections(node);
893  } else {
894  NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
895  ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
896  }
897  break;
898  }
899  case IrOpcode::kWord64AtomicStore: {
900  DCHECK_EQ(5, node->InputCount());
901  MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
902  if (rep == MachineRepresentation::kWord64) {
903  Node* value = node->InputAt(2);
904  node->ReplaceInput(2, GetReplacementLow(value));
905  node->InsertInput(zone(), 3, GetReplacementHigh(value));
906  NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
907  } else {
908  DefaultLowering(node, true);
909  NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
910  }
911  break;
912  }
913 #define ATOMIC_CASE(name) \
914  case IrOpcode::kWord64Atomic##name: { \
915  MachineType type = AtomicOpType(node->op()); \
916  if (type == MachineType::Uint64()) { \
917  LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \
918  } else { \
919  LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
920  } \
921  break; \
922  }
923  ATOMIC_CASE(Add)
924  ATOMIC_CASE(Sub)
925  ATOMIC_CASE(And)
926  ATOMIC_CASE(Or)
927  ATOMIC_CASE(Xor)
928  ATOMIC_CASE(Exchange)
929 #undef ATOMIC_CASE
930  case IrOpcode::kWord64AtomicCompareExchange: {
931  MachineType type = AtomicOpType(node->op());
932  if (type == MachineType::Uint64()) {
933  Node* old_value = node->InputAt(2);
934  Node* new_value = node->InputAt(3);
935  node->ReplaceInput(2, GetReplacementLow(old_value));
936  node->ReplaceInput(3, GetReplacementHigh(old_value));
937  node->InsertInput(zone(), 4, GetReplacementLow(new_value));
938  node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
939  NodeProperties::ChangeOp(node,
940  machine()->Word32AtomicPairCompareExchange());
941  ReplaceNodeWithProjections(node);
942  } else {
943  DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
944  type == MachineType::Uint8());
945  Node* old_value = node->InputAt(2);
946  node->ReplaceInput(2, GetReplacementLow(old_value));
947  Node* new_value = node->InputAt(3);
948  node->ReplaceInput(3, GetReplacementLow(new_value));
949  NodeProperties::ChangeOp(node,
950  machine()->Word32AtomicCompareExchange(type));
951  ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
952  }
953  break;
954  }
955 
956  default: { DefaultLowering(node); }
957  }
958 } // NOLINT(readability/fn_size)
959 
960 void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
961  const Operator* low_word_op) {
962  DCHECK_EQ(2, node->InputCount());
963  Node* left = node->InputAt(0);
964  Node* right = node->InputAt(1);
965  Node* replacement = graph()->NewNode(
966  machine()->Word32Or(),
967  graph()->NewNode(high_word_op, GetReplacementHigh(left),
968  GetReplacementHigh(right)),
969  graph()->NewNode(
970  machine()->Word32And(),
971  graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
972  GetReplacementHigh(right)),
973  graph()->NewNode(low_word_op, GetReplacementLow(left),
974  GetReplacementLow(right))));
975 
976  ReplaceNode(node, replacement, nullptr);
977 }
978 
979 bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
980  bool something_changed = false;
981  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
982  Node* input = node->InputAt(i);
983  if (HasReplacementLow(input)) {
984  something_changed = true;
985  node->ReplaceInput(i, GetReplacementLow(input));
986  }
987  if (!low_word_only && HasReplacementHigh(input)) {
988  something_changed = true;
989  node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
990  }
991  }
992  return something_changed;
993 }
994 
995 void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
996  // if new_low == nullptr, then also new_high == nullptr.
997  DCHECK(new_low != nullptr || new_high == nullptr);
998  replacements_[old->id()].low = new_low;
999  replacements_[old->id()].high = new_high;
1000 }
1001 
1002 bool Int64Lowering::HasReplacementLow(Node* node) {
1003  return replacements_[node->id()].low != nullptr;
1004 }
1005 
1006 Node* Int64Lowering::GetReplacementLow(Node* node) {
1007  Node* result = replacements_[node->id()].low;
1008  DCHECK(result);
1009  return result;
1010 }
1011 
1012 bool Int64Lowering::HasReplacementHigh(Node* node) {
1013  return replacements_[node->id()].high != nullptr;
1014 }
1015 
1016 Node* Int64Lowering::GetReplacementHigh(Node* node) {
1017  Node* result = replacements_[node->id()].high;
1018  DCHECK(result);
1019  return result;
1020 }
1021 
1022 void Int64Lowering::PreparePhiReplacement(Node* phi) {
1023  MachineRepresentation rep = PhiRepresentationOf(phi->op());
1024  if (rep == MachineRepresentation::kWord64) {
1025  // We have to create the replacements for a phi node before we actually
1026  // lower the phi to break potential cycles in the graph. The replacements of
1027  // input nodes do not exist yet, so we use a placeholder node to pass the
1028  // graph verifier.
1029  int value_count = phi->op()->ValueInputCount();
1030  Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1031  Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1032  for (int i = 0; i < value_count; i++) {
1033  inputs_low[i] = placeholder_;
1034  inputs_high[i] = placeholder_;
1035  }
1036  inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1037  inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1038  ReplaceNode(phi,
1039  graph()->NewNode(
1040  common()->Phi(MachineRepresentation::kWord32, value_count),
1041  value_count + 1, inputs_low, false),
1042  graph()->NewNode(
1043  common()->Phi(MachineRepresentation::kWord32, value_count),
1044  value_count + 1, inputs_high, false));
1045  }
1046 }
1047 
1048 void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1049  DCHECK(node != nullptr);
1050  Node* low_node =
1051  graph()->NewNode(common()->Projection(0), node, graph()->start());
1052  Node* high_node =
1053  graph()->NewNode(common()->Projection(1), node, graph()->start());
1054  ReplaceNode(node, low_node, high_node);
1055 }
1056 
1057 } // namespace compiler
1058 } // namespace internal
1059 } // namespace v8
int64_t
v8
Definition: libplatform.h:13