V8 API Reference, 7.2.502.16 (for Deno 0.2.4)
wasm-compiler.cc
1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "src/compiler/wasm-compiler.h"
6 
7 #include <memory>
8 
9 #include "src/assembler-inl.h"
10 #include "src/assembler.h"
11 #include "src/base/optional.h"
12 #include "src/base/platform/elapsed-timer.h"
13 #include "src/base/platform/platform.h"
14 #include "src/base/v8-fallthrough.h"
15 #include "src/builtins/builtins.h"
16 #include "src/code-factory.h"
17 #include "src/compiler/backend/code-generator.h"
18 #include "src/compiler/backend/instruction-selector.h"
19 #include "src/compiler/common-operator.h"
20 #include "src/compiler/compiler-source-position-table.h"
21 #include "src/compiler/diamond.h"
22 #include "src/compiler/graph-visualizer.h"
23 #include "src/compiler/graph.h"
24 #include "src/compiler/int64-lowering.h"
25 #include "src/compiler/js-graph.h"
26 #include "src/compiler/js-operator.h"
27 #include "src/compiler/linkage.h"
28 #include "src/compiler/machine-operator.h"
29 #include "src/compiler/node-matchers.h"
30 #include "src/compiler/node-origin-table.h"
31 #include "src/compiler/pipeline.h"
32 #include "src/compiler/simd-scalar-lowering.h"
33 #include "src/compiler/zone-stats.h"
34 #include "src/counters.h"
35 #include "src/heap/factory.h"
36 #include "src/isolate-inl.h"
37 #include "src/log-inl.h"
38 #include "src/optimized-compilation-info.h"
39 #include "src/tracing/trace-event.h"
40 #include "src/trap-handler/trap-handler.h"
41 #include "src/wasm/function-body-decoder.h"
42 #include "src/wasm/function-compiler.h"
43 #include "src/wasm/graph-builder-interface.h"
44 #include "src/wasm/jump-table-assembler.h"
45 #include "src/wasm/memory-tracing.h"
46 #include "src/wasm/object-access.h"
47 #include "src/wasm/wasm-code-manager.h"
48 #include "src/wasm/wasm-limits.h"
49 #include "src/wasm/wasm-linkage.h"
50 #include "src/wasm/wasm-module.h"
51 #include "src/wasm/wasm-objects-inl.h"
52 #include "src/wasm/wasm-opcodes.h"
53 #include "src/wasm/wasm-text.h"
54 
55 namespace v8 {
56 namespace internal {
57 namespace compiler {
58 
59 namespace {
60 
61 // TODO(titzer): pull WASM_64 up to a common header.
62 #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
63 #define WASM_64 1
64 #else
65 #define WASM_64 0
66 #endif
67 
68 #define FATAL_UNSUPPORTED_OPCODE(opcode) \
69  FATAL("Unsupported opcode 0x%x:%s", (opcode), \
70  wasm::WasmOpcodes::OpcodeName(opcode));
71 
72 MachineType assert_size(int expected_size, MachineType type) {
73  DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()));
74  return type;
75 }
76 
77 #define WASM_INSTANCE_OBJECT_SIZE(name) \
78  (WasmInstanceObject::k##name##OffsetEnd - \
79  WasmInstanceObject::k##name##Offset + 1) // NOLINT(whitespace/indent)
80 
81 #define WASM_INSTANCE_OBJECT_OFFSET(name) \
82  wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
83 
84 #define LOAD_RAW(base_pointer, byte_offset, type) \
85  SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), base_pointer, \
86  mcgraph()->Int32Constant(byte_offset), Effect(), \
87  Control()))
88 
89 #define LOAD_INSTANCE_FIELD(name, type) \
90  LOAD_RAW(instance_node_.get(), WASM_INSTANCE_OBJECT_OFFSET(name), \
91  assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type))
92 
93 #define LOAD_TAGGED_POINTER(base_pointer, byte_offset) \
94  LOAD_RAW(base_pointer, byte_offset, MachineType::TaggedPointer())
95 
96 #define LOAD_TAGGED_ANY(base_pointer, byte_offset) \
97  LOAD_RAW(base_pointer, byte_offset, MachineType::AnyTagged())
98 
99 #define LOAD_FIXED_ARRAY_SLOT(array_node, index, type) \
100  LOAD_RAW(array_node, \
101  wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), type)
102 
103 #define LOAD_FIXED_ARRAY_SLOT_SMI(array_node, index) \
104  LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedSigned())
105 
106 #define LOAD_FIXED_ARRAY_SLOT_PTR(array_node, index) \
107  LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedPointer())
108 
109 #define LOAD_FIXED_ARRAY_SLOT_ANY(array_node, index) \
110  LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::AnyTagged())
111 
112 // This can be used to store tagged Smi values only.
113 #define STORE_FIXED_ARRAY_SLOT_SMI(array_node, index, value) \
114  SetEffect(graph()->NewNode( \
115  mcgraph()->machine()->Store(StoreRepresentation( \
116  MachineRepresentation::kTaggedSigned, kNoWriteBarrier)), \
117  array_node, \
118  mcgraph()->Int32Constant( \
119  wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
120  value, Effect(), Control()))
121 
122 // This can be used to store any tagged (Smi and HeapObject) value.
123 #define STORE_FIXED_ARRAY_SLOT_ANY(array_node, index, value) \
124  SetEffect(graph()->NewNode( \
125  mcgraph()->machine()->Store(StoreRepresentation( \
126  MachineRepresentation::kTagged, kFullWriteBarrier)), \
127  array_node, \
128  mcgraph()->Int32Constant( \
129  wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
130  value, Effect(), Control()))
131 
132 void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
133  Graph* g = mcgraph->graph();
134  if (g->end()) {
135  NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
136  } else {
137  g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
138  }
139 }
140 
141 bool ContainsSimd(wasm::FunctionSig* sig) {
142  for (auto type : sig->all()) {
143  if (type == wasm::kWasmS128) return true;
144  }
145  return false;
146 }
147 
148 bool ContainsInt64(wasm::FunctionSig* sig) {
149  for (auto type : sig->all()) {
150  if (type == wasm::kWasmI64) return true;
151  }
152  return false;
153 }
154 } // namespace
155 
156 WasmGraphBuilder::WasmGraphBuilder(
157  wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
158  wasm::FunctionSig* sig,
159  compiler::SourcePositionTable* source_position_table)
160  : zone_(zone),
161  mcgraph_(mcgraph),
162  env_(env),
163  cur_buffer_(def_buffer_),
164  cur_bufsize_(kDefaultBufferSize),
165  has_simd_(ContainsSimd(sig)),
166  untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
167  sig_(sig),
168  source_position_table_(source_position_table) {
169  DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
170  DCHECK_NOT_NULL(mcgraph_);
171 }
172 
173 Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
174 
175 Node* WasmGraphBuilder::Start(unsigned params) {
176  Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
177  graph()->SetStart(start);
178  return start;
179 }
180 
181 Node* WasmGraphBuilder::Param(unsigned index) {
182  return graph()->NewNode(mcgraph()->common()->Parameter(index),
183  graph()->start());
184 }
185 
186 Node* WasmGraphBuilder::Loop(Node* entry) {
187  return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
188 }
189 
190 Node* WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
191  Node* terminate =
192  graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
193  MergeControlToEnd(mcgraph(), terminate);
194  return terminate;
195 }
196 
197 Node* WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
198  Node* terminate =
199  graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
200  MergeControlToEnd(mcgraph(), terminate);
201  return terminate;
202 }
203 
204 bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
205  return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
206  NodeProperties::GetControlInput(phi) == merge;
207 }
208 
209 bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
210  Node** if_exception) {
211  if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
212  return false;
213  }
214 
215  *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
216  *if_exception =
217  graph()->NewNode(mcgraph()->common()->IfException(), node, node);
218 
219  return true;
220 }
221 
222 void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
223  DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
224  merge->AppendInput(mcgraph()->zone(), from);
225  int new_size = merge->InputCount();
226  NodeProperties::ChangeOp(
227  merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
228 }
229 
230 void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
231  DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
232  int new_size = phi->InputCount();
233  phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
234  NodeProperties::ChangeOp(
235  phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
236 }
237 
238 Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
239  return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
240 }
241 
242 Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
243  Node* control) {
244  DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
245  Node** buf = Realloc(vals, count, count + 1);
246  buf[count] = control;
247  return graph()->NewNode(
248  mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
249  count),
250  count + 1, buf);
251 }
252 
253 Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
254  Node* control) {
255  DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
256  Node** buf = Realloc(effects, count, count + 1);
257  buf[count] = control;
258  return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
259  buf);
260 }
261 
262 Node* WasmGraphBuilder::RefNull() {
263  return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
264 }
265 
266 Node* WasmGraphBuilder::NoContextConstant() {
267  // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
268  return mcgraph()->IntPtrConstant(0);
269 }
270 
271 Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
272  return mcgraph()->Uint32Constant(value);
273 }
274 
275 Node* WasmGraphBuilder::Int32Constant(int32_t value) {
276  return mcgraph()->Int32Constant(value);
277 }
278 
279 Node* WasmGraphBuilder::Int64Constant(int64_t value) {
280  return mcgraph()->Int64Constant(value);
281 }
282 
283 Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
284  return mcgraph()->IntPtrConstant(value);
285 }
286 
287 void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
288  Node** effect, Node** control) {
289  DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks.
290  if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
291  return;
292  }
293  if (effect == nullptr) effect = effect_;
294  if (control == nullptr) control = control_;
295 
296  // This instruction sequence is matched in the instruction selector to
297  // load the stack pointer directly on some platforms. Hence, when modifying
298  // please also fix WasmStackCheckMatcher in node-matchers.h
299 
300  Node* limit_address = graph()->NewNode(
301  mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
302  mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
303  *effect, *control);
304  Node* limit = graph()->NewNode(
305  mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
306  mcgraph()->IntPtrConstant(0), limit_address, *control);
307  *effect = limit;
308  Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
309 
310  Node* check =
311  graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
312 
313  Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
314  stack_check.Chain(*control);
315 
316  if (stack_check_call_operator_ == nullptr) {
317  // Build and cache the stack check call operator and the constant
318  // representing the stack check code.
319  auto call_descriptor = Linkage::GetStubCallDescriptor(
320  mcgraph()->zone(), // zone
321  NoContextDescriptor{}, // descriptor
322  0, // stack parameter count
323  CallDescriptor::kNoFlags, // flags
324  Operator::kNoProperties, // properties
325  StubCallMode::kCallWasmRuntimeStub); // stub call mode
326  // A direct call to a wasm runtime stub defined in this module.
327  // Just encode the stub index. This will be patched at relocation.
328  stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
329  wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
330  stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
331  }
332 
333  Node* call = graph()->NewNode(stack_check_call_operator_.get(),
334  stack_check_code_node_.get(), *effect,
335  stack_check.if_false);
336 
337  SetSourcePosition(call, position);
338 
339  Node* ephi = stack_check.EffectPhi(*effect, call);
340 
341  *control = stack_check.merge;
342  *effect = ephi;
343 }
344 
345 void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
346  if (!needs_stack_check_) return;
347 
348  Node* start = graph()->start();
349  // Place a stack check which uses a dummy node as control and effect.
350  Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
351  Node* control = dummy;
352  Node* effect = dummy;
353  // The function-prologue stack check is associated with position 0, which
354  // is never a position of any instruction in the function.
355  StackCheck(0, &effect, &control);
356 
357  // In testing, no steck checks were emitted. Nothing to rewire then.
358  if (effect == dummy) return;
359 
360  // Now patch all control uses of {start} to use {control} and all effect uses
361  // to use {effect} instead. Then rewire the dummy node to use start instead.
362  NodeProperties::ReplaceUses(start, start, effect, control);
363  NodeProperties::ReplaceUses(dummy, nullptr, start, start);
364 }
365 
366 Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
367  wasm::WasmCodePosition position) {
368  const Operator* op;
369  MachineOperatorBuilder* m = mcgraph()->machine();
370  switch (opcode) {
371  case wasm::kExprI32Add:
372  op = m->Int32Add();
373  break;
374  case wasm::kExprI32Sub:
375  op = m->Int32Sub();
376  break;
377  case wasm::kExprI32Mul:
378  op = m->Int32Mul();
379  break;
380  case wasm::kExprI32DivS:
381  return BuildI32DivS(left, right, position);
382  case wasm::kExprI32DivU:
383  return BuildI32DivU(left, right, position);
384  case wasm::kExprI32RemS:
385  return BuildI32RemS(left, right, position);
386  case wasm::kExprI32RemU:
387  return BuildI32RemU(left, right, position);
388  case wasm::kExprI32And:
389  op = m->Word32And();
390  break;
391  case wasm::kExprI32Ior:
392  op = m->Word32Or();
393  break;
394  case wasm::kExprI32Xor:
395  op = m->Word32Xor();
396  break;
397  case wasm::kExprI32Shl:
398  op = m->Word32Shl();
399  right = MaskShiftCount32(right);
400  break;
401  case wasm::kExprI32ShrU:
402  op = m->Word32Shr();
403  right = MaskShiftCount32(right);
404  break;
405  case wasm::kExprI32ShrS:
406  op = m->Word32Sar();
407  right = MaskShiftCount32(right);
408  break;
409  case wasm::kExprI32Ror:
410  op = m->Word32Ror();
411  right = MaskShiftCount32(right);
412  break;
413  case wasm::kExprI32Rol:
414  right = MaskShiftCount32(right);
415  return BuildI32Rol(left, right);
416  case wasm::kExprI32Eq:
417  op = m->Word32Equal();
418  break;
419  case wasm::kExprI32Ne:
420  return Invert(Binop(wasm::kExprI32Eq, left, right));
421  case wasm::kExprI32LtS:
422  op = m->Int32LessThan();
423  break;
424  case wasm::kExprI32LeS:
425  op = m->Int32LessThanOrEqual();
426  break;
427  case wasm::kExprI32LtU:
428  op = m->Uint32LessThan();
429  break;
430  case wasm::kExprI32LeU:
431  op = m->Uint32LessThanOrEqual();
432  break;
433  case wasm::kExprI32GtS:
434  op = m->Int32LessThan();
435  std::swap(left, right);
436  break;
437  case wasm::kExprI32GeS:
438  op = m->Int32LessThanOrEqual();
439  std::swap(left, right);
440  break;
441  case wasm::kExprI32GtU:
442  op = m->Uint32LessThan();
443  std::swap(left, right);
444  break;
445  case wasm::kExprI32GeU:
446  op = m->Uint32LessThanOrEqual();
447  std::swap(left, right);
448  break;
449  case wasm::kExprI64And:
450  op = m->Word64And();
451  break;
452  case wasm::kExprI64Add:
453  op = m->Int64Add();
454  break;
455  case wasm::kExprI64Sub:
456  op = m->Int64Sub();
457  break;
458  case wasm::kExprI64Mul:
459  op = m->Int64Mul();
460  break;
461  case wasm::kExprI64DivS:
462  return BuildI64DivS(left, right, position);
463  case wasm::kExprI64DivU:
464  return BuildI64DivU(left, right, position);
465  case wasm::kExprI64RemS:
466  return BuildI64RemS(left, right, position);
467  case wasm::kExprI64RemU:
468  return BuildI64RemU(left, right, position);
469  case wasm::kExprI64Ior:
470  op = m->Word64Or();
471  break;
472  case wasm::kExprI64Xor:
473  op = m->Word64Xor();
474  break;
475  case wasm::kExprI64Shl:
476  op = m->Word64Shl();
477  right = MaskShiftCount64(right);
478  break;
479  case wasm::kExprI64ShrU:
480  op = m->Word64Shr();
481  right = MaskShiftCount64(right);
482  break;
483  case wasm::kExprI64ShrS:
484  op = m->Word64Sar();
485  right = MaskShiftCount64(right);
486  break;
487  case wasm::kExprI64Eq:
488  op = m->Word64Equal();
489  break;
490  case wasm::kExprI64Ne:
491  return Invert(Binop(wasm::kExprI64Eq, left, right));
492  case wasm::kExprI64LtS:
493  op = m->Int64LessThan();
494  break;
495  case wasm::kExprI64LeS:
496  op = m->Int64LessThanOrEqual();
497  break;
498  case wasm::kExprI64LtU:
499  op = m->Uint64LessThan();
500  break;
501  case wasm::kExprI64LeU:
502  op = m->Uint64LessThanOrEqual();
503  break;
504  case wasm::kExprI64GtS:
505  op = m->Int64LessThan();
506  std::swap(left, right);
507  break;
508  case wasm::kExprI64GeS:
509  op = m->Int64LessThanOrEqual();
510  std::swap(left, right);
511  break;
512  case wasm::kExprI64GtU:
513  op = m->Uint64LessThan();
514  std::swap(left, right);
515  break;
516  case wasm::kExprI64GeU:
517  op = m->Uint64LessThanOrEqual();
518  std::swap(left, right);
519  break;
520  case wasm::kExprI64Ror:
521  op = m->Word64Ror();
522  right = MaskShiftCount64(right);
523  break;
524  case wasm::kExprI64Rol:
525  return BuildI64Rol(left, right);
526  case wasm::kExprF32CopySign:
527  return BuildF32CopySign(left, right);
528  case wasm::kExprF64CopySign:
529  return BuildF64CopySign(left, right);
530  case wasm::kExprF32Add:
531  op = m->Float32Add();
532  break;
533  case wasm::kExprF32Sub:
534  op = m->Float32Sub();
535  break;
536  case wasm::kExprF32Mul:
537  op = m->Float32Mul();
538  break;
539  case wasm::kExprF32Div:
540  op = m->Float32Div();
541  break;
542  case wasm::kExprF32Eq:
543  op = m->Float32Equal();
544  break;
545  case wasm::kExprF32Ne:
546  return Invert(Binop(wasm::kExprF32Eq, left, right));
547  case wasm::kExprF32Lt:
548  op = m->Float32LessThan();
549  break;
550  case wasm::kExprF32Ge:
551  op = m->Float32LessThanOrEqual();
552  std::swap(left, right);
553  break;
554  case wasm::kExprF32Gt:
555  op = m->Float32LessThan();
556  std::swap(left, right);
557  break;
558  case wasm::kExprF32Le:
559  op = m->Float32LessThanOrEqual();
560  break;
561  case wasm::kExprF64Add:
562  op = m->Float64Add();
563  break;
564  case wasm::kExprF64Sub:
565  op = m->Float64Sub();
566  break;
567  case wasm::kExprF64Mul:
568  op = m->Float64Mul();
569  break;
570  case wasm::kExprF64Div:
571  op = m->Float64Div();
572  break;
573  case wasm::kExprF64Eq:
574  op = m->Float64Equal();
575  break;
576  case wasm::kExprF64Ne:
577  return Invert(Binop(wasm::kExprF64Eq, left, right));
578  case wasm::kExprF64Lt:
579  op = m->Float64LessThan();
580  break;
581  case wasm::kExprF64Le:
582  op = m->Float64LessThanOrEqual();
583  break;
584  case wasm::kExprF64Gt:
585  op = m->Float64LessThan();
586  std::swap(left, right);
587  break;
588  case wasm::kExprF64Ge:
589  op = m->Float64LessThanOrEqual();
590  std::swap(left, right);
591  break;
592  case wasm::kExprF32Min:
593  op = m->Float32Min();
594  break;
595  case wasm::kExprF64Min:
596  op = m->Float64Min();
597  break;
598  case wasm::kExprF32Max:
599  op = m->Float32Max();
600  break;
601  case wasm::kExprF64Max:
602  op = m->Float64Max();
603  break;
604  case wasm::kExprF64Pow:
605  return BuildF64Pow(left, right);
606  case wasm::kExprF64Atan2:
607  op = m->Float64Atan2();
608  break;
609  case wasm::kExprF64Mod:
610  return BuildF64Mod(left, right);
611  case wasm::kExprI32AsmjsDivS:
612  return BuildI32AsmjsDivS(left, right);
613  case wasm::kExprI32AsmjsDivU:
614  return BuildI32AsmjsDivU(left, right);
615  case wasm::kExprI32AsmjsRemS:
616  return BuildI32AsmjsRemS(left, right);
617  case wasm::kExprI32AsmjsRemU:
618  return BuildI32AsmjsRemU(left, right);
619  case wasm::kExprI32AsmjsStoreMem8:
620  return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
621  case wasm::kExprI32AsmjsStoreMem16:
622  return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
623  case wasm::kExprI32AsmjsStoreMem:
624  return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
625  case wasm::kExprF32AsmjsStoreMem:
626  return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
627  case wasm::kExprF64AsmjsStoreMem:
628  return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
629  default:
630  FATAL_UNSUPPORTED_OPCODE(opcode);
631  }
632  return graph()->NewNode(op, left, right);
633 }
634 
635 Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
636  wasm::WasmCodePosition position) {
637  const Operator* op;
638  MachineOperatorBuilder* m = mcgraph()->machine();
639  switch (opcode) {
640  case wasm::kExprI32Eqz:
641  op = m->Word32Equal();
642  return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
643  case wasm::kExprF32Abs:
644  op = m->Float32Abs();
645  break;
646  case wasm::kExprF32Neg: {
647  op = m->Float32Neg();
648  break;
649  }
650  case wasm::kExprF32Sqrt:
651  op = m->Float32Sqrt();
652  break;
653  case wasm::kExprF64Abs:
654  op = m->Float64Abs();
655  break;
656  case wasm::kExprF64Neg: {
657  op = m->Float64Neg();
658  break;
659  }
660  case wasm::kExprF64Sqrt:
661  op = m->Float64Sqrt();
662  break;
663  case wasm::kExprI32SConvertF32:
664  case wasm::kExprI32UConvertF32:
665  case wasm::kExprI32SConvertF64:
666  case wasm::kExprI32UConvertF64:
667  case wasm::kExprI32SConvertSatF64:
668  case wasm::kExprI32UConvertSatF64:
669  case wasm::kExprI32SConvertSatF32:
670  case wasm::kExprI32UConvertSatF32:
671  return BuildIntConvertFloat(input, position, opcode);
672  case wasm::kExprI32AsmjsSConvertF64:
673  return BuildI32AsmjsSConvertF64(input);
674  case wasm::kExprI32AsmjsUConvertF64:
675  return BuildI32AsmjsUConvertF64(input);
676  case wasm::kExprF32ConvertF64:
677  op = m->TruncateFloat64ToFloat32();
678  break;
679  case wasm::kExprF64SConvertI32:
680  op = m->ChangeInt32ToFloat64();
681  break;
682  case wasm::kExprF64UConvertI32:
683  op = m->ChangeUint32ToFloat64();
684  break;
685  case wasm::kExprF32SConvertI32:
686  op = m->RoundInt32ToFloat32();
687  break;
688  case wasm::kExprF32UConvertI32:
689  op = m->RoundUint32ToFloat32();
690  break;
691  case wasm::kExprI32AsmjsSConvertF32:
692  return BuildI32AsmjsSConvertF32(input);
693  case wasm::kExprI32AsmjsUConvertF32:
694  return BuildI32AsmjsUConvertF32(input);
695  case wasm::kExprF64ConvertF32:
696  op = m->ChangeFloat32ToFloat64();
697  break;
698  case wasm::kExprF32ReinterpretI32:
699  op = m->BitcastInt32ToFloat32();
700  break;
701  case wasm::kExprI32ReinterpretF32:
702  op = m->BitcastFloat32ToInt32();
703  break;
704  case wasm::kExprI32Clz:
705  op = m->Word32Clz();
706  break;
707  case wasm::kExprI32Ctz: {
708  if (m->Word32Ctz().IsSupported()) {
709  op = m->Word32Ctz().op();
710  break;
711  } else if (m->Word32ReverseBits().IsSupported()) {
712  Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
713  Node* result = graph()->NewNode(m->Word32Clz(), reversed);
714  return result;
715  } else {
716  return BuildI32Ctz(input);
717  }
718  }
719  case wasm::kExprI32Popcnt: {
720  if (m->Word32Popcnt().IsSupported()) {
721  op = m->Word32Popcnt().op();
722  break;
723  } else {
724  return BuildI32Popcnt(input);
725  }
726  }
727  case wasm::kExprF32Floor: {
728  if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
729  op = m->Float32RoundDown().op();
730  break;
731  }
732  case wasm::kExprF32Ceil: {
733  if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
734  op = m->Float32RoundUp().op();
735  break;
736  }
737  case wasm::kExprF32Trunc: {
738  if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
739  op = m->Float32RoundTruncate().op();
740  break;
741  }
742  case wasm::kExprF32NearestInt: {
743  if (!m->Float32RoundTiesEven().IsSupported())
744  return BuildF32NearestInt(input);
745  op = m->Float32RoundTiesEven().op();
746  break;
747  }
748  case wasm::kExprF64Floor: {
749  if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
750  op = m->Float64RoundDown().op();
751  break;
752  }
753  case wasm::kExprF64Ceil: {
754  if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
755  op = m->Float64RoundUp().op();
756  break;
757  }
758  case wasm::kExprF64Trunc: {
759  if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
760  op = m->Float64RoundTruncate().op();
761  break;
762  }
763  case wasm::kExprF64NearestInt: {
764  if (!m->Float64RoundTiesEven().IsSupported())
765  return BuildF64NearestInt(input);
766  op = m->Float64RoundTiesEven().op();
767  break;
768  }
769  case wasm::kExprF64Acos: {
770  return BuildF64Acos(input);
771  }
772  case wasm::kExprF64Asin: {
773  return BuildF64Asin(input);
774  }
775  case wasm::kExprF64Atan:
776  op = m->Float64Atan();
777  break;
778  case wasm::kExprF64Cos: {
779  op = m->Float64Cos();
780  break;
781  }
782  case wasm::kExprF64Sin: {
783  op = m->Float64Sin();
784  break;
785  }
786  case wasm::kExprF64Tan: {
787  op = m->Float64Tan();
788  break;
789  }
790  case wasm::kExprF64Exp: {
791  op = m->Float64Exp();
792  break;
793  }
794  case wasm::kExprF64Log:
795  op = m->Float64Log();
796  break;
797  case wasm::kExprI32ConvertI64:
798  op = m->TruncateInt64ToInt32();
799  break;
800  case wasm::kExprI64SConvertI32:
801  op = m->ChangeInt32ToInt64();
802  break;
803  case wasm::kExprI64UConvertI32:
804  op = m->ChangeUint32ToUint64();
805  break;
806  case wasm::kExprF64ReinterpretI64:
807  op = m->BitcastInt64ToFloat64();
808  break;
809  case wasm::kExprI64ReinterpretF64:
810  op = m->BitcastFloat64ToInt64();
811  break;
812  case wasm::kExprI64Clz:
813  op = m->Word64Clz();
814  break;
815  case wasm::kExprI64Ctz: {
816  OptionalOperator ctz64 = m->Word64Ctz();
817  if (ctz64.IsSupported()) {
818  op = ctz64.op();
819  break;
820  } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
821  op = ctz64.placeholder();
822  break;
823  } else if (m->Word64ReverseBits().IsSupported()) {
824  Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
825  Node* result = graph()->NewNode(m->Word64Clz(), reversed);
826  return result;
827  } else {
828  return BuildI64Ctz(input);
829  }
830  }
831  case wasm::kExprI64Popcnt: {
832  OptionalOperator popcnt64 = m->Word64Popcnt();
833  if (popcnt64.IsSupported()) {
834  op = popcnt64.op();
835  } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
836  op = popcnt64.placeholder();
837  } else {
838  return BuildI64Popcnt(input);
839  }
840  break;
841  }
842  case wasm::kExprI64Eqz:
843  op = m->Word64Equal();
844  return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
845  case wasm::kExprF32SConvertI64:
846  if (m->Is32()) {
847  return BuildF32SConvertI64(input);
848  }
849  op = m->RoundInt64ToFloat32();
850  break;
851  case wasm::kExprF32UConvertI64:
852  if (m->Is32()) {
853  return BuildF32UConvertI64(input);
854  }
855  op = m->RoundUint64ToFloat32();
856  break;
857  case wasm::kExprF64SConvertI64:
858  if (m->Is32()) {
859  return BuildF64SConvertI64(input);
860  }
861  op = m->RoundInt64ToFloat64();
862  break;
863  case wasm::kExprF64UConvertI64:
864  if (m->Is32()) {
865  return BuildF64UConvertI64(input);
866  }
867  op = m->RoundUint64ToFloat64();
868  break;
869  case wasm::kExprI32SExtendI8:
870  op = m->SignExtendWord8ToInt32();
871  break;
872  case wasm::kExprI32SExtendI16:
873  op = m->SignExtendWord16ToInt32();
874  break;
875  case wasm::kExprI64SExtendI8:
876  op = m->SignExtendWord8ToInt64();
877  break;
878  case wasm::kExprI64SExtendI16:
879  op = m->SignExtendWord16ToInt64();
880  break;
881  case wasm::kExprI64SExtendI32:
882  op = m->SignExtendWord32ToInt64();
883  break;
884  case wasm::kExprI64SConvertF32:
885  case wasm::kExprI64UConvertF32:
886  case wasm::kExprI64SConvertF64:
887  case wasm::kExprI64UConvertF64:
888  case wasm::kExprI64SConvertSatF32:
889  case wasm::kExprI64UConvertSatF32:
890  case wasm::kExprI64SConvertSatF64:
891  case wasm::kExprI64UConvertSatF64:
892  return mcgraph()->machine()->Is32()
893  ? BuildCcallConvertFloat(input, position, opcode)
894  : BuildIntConvertFloat(input, position, opcode);
895  case wasm::kExprRefIsNull:
896  return graph()->NewNode(m->WordEqual(), input, RefNull());
897  case wasm::kExprI32AsmjsLoadMem8S:
898  return BuildAsmjsLoadMem(MachineType::Int8(), input);
899  case wasm::kExprI32AsmjsLoadMem8U:
900  return BuildAsmjsLoadMem(MachineType::Uint8(), input);
901  case wasm::kExprI32AsmjsLoadMem16S:
902  return BuildAsmjsLoadMem(MachineType::Int16(), input);
903  case wasm::kExprI32AsmjsLoadMem16U:
904  return BuildAsmjsLoadMem(MachineType::Uint16(), input);
905  case wasm::kExprI32AsmjsLoadMem:
906  return BuildAsmjsLoadMem(MachineType::Int32(), input);
907  case wasm::kExprF32AsmjsLoadMem:
908  return BuildAsmjsLoadMem(MachineType::Float32(), input);
909  case wasm::kExprF64AsmjsLoadMem:
910  return BuildAsmjsLoadMem(MachineType::Float64(), input);
911  default:
912  FATAL_UNSUPPORTED_OPCODE(opcode);
913  }
914  return graph()->NewNode(op, input);
915 }
916 
917 Node* WasmGraphBuilder::Float32Constant(float value) {
918  return mcgraph()->Float32Constant(value);
919 }
920 
921 Node* WasmGraphBuilder::Float64Constant(double value) {
922  return mcgraph()->Float64Constant(value);
923 }
924 
925 namespace {
926 Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
927  Node** false_node, Node* control, BranchHint hint) {
928  DCHECK_NOT_NULL(cond);
929  DCHECK_NOT_NULL(control);
930  Node* branch =
931  mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
932  *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
933  *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
934  return branch;
935 }
936 } // namespace
937 
938 Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
939  Node** false_node) {
940  return Branch(mcgraph(), cond, true_node, false_node, Control(),
941  BranchHint::kNone);
942 }
943 
944 Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
945  Node** false_node) {
946  return Branch(mcgraph(), cond, true_node, false_node, Control(),
947  BranchHint::kTrue);
948 }
949 
950 Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
951  Node** false_node) {
952  return Branch(mcgraph(), cond, true_node, false_node, Control(),
953  BranchHint::kFalse);
954 }
955 
956 TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
957  // TODO(wasm): "!env_" should not happen when compiling an actual wasm
958  // function.
959  if (!env_ || !env_->runtime_exception_support) {
960  // We use TrapId::kInvalid as a marker to tell the code generator
961  // to generate a call to a testing c-function instead of a runtime
962  // stub. This code should only be called from a cctest.
963  return TrapId::kInvalid;
964  }
965 
966  switch (reason) {
967 #define TRAPREASON_TO_TRAPID(name) \
968  case wasm::k##name: \
969  static_assert( \
970  static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
971  "trap id mismatch"); \
972  return TrapId::k##name;
973  FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
974 #undef TRAPREASON_TO_TRAPID
975  default:
976  UNREACHABLE();
977  }
978 }
979 
980 Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
981  wasm::WasmCodePosition position) {
982  TrapId trap_id = GetTrapIdForTrap(reason);
983  Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
984  cond, Effect(), Control()));
985  SetSourcePosition(node, position);
986  return node;
987 }
988 
989 Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
990  wasm::WasmCodePosition position) {
991  TrapId trap_id = GetTrapIdForTrap(reason);
992  Node* node = SetControl(graph()->NewNode(
993  mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
994  SetSourcePosition(node, position);
995  return node;
996 }
997 
998 // Add a check that traps if {node} is equal to {val}.
999 Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
1000  int32_t val,
1001  wasm::WasmCodePosition position) {
1002  Int32Matcher m(node);
1003  if (m.HasValue() && !m.Is(val)) return graph()->start();
1004  if (val == 0) {
1005  return TrapIfFalse(reason, node, position);
1006  } else {
1007  return TrapIfTrue(reason,
1008  graph()->NewNode(mcgraph()->machine()->Word32Equal(),
1009  node, mcgraph()->Int32Constant(val)),
1010  position);
1011  }
1012 }
1013 
1014 // Add a check that traps if {node} is zero.
1015 Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
1016  wasm::WasmCodePosition position) {
1017  return TrapIfEq32(reason, node, 0, position);
1018 }
1019 
1020 // Add a check that traps if {node} is equal to {val}.
1021 Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
1022  int64_t val,
1023  wasm::WasmCodePosition position) {
1024  Int64Matcher m(node);
1025  if (m.HasValue() && !m.Is(val)) return graph()->start();
1026  return TrapIfTrue(reason,
1027  graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
1028  mcgraph()->Int64Constant(val)),
1029  position);
1030 }
1031 
1032 // Add a check that traps if {node} is zero.
1033 Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
1034  wasm::WasmCodePosition position) {
1035  return TrapIfEq64(reason, node, 0, position);
1036 }
1037 
1038 Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
1039  return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
1040 }
1041 
1042 Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1043  DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1044  return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1045 }
1046 
1047 Node* WasmGraphBuilder::IfDefault(Node* sw) {
1048  DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1049  return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1050 }
1051 
1052 Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1053  static const int kStackAllocatedNodeBufferSize = 8;
1054  Node* stack_buffer[kStackAllocatedNodeBufferSize];
1055  std::vector<Node*> heap_buffer;
1056 
1057  Node** buf = stack_buffer;
1058  if (count + 3 > kStackAllocatedNodeBufferSize) {
1059  heap_buffer.resize(count + 3);
1060  buf = heap_buffer.data();
1061  }
1062 
1063  buf[0] = mcgraph()->Int32Constant(0);
1064  memcpy(buf + 1, vals, sizeof(void*) * count);
1065  buf[count + 1] = Effect();
1066  buf[count + 2] = Control();
1067  Node* ret =
1068  graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1069 
1070  MergeControlToEnd(mcgraph(), ret);
1071  return ret;
1072 }
1073 
1074 Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1075 
1076 Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1077  TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1078  ReturnVoid();
1079  return nullptr;
1080 }
1081 
1082 Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1083  static const int32_t kMask32 = 0x1F;
1084  if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1085  // Shifts by constants are so common we pattern-match them here.
1086  Int32Matcher match(node);
1087  if (match.HasValue()) {
1088  int32_t masked = (match.Value() & kMask32);
1089  if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1090  } else {
1091  node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1092  mcgraph()->Int32Constant(kMask32));
1093  }
1094  }
1095  return node;
1096 }
1097 
1098 Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1099  static const int64_t kMask64 = 0x3F;
1100  if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1101  // Shifts by constants are so common we pattern-match them here.
1102  Int64Matcher match(node);
1103  if (match.HasValue()) {
1104  int64_t masked = (match.Value() & kMask64);
1105  if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1106  } else {
1107  node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1108  mcgraph()->Int64Constant(kMask64));
1109  }
1110  }
1111  return node;
1112 }
1113 
1114 static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1115  size_t size_in_bytes) {
1116  switch (size_in_bytes) {
1117  case 4:
1118  case 16:
1119  return true;
1120  case 8:
1121  return m->Is64();
1122  default:
1123  break;
1124  }
1125  return false;
1126 }
1127 
1128 Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1129  Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1130  Node* result;
1131  Node* value = node;
1132  MachineOperatorBuilder* m = mcgraph()->machine();
1133  int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1134  int valueSizeInBits = 8 * valueSizeInBytes;
1135  bool isFloat = false;
1136 
1137  switch (wasmtype) {
1138  case wasm::kWasmF64:
1139  value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1140  isFloat = true;
1141  V8_FALLTHROUGH;
1142  case wasm::kWasmI64:
1143  result = mcgraph()->Int64Constant(0);
1144  break;
1145  case wasm::kWasmF32:
1146  value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1147  isFloat = true;
1148  V8_FALLTHROUGH;
1149  case wasm::kWasmI32:
1150  result = mcgraph()->Int32Constant(0);
1151  break;
1152  case wasm::kWasmS128:
1153  DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1154  break;
1155  default:
1156  UNREACHABLE();
1157  break;
1158  }
1159 
1160  if (mem_rep == MachineRepresentation::kWord8) {
1161  // No need to change endianness for byte size, return original node
1162  return node;
1163  }
1164  if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1165  // In case we store lower part of WasmI64 expression, we can truncate
1166  // upper 32bits
1167  value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1168  valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1169  valueSizeInBits = 8 * valueSizeInBytes;
1170  if (mem_rep == MachineRepresentation::kWord16) {
1171  value =
1172  graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1173  }
1174  } else if (wasmtype == wasm::kWasmI32 &&
1175  mem_rep == MachineRepresentation::kWord16) {
1176  value =
1177  graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1178  }
1179 
1180  int i;
1181  uint32_t shiftCount;
1182 
1183  if (ReverseBytesSupported(m, valueSizeInBytes)) {
1184  switch (valueSizeInBytes) {
1185  case 4:
1186  result = graph()->NewNode(m->Word32ReverseBytes(), value);
1187  break;
1188  case 8:
1189  result = graph()->NewNode(m->Word64ReverseBytes(), value);
1190  break;
1191  case 16: {
1192  Node* byte_reversed_lanes[4];
1193  for (int lane = 0; lane < 4; lane++) {
1194  byte_reversed_lanes[lane] = graph()->NewNode(
1195  m->Word32ReverseBytes(),
1196  graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1197  value));
1198  }
1199 
1200  // This is making a copy of the value.
1201  result =
1202  graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1203 
1204  for (int lane = 0; lane < 4; lane++) {
1205  result =
1206  graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1207  result, byte_reversed_lanes[lane]);
1208  }
1209 
1210  break;
1211  }
1212  default:
1213  UNREACHABLE();
1214  break;
1215  }
1216  } else {
1217  for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1218  i += 8, shiftCount -= 16) {
1219  Node* shiftLower;
1220  Node* shiftHigher;
1221  Node* lowerByte;
1222  Node* higherByte;
1223 
1224  DCHECK_LT(0, shiftCount);
1225  DCHECK_EQ(0, (shiftCount + 8) % 16);
1226 
1227  if (valueSizeInBits > 32) {
1228  shiftLower = graph()->NewNode(m->Word64Shl(), value,
1229  mcgraph()->Int64Constant(shiftCount));
1230  shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1231  mcgraph()->Int64Constant(shiftCount));
1232  lowerByte = graph()->NewNode(
1233  m->Word64And(), shiftLower,
1234  mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1235  << (valueSizeInBits - 8 - i)));
1236  higherByte = graph()->NewNode(
1237  m->Word64And(), shiftHigher,
1238  mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1239  result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1240  result = graph()->NewNode(m->Word64Or(), result, higherByte);
1241  } else {
1242  shiftLower = graph()->NewNode(m->Word32Shl(), value,
1243  mcgraph()->Int32Constant(shiftCount));
1244  shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1245  mcgraph()->Int32Constant(shiftCount));
1246  lowerByte = graph()->NewNode(
1247  m->Word32And(), shiftLower,
1248  mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1249  << (valueSizeInBits - 8 - i)));
1250  higherByte = graph()->NewNode(
1251  m->Word32And(), shiftHigher,
1252  mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1253  result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1254  result = graph()->NewNode(m->Word32Or(), result, higherByte);
1255  }
1256  }
1257  }
1258 
1259  if (isFloat) {
1260  switch (wasmtype) {
1261  case wasm::kWasmF64:
1262  result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1263  break;
1264  case wasm::kWasmF32:
1265  result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1266  break;
1267  default:
1268  UNREACHABLE();
1269  break;
1270  }
1271  }
1272 
1273  return result;
1274 }
1275 
1276 Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1277  MachineType memtype,
1278  wasm::ValueType wasmtype) {
1279  Node* result;
1280  Node* value = node;
1281  MachineOperatorBuilder* m = mcgraph()->machine();
1282  int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1283  int valueSizeInBits = 8 * valueSizeInBytes;
1284  bool isFloat = false;
1285 
1286  switch (memtype.representation()) {
1287  case MachineRepresentation::kFloat64:
1288  value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1289  isFloat = true;
1290  V8_FALLTHROUGH;
1291  case MachineRepresentation::kWord64:
1292  result = mcgraph()->Int64Constant(0);
1293  break;
1294  case MachineRepresentation::kFloat32:
1295  value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1296  isFloat = true;
1297  V8_FALLTHROUGH;
1298  case MachineRepresentation::kWord32:
1299  case MachineRepresentation::kWord16:
1300  result = mcgraph()->Int32Constant(0);
1301  break;
1302  case MachineRepresentation::kWord8:
1303  // No need to change endianness for byte size, return original node
1304  return node;
1305  break;
1306  case MachineRepresentation::kSimd128:
1307  DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1308  break;
1309  default:
1310  UNREACHABLE();
1311  break;
1312  }
1313 
1314  int i;
1315  uint32_t shiftCount;
1316 
1317  if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1318  switch (valueSizeInBytes) {
1319  case 2:
1320  result =
1321  graph()->NewNode(m->Word32ReverseBytes(),
1322  graph()->NewNode(m->Word32Shl(), value,
1323  mcgraph()->Int32Constant(16)));
1324  break;
1325  case 4:
1326  result = graph()->NewNode(m->Word32ReverseBytes(), value);
1327  break;
1328  case 8:
1329  result = graph()->NewNode(m->Word64ReverseBytes(), value);
1330  break;
1331  case 16: {
1332  Node* byte_reversed_lanes[4];
1333  for (int lane = 0; lane < 4; lane++) {
1334  byte_reversed_lanes[lane] = graph()->NewNode(
1335  m->Word32ReverseBytes(),
1336  graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1337  value));
1338  }
1339 
1340  // This is making a copy of the value.
1341  result =
1342  graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1343 
1344  for (int lane = 0; lane < 4; lane++) {
1345  result =
1346  graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1347  result, byte_reversed_lanes[lane]);
1348  }
1349 
1350  break;
1351  }
1352  default:
1353  UNREACHABLE();
1354  }
1355  } else {
1356  for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1357  i += 8, shiftCount -= 16) {
1358  Node* shiftLower;
1359  Node* shiftHigher;
1360  Node* lowerByte;
1361  Node* higherByte;
1362 
1363  DCHECK_LT(0, shiftCount);
1364  DCHECK_EQ(0, (shiftCount + 8) % 16);
1365 
1366  if (valueSizeInBits > 32) {
1367  shiftLower = graph()->NewNode(m->Word64Shl(), value,
1368  mcgraph()->Int64Constant(shiftCount));
1369  shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1370  mcgraph()->Int64Constant(shiftCount));
1371  lowerByte = graph()->NewNode(
1372  m->Word64And(), shiftLower,
1373  mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1374  << (valueSizeInBits - 8 - i)));
1375  higherByte = graph()->NewNode(
1376  m->Word64And(), shiftHigher,
1377  mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1378  result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1379  result = graph()->NewNode(m->Word64Or(), result, higherByte);
1380  } else {
1381  shiftLower = graph()->NewNode(m->Word32Shl(), value,
1382  mcgraph()->Int32Constant(shiftCount));
1383  shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1384  mcgraph()->Int32Constant(shiftCount));
1385  lowerByte = graph()->NewNode(
1386  m->Word32And(), shiftLower,
1387  mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1388  << (valueSizeInBits - 8 - i)));
1389  higherByte = graph()->NewNode(
1390  m->Word32And(), shiftHigher,
1391  mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1392  result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1393  result = graph()->NewNode(m->Word32Or(), result, higherByte);
1394  }
1395  }
1396  }
1397 
1398  if (isFloat) {
1399  switch (memtype.representation()) {
1400  case MachineRepresentation::kFloat64:
1401  result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1402  break;
1403  case MachineRepresentation::kFloat32:
1404  result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1405  break;
1406  default:
1407  UNREACHABLE();
1408  break;
1409  }
1410  }
1411 
1412  // We need to sign extend the value
1413  if (memtype.IsSigned()) {
1414  DCHECK(!isFloat);
1415  if (valueSizeInBits < 32) {
1416  Node* shiftBitCount;
1417  // Perform sign extension using following trick
1418  // result = (x << machine_width - type_width) >> (machine_width -
1419  // type_width)
1420  if (wasmtype == wasm::kWasmI64) {
1421  shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1422  result = graph()->NewNode(
1423  m->Word64Sar(),
1424  graph()->NewNode(m->Word64Shl(),
1425  graph()->NewNode(m->ChangeInt32ToInt64(), result),
1426  shiftBitCount),
1427  shiftBitCount);
1428  } else if (wasmtype == wasm::kWasmI32) {
1429  shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1430  result = graph()->NewNode(
1431  m->Word32Sar(),
1432  graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1433  shiftBitCount);
1434  }
1435  }
1436  }
1437 
1438  return result;
1439 }
1440 
1441 Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1442  Node* result = Unop(
1443  wasm::kExprF32ReinterpretI32,
1444  Binop(wasm::kExprI32Ior,
1445  Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1446  mcgraph()->Int32Constant(0x7FFFFFFF)),
1447  Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1448  mcgraph()->Int32Constant(0x80000000))));
1449 
1450  return result;
1451 }
1452 
1453 Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1454 #if WASM_64
1455  Node* result = Unop(
1456  wasm::kExprF64ReinterpretI64,
1457  Binop(wasm::kExprI64Ior,
1458  Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1459  mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1460  Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1461  mcgraph()->Int64Constant(0x8000000000000000))));
1462 
1463  return result;
1464 #else
1465  MachineOperatorBuilder* m = mcgraph()->machine();
1466 
1467  Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1468  Node* high_word_right =
1469  graph()->NewNode(m->Float64ExtractHighWord32(), right);
1470 
1471  Node* new_high_word = Binop(wasm::kExprI32Ior,
1472  Binop(wasm::kExprI32And, high_word_left,
1473  mcgraph()->Int32Constant(0x7FFFFFFF)),
1474  Binop(wasm::kExprI32And, high_word_right,
1475  mcgraph()->Int32Constant(0x80000000)));
1476 
1477  return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1478 #endif
1479 }
1480 
1481 namespace {
1482 
1483 MachineType IntConvertType(wasm::WasmOpcode opcode) {
1484  switch (opcode) {
1485  case wasm::kExprI32SConvertF32:
1486  case wasm::kExprI32SConvertF64:
1487  case wasm::kExprI32SConvertSatF32:
1488  case wasm::kExprI32SConvertSatF64:
1489  return MachineType::Int32();
1490  case wasm::kExprI32UConvertF32:
1491  case wasm::kExprI32UConvertF64:
1492  case wasm::kExprI32UConvertSatF32:
1493  case wasm::kExprI32UConvertSatF64:
1494  return MachineType::Uint32();
1495  case wasm::kExprI64SConvertF32:
1496  case wasm::kExprI64SConvertF64:
1497  case wasm::kExprI64SConvertSatF32:
1498  case wasm::kExprI64SConvertSatF64:
1499  return MachineType::Int64();
1500  case wasm::kExprI64UConvertF32:
1501  case wasm::kExprI64UConvertF64:
1502  case wasm::kExprI64UConvertSatF32:
1503  case wasm::kExprI64UConvertSatF64:
1504  return MachineType::Uint64();
1505  default:
1506  UNREACHABLE();
1507  }
1508 }
1509 
1510 MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1511  switch (opcode) {
1512  case wasm::kExprI32SConvertF32:
1513  case wasm::kExprI32UConvertF32:
1514  case wasm::kExprI32SConvertSatF32:
1515  case wasm::kExprI64SConvertF32:
1516  case wasm::kExprI64UConvertF32:
1517  case wasm::kExprI32UConvertSatF32:
1518  case wasm::kExprI64SConvertSatF32:
1519  case wasm::kExprI64UConvertSatF32:
1520  return MachineType::Float32();
1521  case wasm::kExprI32SConvertF64:
1522  case wasm::kExprI32UConvertF64:
1523  case wasm::kExprI64SConvertF64:
1524  case wasm::kExprI64UConvertF64:
1525  case wasm::kExprI32SConvertSatF64:
1526  case wasm::kExprI32UConvertSatF64:
1527  case wasm::kExprI64SConvertSatF64:
1528  case wasm::kExprI64UConvertSatF64:
1529  return MachineType::Float64();
1530  default:
1531  UNREACHABLE();
1532  }
1533 }
1534 
1535 const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1536  switch (opcode) {
1537  case wasm::kExprI32SConvertF32:
1538  case wasm::kExprI32SConvertSatF32:
1539  return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1540  case wasm::kExprI32UConvertF32:
1541  case wasm::kExprI32UConvertSatF32:
1542  return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1543  case wasm::kExprI32SConvertF64:
1544  case wasm::kExprI32SConvertSatF64:
1545  return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1546  case wasm::kExprI32UConvertF64:
1547  case wasm::kExprI32UConvertSatF64:
1548  return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1549  case wasm::kExprI64SConvertF32:
1550  case wasm::kExprI64SConvertSatF32:
1551  return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1552  case wasm::kExprI64UConvertF32:
1553  case wasm::kExprI64UConvertSatF32:
1554  return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1555  case wasm::kExprI64SConvertF64:
1556  case wasm::kExprI64SConvertSatF64:
1557  return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1558  case wasm::kExprI64UConvertF64:
1559  case wasm::kExprI64UConvertSatF64:
1560  return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1561  default:
1562  UNREACHABLE();
1563  }
1564 }
1565 
1566 wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1567  switch (opcode) {
1568  case wasm::kExprI32SConvertF32:
1569  case wasm::kExprI32SConvertSatF32:
1570  return wasm::kExprF32SConvertI32;
1571  case wasm::kExprI32UConvertF32:
1572  case wasm::kExprI32UConvertSatF32:
1573  return wasm::kExprF32UConvertI32;
1574  case wasm::kExprI32SConvertF64:
1575  case wasm::kExprI32SConvertSatF64:
1576  return wasm::kExprF64SConvertI32;
1577  case wasm::kExprI32UConvertF64:
1578  case wasm::kExprI32UConvertSatF64:
1579  return wasm::kExprF64UConvertI32;
1580  default:
1581  UNREACHABLE();
1582  }
1583 }
1584 
1585 bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1586  switch (opcode) {
1587  case wasm::kExprI32SConvertF32:
1588  case wasm::kExprI32UConvertF32:
1589  case wasm::kExprI32SConvertF64:
1590  case wasm::kExprI32UConvertF64:
1591  case wasm::kExprI64SConvertF32:
1592  case wasm::kExprI64UConvertF32:
1593  case wasm::kExprI64SConvertF64:
1594  case wasm::kExprI64UConvertF64:
1595  return true;
1596  case wasm::kExprI32SConvertSatF64:
1597  case wasm::kExprI32UConvertSatF64:
1598  case wasm::kExprI32SConvertSatF32:
1599  case wasm::kExprI32UConvertSatF32:
1600  case wasm::kExprI64SConvertSatF32:
1601  case wasm::kExprI64UConvertSatF32:
1602  case wasm::kExprI64SConvertSatF64:
1603  case wasm::kExprI64UConvertSatF64:
1604  return false;
1605  default:
1606  UNREACHABLE();
1607  }
1608 }
1609 
1610 Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1611  switch (ty.representation()) {
1612  case MachineRepresentation::kWord32:
1613  return builder->Int32Constant(0);
1614  case MachineRepresentation::kWord64:
1615  return builder->Int64Constant(0);
1616  case MachineRepresentation::kFloat32:
1617  return builder->Float32Constant(0.0);
1618  case MachineRepresentation::kFloat64:
1619  return builder->Float64Constant(0.0);
1620  default:
1621  UNREACHABLE();
1622  }
1623 }
1624 
1625 Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1626  switch (ty.semantic()) {
1627  case MachineSemantic::kInt32:
1628  return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1629  case MachineSemantic::kUint32:
1630  return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1631  case MachineSemantic::kInt64:
1632  return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1633  case MachineSemantic::kUint64:
1634  return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1635  default:
1636  UNREACHABLE();
1637  }
1638 }
1639 
1640 Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1641  switch (ty.semantic()) {
1642  case MachineSemantic::kInt32:
1643  return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1644  case MachineSemantic::kUint32:
1645  return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1646  case MachineSemantic::kInt64:
1647  return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1648  case MachineSemantic::kUint64:
1649  return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1650  default:
1651  UNREACHABLE();
1652  }
1653 }
1654 
1655 wasm::WasmOpcode TruncOp(const MachineType& ty) {
1656  switch (ty.representation()) {
1657  case MachineRepresentation::kFloat32:
1658  return wasm::kExprF32Trunc;
1659  case MachineRepresentation::kFloat64:
1660  return wasm::kExprF64Trunc;
1661  default:
1662  UNREACHABLE();
1663  }
1664 }
1665 
1666 wasm::WasmOpcode NeOp(const MachineType& ty) {
1667  switch (ty.representation()) {
1668  case MachineRepresentation::kFloat32:
1669  return wasm::kExprF32Ne;
1670  case MachineRepresentation::kFloat64:
1671  return wasm::kExprF64Ne;
1672  default:
1673  UNREACHABLE();
1674  }
1675 }
1676 
1677 wasm::WasmOpcode LtOp(const MachineType& ty) {
1678  switch (ty.representation()) {
1679  case MachineRepresentation::kFloat32:
1680  return wasm::kExprF32Lt;
1681  case MachineRepresentation::kFloat64:
1682  return wasm::kExprF64Lt;
1683  default:
1684  UNREACHABLE();
1685  }
1686 }
1687 
1688 Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1689  const MachineType& int_ty, const MachineType& float_ty,
1690  Node* trunc, Node* converted_value) {
1691  if (int_ty.representation() == MachineRepresentation::kWord32) {
1692  Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1693  return builder->Binop(NeOp(float_ty), trunc, check);
1694  }
1695  return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1696  trunc, builder->graph()->start());
1697 }
1698 
1699 Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1700  const MachineType& int_ty,
1701  const MachineType& float_ty, Node* trunc,
1702  Node* converted_value) {
1703  Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1704  converted_value);
1705  if (int_ty.representation() == MachineRepresentation::kWord64) {
1706  test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1707  }
1708  return test;
1709 }
1710 
1711 } // namespace
1712 
1713 Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1714  wasm::WasmCodePosition position,
1715  wasm::WasmOpcode opcode) {
1716  const MachineType int_ty = IntConvertType(opcode);
1717  const MachineType float_ty = FloatConvertType(opcode);
1718  const Operator* conv_op = ConvertOp(this, opcode);
1719  Node* trunc = nullptr;
1720  Node* converted_value = nullptr;
1721  const bool is_int32 =
1722  int_ty.representation() == MachineRepresentation::kWord32;
1723  if (is_int32) {
1724  trunc = Unop(TruncOp(float_ty), input);
1725  converted_value = graph()->NewNode(conv_op, trunc);
1726  } else {
1727  trunc = graph()->NewNode(conv_op, input);
1728  converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1729  trunc, graph()->start());
1730  }
1731  if (IsTrappingConvertOp(opcode)) {
1732  Node* test =
1733  ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1734  if (is_int32) {
1735  TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1736  } else {
1737  ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1738  }
1739  return converted_value;
1740  }
1741  Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1742  converted_value);
1743  Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1744  tl_d.Chain(Control());
1745  Node* nan_test = Binop(NeOp(float_ty), input, input);
1746  Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1747  nan_d.Nest(tl_d, true);
1748  Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1749  Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1750  sat_d.Nest(nan_d, false);
1751  Node* sat_val =
1752  sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1753  Node* nan_val =
1754  nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1755  return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1756 }
1757 
1758 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1759  MachineOperatorBuilder* m = mcgraph()->machine();
1760  // asm.js must use the wacky JS semantics.
1761  input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1762  return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1763 }
1764 
1765 Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1766  MachineOperatorBuilder* m = mcgraph()->machine();
1767  // asm.js must use the wacky JS semantics.
1768  return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1769 }
1770 
1771 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1772  MachineOperatorBuilder* m = mcgraph()->machine();
1773  // asm.js must use the wacky JS semantics.
1774  input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1775  return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1776 }
1777 
1778 Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1779  MachineOperatorBuilder* m = mcgraph()->machine();
1780  // asm.js must use the wacky JS semantics.
1781  return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1782 }
1783 
1784 Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1785  MachineRepresentation input_type) {
1786  Node* stack_slot_param =
1787  graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1788 
1789  const Operator* store_op = mcgraph()->machine()->Store(
1790  StoreRepresentation(input_type, kNoWriteBarrier));
1791  SetEffect(graph()->NewNode(store_op, stack_slot_param,
1792  mcgraph()->Int32Constant(0), input, Effect(),
1793  Control()));
1794 
1795  MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1796  MachineSignature sig(1, 1, sig_types);
1797 
1798  Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1799 
1800  return BuildCCall(&sig, function, stack_slot_param);
1801 }
1802 
1803 Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1804  return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1805  MachineRepresentation::kWord32);
1806 }
1807 
1808 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1809  return Unop(wasm::kExprI64UConvertI32,
1810  BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1811  MachineRepresentation::kWord64));
1812 }
1813 
1814 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1815  return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1816  MachineRepresentation::kWord32);
1817 }
1818 
1819 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1820  return Unop(
1821  wasm::kExprI64UConvertI32,
1822  BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1823  MachineRepresentation::kWord64));
1824 }
1825 
1826 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1827  MachineType type = MachineType::Float32();
1828  ExternalReference ref = ExternalReference::wasm_f32_trunc();
1829 
1830  return BuildCFuncInstruction(ref, type, input);
1831 }
1832 
1833 Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1834  MachineType type = MachineType::Float32();
1835  ExternalReference ref = ExternalReference::wasm_f32_floor();
1836  return BuildCFuncInstruction(ref, type, input);
1837 }
1838 
1839 Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1840  MachineType type = MachineType::Float32();
1841  ExternalReference ref = ExternalReference::wasm_f32_ceil();
1842  return BuildCFuncInstruction(ref, type, input);
1843 }
1844 
1845 Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1846  MachineType type = MachineType::Float32();
1847  ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1848  return BuildCFuncInstruction(ref, type, input);
1849 }
1850 
1851 Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1852  MachineType type = MachineType::Float64();
1853  ExternalReference ref = ExternalReference::wasm_f64_trunc();
1854  return BuildCFuncInstruction(ref, type, input);
1855 }
1856 
1857 Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1858  MachineType type = MachineType::Float64();
1859  ExternalReference ref = ExternalReference::wasm_f64_floor();
1860  return BuildCFuncInstruction(ref, type, input);
1861 }
1862 
1863 Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1864  MachineType type = MachineType::Float64();
1865  ExternalReference ref = ExternalReference::wasm_f64_ceil();
1866  return BuildCFuncInstruction(ref, type, input);
1867 }
1868 
1869 Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1870  MachineType type = MachineType::Float64();
1871  ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1872  return BuildCFuncInstruction(ref, type, input);
1873 }
1874 
1875 Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1876  MachineType type = MachineType::Float64();
1877  ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1878  return BuildCFuncInstruction(ref, type, input);
1879 }
1880 
1881 Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1882  MachineType type = MachineType::Float64();
1883  ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1884  return BuildCFuncInstruction(ref, type, input);
1885 }
1886 
1887 Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1888  MachineType type = MachineType::Float64();
1889  ExternalReference ref = ExternalReference::wasm_float64_pow();
1890  return BuildCFuncInstruction(ref, type, left, right);
1891 }
1892 
1893 Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1894  MachineType type = MachineType::Float64();
1895  ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1896  return BuildCFuncInstruction(ref, type, left, right);
1897 }
1898 
1899 Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1900  MachineType type, Node* input0,
1901  Node* input1) {
1902  // We do truncation by calling a C function which calculates the result.
1903  // The input is passed to the C function as a byte buffer holding the two
1904  // input doubles. We reserve this byte buffer as a stack slot, store the
1905  // parameters in this buffer slots, pass a pointer to the buffer to the C
1906  // function, and after calling the C function we collect the return value from
1907  // the buffer.
1908 
1909  const int type_size = ElementSizeInBytes(type.representation());
1910  const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1911  Node* stack_slot =
1912  graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1913 
1914  const Operator* store_op = mcgraph()->machine()->Store(
1915  StoreRepresentation(type.representation(), kNoWriteBarrier));
1916  SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1917  input0, Effect(), Control()));
1918 
1919  Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1920 
1921  if (input1 != nullptr) {
1922  SetEffect(graph()->NewNode(store_op, stack_slot,
1923  mcgraph()->Int32Constant(type_size), input1,
1924  Effect(), Control()));
1925  }
1926 
1927  MachineType sig_types[] = {MachineType::Pointer()};
1928  MachineSignature sig(0, 1, sig_types);
1929  BuildCCall(&sig, function, stack_slot);
1930 
1931  return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
1932  stack_slot, mcgraph()->Int32Constant(0),
1933  Effect(), Control()));
1934 }
1935 
1936 Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1937  // TODO(titzer/bradnelson): Check handlng of asm.js case.
1938  return BuildIntToFloatConversionInstruction(
1939  input, ExternalReference::wasm_int64_to_float32(),
1940  MachineRepresentation::kWord64, MachineType::Float32());
1941 }
1942 Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1943  // TODO(titzer/bradnelson): Check handlng of asm.js case.
1944  return BuildIntToFloatConversionInstruction(
1945  input, ExternalReference::wasm_uint64_to_float32(),
1946  MachineRepresentation::kWord64, MachineType::Float32());
1947 }
1948 Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1949  return BuildIntToFloatConversionInstruction(
1950  input, ExternalReference::wasm_int64_to_float64(),
1951  MachineRepresentation::kWord64, MachineType::Float64());
1952 }
1953 Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1954  return BuildIntToFloatConversionInstruction(
1955  input, ExternalReference::wasm_uint64_to_float64(),
1956  MachineRepresentation::kWord64, MachineType::Float64());
1957 }
1958 
1959 Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1960  Node* input, ExternalReference ref,
1961  MachineRepresentation parameter_representation,
1962  const MachineType result_type) {
1963  int stack_slot_size =
1964  std::max(ElementSizeInBytes(parameter_representation),
1965  ElementSizeInBytes(result_type.representation()));
1966  Node* stack_slot =
1967  graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1968  const Operator* store_op = mcgraph()->machine()->Store(
1969  StoreRepresentation(parameter_representation, kNoWriteBarrier));
1970  SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1971  input, Effect(), Control()));
1972  MachineType sig_types[] = {MachineType::Pointer()};
1973  MachineSignature sig(0, 1, sig_types);
1974  Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1975  BuildCCall(&sig, function, stack_slot);
1976  return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
1977  stack_slot, mcgraph()->Int32Constant(0),
1978  Effect(), Control()));
1979 }
1980 
1981 namespace {
1982 
1983 ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1984  wasm::WasmOpcode opcode) {
1985  switch (opcode) {
1986  case wasm::kExprI64SConvertF32:
1987  case wasm::kExprI64SConvertSatF32:
1988  return ExternalReference::wasm_float32_to_int64();
1989  case wasm::kExprI64UConvertF32:
1990  case wasm::kExprI64UConvertSatF32:
1991  return ExternalReference::wasm_float32_to_uint64();
1992  case wasm::kExprI64SConvertF64:
1993  case wasm::kExprI64SConvertSatF64:
1994  return ExternalReference::wasm_float64_to_int64();
1995  case wasm::kExprI64UConvertF64:
1996  case wasm::kExprI64UConvertSatF64:
1997  return ExternalReference::wasm_float64_to_uint64();
1998  default:
1999  UNREACHABLE();
2000  }
2001 }
2002 
2003 } // namespace
2004 
2005 Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
2006  wasm::WasmCodePosition position,
2007  wasm::WasmOpcode opcode) {
2008  const MachineType int_ty = IntConvertType(opcode);
2009  const MachineType float_ty = FloatConvertType(opcode);
2010  ExternalReference call_ref = convert_ccall_ref(this, opcode);
2011  int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
2012  ElementSizeInBytes(float_ty.representation()));
2013  Node* stack_slot =
2014  graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
2015  const Operator* store_op = mcgraph()->machine()->Store(
2016  StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
2017  SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
2018  Effect(), Control()));
2019  MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2020  MachineSignature sig(1, 1, sig_types);
2021  Node* function =
2022  graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
2023  Node* overflow = BuildCCall(&sig, function, stack_slot);
2024  if (IsTrappingConvertOp(opcode)) {
2025  ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
2026  return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
2027  stack_slot, Int32Constant(0), Effect(),
2028  Control()));
2029  }
2030  Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
2031  Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
2032  tl_d.Chain(Control());
2033  Node* nan_test = Binop(NeOp(float_ty), input, input);
2034  Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2035  nan_d.Nest(tl_d, true);
2036  Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2037  Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2038  sat_d.Nest(nan_d, false);
2039  Node* sat_val =
2040  sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2041  Node* load =
2042  SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
2043  Int32Constant(0), Effect(), Control()));
2044  Node* nan_val =
2045  nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2046  return tl_d.Phi(int_ty.representation(), nan_val, load);
2047 }
2048 
2049 Node* WasmGraphBuilder::MemoryGrow(Node* input) {
2050  needs_stack_check_ = true;
2051 
2052  WasmMemoryGrowDescriptor interface_descriptor;
2053  auto call_descriptor = Linkage::GetStubCallDescriptor(
2054  mcgraph()->zone(), // zone
2055  interface_descriptor, // descriptor
2056  interface_descriptor.GetStackParameterCount(), // stack parameter count
2057  CallDescriptor::kNoFlags, // flags
2058  Operator::kNoProperties, // properties
2059  StubCallMode::kCallWasmRuntimeStub); // stub call mode
2060  // A direct call to a wasm runtime stub defined in this module.
2061  // Just encode the stub index. This will be patched at relocation.
2062  Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2063  wasm::WasmCode::kWasmMemoryGrow, RelocInfo::WASM_STUB_CALL);
2064  return SetEffect(
2065  SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
2066  call_target, input, Effect(), Control())));
2067 }
2068 
2069 #ifdef DEBUG
2070 
2071 namespace {
2072 
2073 constexpr uint32_t kBytesPerExceptionValuesArrayElement = 2;
2074 
2075 size_t ComputeEncodedElementSize(wasm::ValueType type) {
2076  size_t byte_size =
2077  static_cast<size_t>(wasm::ValueTypes::ElementSizeInBytes(type));
2078  DCHECK_EQ(byte_size % kBytesPerExceptionValuesArrayElement, 0);
2079  DCHECK_LE(1, byte_size / kBytesPerExceptionValuesArrayElement);
2080  return byte_size / kBytesPerExceptionValuesArrayElement;
2081 }
2082 
2083 } // namespace
2084 
2085 #endif // DEBUG
2086 
2087 uint32_t WasmGraphBuilder::GetExceptionEncodedSize(
2088  const wasm::WasmException* exception) const {
2089  const wasm::WasmExceptionSig* sig = exception->sig;
2090  uint32_t encoded_size = 0;
2091  for (size_t i = 0; i < sig->parameter_count(); ++i) {
2092  switch (sig->GetParam(i)) {
2093  case wasm::kWasmI32:
2094  case wasm::kWasmF32:
2095  DCHECK_EQ(2, ComputeEncodedElementSize(sig->GetParam(i)));
2096  encoded_size += 2;
2097  break;
2098  case wasm::kWasmI64:
2099  case wasm::kWasmF64:
2100  DCHECK_EQ(4, ComputeEncodedElementSize(sig->GetParam(i)));
2101  encoded_size += 4;
2102  break;
2103  case wasm::kWasmS128:
2104  // TODO(mstarzinger): Implement and test this case.
2105  UNIMPLEMENTED();
2106  break;
2107  case wasm::kWasmAnyRef:
2108  encoded_size += 1;
2109  break;
2110  default:
2111  UNREACHABLE();
2112  }
2113  }
2114  return encoded_size;
2115 }
2116 
2117 Node* WasmGraphBuilder::Throw(uint32_t exception_index,
2118  const wasm::WasmException* exception,
2119  const Vector<Node*> values) {
2120  needs_stack_check_ = true;
2121  uint32_t encoded_size = GetExceptionEncodedSize(exception);
2122  Node* create_parameters[] = {
2123  LoadExceptionTagFromTable(exception_index),
2124  BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2125  Node* except_obj =
2126  BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2127  arraysize(create_parameters));
2128  Node* values_array =
2129  BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2130  uint32_t index = 0;
2131  const wasm::WasmExceptionSig* sig = exception->sig;
2132  MachineOperatorBuilder* m = mcgraph()->machine();
2133  for (size_t i = 0; i < sig->parameter_count(); ++i) {
2134  Node* value = values[i];
2135  switch (sig->GetParam(i)) {
2136  case wasm::kWasmF32:
2137  value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2138  V8_FALLTHROUGH;
2139  case wasm::kWasmI32:
2140  BuildEncodeException32BitValue(values_array, &index, value);
2141  break;
2142  case wasm::kWasmF64:
2143  value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2144  V8_FALLTHROUGH;
2145  case wasm::kWasmI64: {
2146  Node* upper32 = graph()->NewNode(
2147  m->TruncateInt64ToInt32(),
2148  Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2149  BuildEncodeException32BitValue(values_array, &index, upper32);
2150  Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2151  BuildEncodeException32BitValue(values_array, &index, lower32);
2152  break;
2153  }
2154  case wasm::kWasmAnyRef:
2155  STORE_FIXED_ARRAY_SLOT_ANY(values_array, index, value);
2156  ++index;
2157  break;
2158  default:
2159  UNREACHABLE();
2160  }
2161  }
2162  DCHECK_EQ(encoded_size, index);
2163  WasmThrowDescriptor interface_descriptor;
2164  auto call_descriptor = Linkage::GetStubCallDescriptor(
2165  mcgraph()->zone(), interface_descriptor,
2166  interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2167  Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2168  Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2169  wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2170  return SetEffect(SetControl(
2171  graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2172  except_obj, Effect(), Control())));
2173 }
2174 
2175 void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
2176  uint32_t* index,
2177  Node* value) {
2178  MachineOperatorBuilder* machine = mcgraph()->machine();
2179  Node* upper_halfword_as_smi = BuildChangeUint31ToSmi(
2180  graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16)));
2181  STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, upper_halfword_as_smi);
2182  ++(*index);
2183  Node* lower_halfword_as_smi = BuildChangeUint31ToSmi(
2184  graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu)));
2185  STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, lower_halfword_as_smi);
2186  ++(*index);
2187 }
2188 
2189 Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
2190  uint32_t* index) {
2191  MachineOperatorBuilder* machine = mcgraph()->machine();
2192  Node* upper =
2193  BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2194  (*index)++;
2195  upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2196  Node* lower =
2197  BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2198  (*index)++;
2199  Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2200  return value;
2201 }
2202 
2203 Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
2204  uint32_t* index) {
2205  Node* upper = Binop(wasm::kExprI64Shl,
2206  Unop(wasm::kExprI64UConvertI32,
2207  BuildDecodeException32BitValue(values_array, index)),
2208  Int64Constant(32));
2209  Node* lower = Unop(wasm::kExprI64UConvertI32,
2210  BuildDecodeException32BitValue(values_array, index));
2211  return Binop(wasm::kExprI64Ior, upper, lower);
2212 }
2213 
2214 Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
2215  needs_stack_check_ = true;
2216  WasmThrowDescriptor interface_descriptor;
2217  auto call_descriptor = Linkage::GetStubCallDescriptor(
2218  mcgraph()->zone(), interface_descriptor,
2219  interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2220  Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2221  Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2222  wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2223  return SetEffect(SetControl(
2224  graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2225  except_obj, Effect(), Control())));
2226 }
2227 
2228 Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
2229  Node* expected_tag) {
2230  MachineOperatorBuilder* machine = mcgraph()->machine();
2231  return graph()->NewNode(machine->WordEqual(), caught_tag, expected_tag);
2232 }
2233 
2234 Node* WasmGraphBuilder::LoadExceptionTagFromTable(uint32_t exception_index) {
2235  Node* exceptions_table =
2236  LOAD_INSTANCE_FIELD(ExceptionsTable, MachineType::TaggedPointer());
2237  Node* tag = LOAD_FIXED_ARRAY_SLOT_PTR(exceptions_table, exception_index);
2238  return tag;
2239 }
2240 
2241 Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
2242  needs_stack_check_ = true;
2243  return BuildCallToRuntime(Runtime::kWasmExceptionGetTag, &except_obj, 1);
2244 }
2245 
2246 Node** WasmGraphBuilder::GetExceptionValues(
2247  Node* except_obj, const wasm::WasmException* exception) {
2248  Node* values_array =
2249  BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2250  uint32_t index = 0;
2251  const wasm::WasmExceptionSig* sig = exception->sig;
2252  Node** values = Buffer(sig->parameter_count());
2253  for (size_t i = 0; i < sig->parameter_count(); ++i) {
2254  Node* value;
2255  switch (sig->GetParam(i)) {
2256  case wasm::kWasmI32:
2257  value = BuildDecodeException32BitValue(values_array, &index);
2258  break;
2259  case wasm::kWasmI64:
2260  value = BuildDecodeException64BitValue(values_array, &index);
2261  break;
2262  case wasm::kWasmF32: {
2263  value = Unop(wasm::kExprF32ReinterpretI32,
2264  BuildDecodeException32BitValue(values_array, &index));
2265  break;
2266  }
2267  case wasm::kWasmF64: {
2268  value = Unop(wasm::kExprF64ReinterpretI64,
2269  BuildDecodeException64BitValue(values_array, &index));
2270  break;
2271  }
2272  case wasm::kWasmAnyRef:
2273  value = LOAD_FIXED_ARRAY_SLOT_ANY(values_array, index);
2274  ++index;
2275  break;
2276  default:
2277  UNREACHABLE();
2278  }
2279  values[i] = value;
2280  }
2281  DCHECK_EQ(index, GetExceptionEncodedSize(exception));
2282  return values;
2283 }
2284 
2285 Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2286  wasm::WasmCodePosition position) {
2287  MachineOperatorBuilder* m = mcgraph()->machine();
2288  ZeroCheck32(wasm::kTrapDivByZero, right, position);
2289  Node* before = Control();
2290  Node* denom_is_m1;
2291  Node* denom_is_not_m1;
2292  BranchExpectFalse(
2293  graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2294  &denom_is_m1, &denom_is_not_m1);
2295  SetControl(denom_is_m1);
2296  TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2297  if (Control() != denom_is_m1) {
2298  SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2299  Control()));
2300  } else {
2301  SetControl(before);
2302  }
2303  return graph()->NewNode(m->Int32Div(), left, right, Control());
2304 }
2305 
2306 Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2307  wasm::WasmCodePosition position) {
2308  MachineOperatorBuilder* m = mcgraph()->machine();
2309 
2310  ZeroCheck32(wasm::kTrapRemByZero, right, position);
2311 
2312  Diamond d(
2313  graph(), mcgraph()->common(),
2314  graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2315  BranchHint::kFalse);
2316  d.Chain(Control());
2317 
2318  return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2319  graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2320 }
2321 
2322 Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2323  wasm::WasmCodePosition position) {
2324  MachineOperatorBuilder* m = mcgraph()->machine();
2325  return graph()->NewNode(m->Uint32Div(), left, right,
2326  ZeroCheck32(wasm::kTrapDivByZero, right, position));
2327 }
2328 
2329 Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2330  wasm::WasmCodePosition position) {
2331  MachineOperatorBuilder* m = mcgraph()->machine();
2332  return graph()->NewNode(m->Uint32Mod(), left, right,
2333  ZeroCheck32(wasm::kTrapRemByZero, right, position));
2334 }
2335 
2336 Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2337  MachineOperatorBuilder* m = mcgraph()->machine();
2338 
2339  Int32Matcher mr(right);
2340  if (mr.HasValue()) {
2341  if (mr.Value() == 0) {
2342  return mcgraph()->Int32Constant(0);
2343  } else if (mr.Value() == -1) {
2344  // The result is the negation of the left input.
2345  return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2346  }
2347  return graph()->NewNode(m->Int32Div(), left, right, Control());
2348  }
2349 
2350  // asm.js semantics return 0 on divide or mod by zero.
2351  if (m->Int32DivIsSafe()) {
2352  // The hardware instruction does the right thing (e.g. arm).
2353  return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2354  }
2355 
2356  // Check denominator for zero.
2357  Diamond z(
2358  graph(), mcgraph()->common(),
2359  graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2360  BranchHint::kFalse);
2361 
2362  // Check numerator for -1. (avoid minint / -1 case).
2363  Diamond n(
2364  graph(), mcgraph()->common(),
2365  graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2366  BranchHint::kFalse);
2367 
2368  Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2369  Node* neg =
2370  graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2371 
2372  return n.Phi(
2373  MachineRepresentation::kWord32, neg,
2374  z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2375 }
2376 
2377 Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2378  CommonOperatorBuilder* c = mcgraph()->common();
2379  MachineOperatorBuilder* m = mcgraph()->machine();
2380  Node* const zero = mcgraph()->Int32Constant(0);
2381 
2382  Int32Matcher mr(right);
2383  if (mr.HasValue()) {
2384  if (mr.Value() == 0 || mr.Value() == -1) {
2385  return zero;
2386  }
2387  return graph()->NewNode(m->Int32Mod(), left, right, Control());
2388  }
2389 
2390  // General case for signed integer modulus, with optimization for (unknown)
2391  // power of 2 right hand side.
2392  //
2393  // if 0 < right then
2394  // msk = right - 1
2395  // if right & msk != 0 then
2396  // left % right
2397  // else
2398  // if left < 0 then
2399  // -(-left & msk)
2400  // else
2401  // left & msk
2402  // else
2403  // if right < -1 then
2404  // left % right
2405  // else
2406  // zero
2407  //
2408  // Note: We do not use the Diamond helper class here, because it really hurts
2409  // readability with nested diamonds.
2410  Node* const minus_one = mcgraph()->Int32Constant(-1);
2411 
2412  const Operator* const merge_op = c->Merge(2);
2413  const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2414 
2415  Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2416  Node* branch0 =
2417  graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2418 
2419  Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2420  Node* true0;
2421  {
2422  Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2423 
2424  Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2425  Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2426 
2427  Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2428  Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2429 
2430  Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2431  Node* false1;
2432  {
2433  Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2434  Node* branch2 =
2435  graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2436 
2437  Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2438  Node* true2 = graph()->NewNode(
2439  m->Int32Sub(), zero,
2440  graph()->NewNode(m->Word32And(),
2441  graph()->NewNode(m->Int32Sub(), zero, left), msk));
2442 
2443  Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2444  Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2445 
2446  if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2447  false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2448  }
2449 
2450  if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2451  true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2452  }
2453 
2454  Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2455  Node* false0;
2456  {
2457  Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2458  Node* branch1 =
2459  graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2460 
2461  Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2462  Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2463 
2464  Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2465  Node* false1 = zero;
2466 
2467  if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2468  false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2469  }
2470 
2471  Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2472  return graph()->NewNode(phi_op, true0, false0, merge0);
2473 }
2474 
2475 Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2476  MachineOperatorBuilder* m = mcgraph()->machine();
2477  // asm.js semantics return 0 on divide or mod by zero.
2478  if (m->Uint32DivIsSafe()) {
2479  // The hardware instruction does the right thing (e.g. arm).
2480  return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2481  }
2482 
2483  // Explicit check for x % 0.
2484  Diamond z(
2485  graph(), mcgraph()->common(),
2486  graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2487  BranchHint::kFalse);
2488 
2489  return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2490  graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2491  z.if_false));
2492 }
2493 
2494 Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2495  MachineOperatorBuilder* m = mcgraph()->machine();
2496  // asm.js semantics return 0 on divide or mod by zero.
2497  // Explicit check for x % 0.
2498  Diamond z(
2499  graph(), mcgraph()->common(),
2500  graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2501  BranchHint::kFalse);
2502 
2503  Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2504  z.if_false);
2505  return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2506  rem);
2507 }
2508 
2509 Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2510  wasm::WasmCodePosition position) {
2511  if (mcgraph()->machine()->Is32()) {
2512  return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2513  MachineType::Int64(), wasm::kTrapDivByZero, position);
2514  }
2515  ZeroCheck64(wasm::kTrapDivByZero, right, position);
2516  Node* before = Control();
2517  Node* denom_is_m1;
2518  Node* denom_is_not_m1;
2519  BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2520  mcgraph()->Int64Constant(-1)),
2521  &denom_is_m1, &denom_is_not_m1);
2522  SetControl(denom_is_m1);
2523  TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2524  std::numeric_limits<int64_t>::min(), position);
2525  if (Control() != denom_is_m1) {
2526  SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2527  Control()));
2528  } else {
2529  SetControl(before);
2530  }
2531  return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2532  Control());
2533 }
2534 
2535 Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2536  wasm::WasmCodePosition position) {
2537  if (mcgraph()->machine()->Is32()) {
2538  return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2539  MachineType::Int64(), wasm::kTrapRemByZero, position);
2540  }
2541  ZeroCheck64(wasm::kTrapRemByZero, right, position);
2542  Diamond d(mcgraph()->graph(), mcgraph()->common(),
2543  graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2544  mcgraph()->Int64Constant(-1)));
2545 
2546  d.Chain(Control());
2547 
2548  Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2549  d.if_false);
2550 
2551  return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2552  rem);
2553 }
2554 
2555 Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2556  wasm::WasmCodePosition position) {
2557  if (mcgraph()->machine()->Is32()) {
2558  return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2559  MachineType::Int64(), wasm::kTrapDivByZero, position);
2560  }
2561  return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2562  ZeroCheck64(wasm::kTrapDivByZero, right, position));
2563 }
2564 Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2565  wasm::WasmCodePosition position) {
2566  if (mcgraph()->machine()->Is32()) {
2567  return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2568  MachineType::Int64(), wasm::kTrapRemByZero, position);
2569  }
2570  return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2571  ZeroCheck64(wasm::kTrapRemByZero, right, position));
2572 }
2573 
2574 Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2575  ExternalReference ref,
2576  MachineType result_type,
2577  wasm::TrapReason trap_zero,
2578  wasm::WasmCodePosition position) {
2579  Node* stack_slot =
2580  graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2581 
2582  const Operator* store_op = mcgraph()->machine()->Store(
2583  StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2584  SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2585  left, Effect(), Control()));
2586  SetEffect(graph()->NewNode(store_op, stack_slot,
2587  mcgraph()->Int32Constant(sizeof(double)), right,
2588  Effect(), Control()));
2589 
2590  MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2591  MachineSignature sig(1, 1, sig_types);
2592 
2593  Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2594  Node* call = BuildCCall(&sig, function, stack_slot);
2595 
2596  ZeroCheck32(trap_zero, call, position);
2597  TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2598  return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
2599  stack_slot, mcgraph()->Int32Constant(0),
2600  Effect(), Control()));
2601 }
2602 
2603 template <typename... Args>
2604 Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2605  Args... args) {
2606  DCHECK_LE(sig->return_count(), 1);
2607  DCHECK_EQ(sizeof...(args), sig->parameter_count());
2608  Node* const call_args[] = {function, args..., Effect(), Control()};
2609 
2610  auto call_descriptor =
2611  Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2612 
2613  const Operator* op = mcgraph()->common()->Call(call_descriptor);
2614  return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
2615 }
2616 
2617 Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2618  Node*** rets,
2619  wasm::WasmCodePosition position,
2620  Node* instance_node,
2621  UseRetpoline use_retpoline) {
2622  if (instance_node == nullptr) {
2623  DCHECK_NOT_NULL(instance_node_);
2624  instance_node = instance_node_.get();
2625  }
2626  needs_stack_check_ = true;
2627  const size_t params = sig->parameter_count();
2628  const size_t extra = 3; // instance_node, effect, and control.
2629  const size_t count = 1 + params + extra;
2630 
2631  // Reallocate the buffer to make space for extra inputs.
2632  args = Realloc(args, 1 + params, count);
2633 
2634  // Make room for the instance_node parameter at index 1, just after code.
2635  memmove(&args[2], &args[1], params * sizeof(Node*));
2636  args[1] = instance_node;
2637 
2638  // Add effect and control inputs.
2639  args[params + 2] = Effect();
2640  args[params + 3] = Control();
2641 
2642  auto call_descriptor =
2643  GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2644  const Operator* op = mcgraph()->common()->Call(call_descriptor);
2645  Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
2646  DCHECK(position == wasm::kNoCodePosition || position > 0);
2647  if (position > 0) SetSourcePosition(call, position);
2648 
2649  size_t ret_count = sig->return_count();
2650  if (ret_count == 0) return call; // No return value.
2651 
2652  *rets = Buffer(ret_count);
2653  if (ret_count == 1) {
2654  // Only a single return value.
2655  (*rets)[0] = call;
2656  } else {
2657  // Create projections for all return values.
2658  for (size_t i = 0; i < ret_count; i++) {
2659  (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2660  graph()->start());
2661  }
2662  }
2663  return call;
2664 }
2665 
2666 Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2667  Node*** rets,
2668  wasm::WasmCodePosition position,
2669  int func_index) {
2670  // Load the imported function refs array from the instance.
2671  Node* imported_function_refs =
2672  LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2673  Node* ref_node =
2674  LOAD_FIXED_ARRAY_SLOT_PTR(imported_function_refs, func_index);
2675 
2676  // Load the target from the imported_targets array at a known offset.
2677  Node* imported_targets =
2678  LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2679  Node* target_node = SetEffect(graph()->NewNode(
2680  mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2681  mcgraph()->Int32Constant(func_index * kPointerSize), Effect(),
2682  Control()));
2683  args[0] = target_node;
2684  return BuildWasmCall(sig, args, rets, position, ref_node,
2685  untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2686 }
2687 
2688 Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2689  Node*** rets,
2690  wasm::WasmCodePosition position,
2691  Node* func_index) {
2692  // Load the imported function refs array from the instance.
2693  Node* imported_function_refs =
2694  LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2695  // Access fixed array at {header_size - tag + func_index * kPointerSize}.
2696  Node* imported_instances_data = graph()->NewNode(
2697  mcgraph()->machine()->IntAdd(), imported_function_refs,
2698  mcgraph()->IntPtrConstant(
2699  wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
2700  Node* func_index_times_pointersize = graph()->NewNode(
2701  mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
2702  mcgraph()->Int32Constant(kPointerSize));
2703  Node* ref_node = SetEffect(
2704  graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
2705  imported_instances_data, func_index_times_pointersize,
2706  Effect(), Control()));
2707 
2708  // Load the target from the imported_targets array at the offset of
2709  // {func_index}.
2710  Node* imported_targets =
2711  LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2712  Node* target_node = SetEffect(graph()->NewNode(
2713  mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2714  func_index_times_pointersize, Effect(), Control()));
2715  args[0] = target_node;
2716  return BuildWasmCall(sig, args, rets, position, ref_node,
2717  untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2718 }
2719 
2720 Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2721  wasm::WasmCodePosition position) {
2722  DCHECK_NULL(args[0]);
2723  wasm::FunctionSig* sig = env_->module->functions[index].sig;
2724 
2725  if (env_ && index < env_->module->num_imported_functions) {
2726  // Call to an imported function.
2727  return BuildImportCall(sig, args, rets, position, index);
2728  }
2729 
2730  // A direct call to a wasm function defined in this module.
2731  // Just encode the function index. This will be patched at instantiation.
2732  Address code = static_cast<Address>(index);
2733  args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2734 
2735  return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2736 }
2737 
2738 Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2739  Node*** rets,
2740  wasm::WasmCodePosition position) {
2741  DCHECK_NOT_NULL(args[0]);
2742  DCHECK_NOT_NULL(env_);
2743 
2744  // Assume only one table for now.
2745  wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2746 
2747  Node* ift_size =
2748  LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2749 
2750  MachineOperatorBuilder* machine = mcgraph()->machine();
2751  Node* key = args[0];
2752 
2753  // Bounds check against the table size.
2754  Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2755  TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2756 
2757  // Mask the key to prevent SSCA.
2758  if (untrusted_code_mitigations_) {
2759  // mask = ((key - size) & ~key) >> 31
2760  Node* neg_key =
2761  graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2762  Node* masked_diff = graph()->NewNode(
2763  machine->Word32And(),
2764  graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2765  Node* mask =
2766  graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2767  key = graph()->NewNode(machine->Word32And(), key, mask);
2768  }
2769 
2770  // Load signature from the table and check.
2771  Node* ift_sig_ids =
2772  LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2773 
2774  int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2775  Node* scaled_key = Uint32ToUintptr(
2776  graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
2777 
2778  Node* loaded_sig =
2779  SetEffect(graph()->NewNode(machine->Load(MachineType::Int32()),
2780  ift_sig_ids, scaled_key, Effect(), Control()));
2781  Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2782  Int32Constant(expected_sig_id));
2783 
2784  TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2785 
2786  Node* ift_targets =
2787  LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2788  Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableRefs,
2789  MachineType::TaggedPointer());
2790 
2791  scaled_key = graph()->NewNode(machine->Word32Shl(), key,
2792  Int32Constant(kPointerSizeLog2));
2793 
2794  Node* target =
2795  SetEffect(graph()->NewNode(machine->Load(MachineType::Pointer()),
2796  ift_targets, scaled_key, Effect(), Control()));
2797 
2798  Node* target_instance = SetEffect(graph()->NewNode(
2799  machine->Load(MachineType::TaggedPointer()),
2800  graph()->NewNode(machine->IntAdd(), ift_instances, scaled_key),
2801  Int32Constant(wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)),
2802  Effect(), Control()));
2803 
2804  args[0] = target;
2805 
2806  return BuildWasmCall(sig, args, rets, position, target_instance,
2807  untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2808 }
2809 
2810 Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2811  // Implement Rol by Ror since TurboFan does not have Rol opcode.
2812  // TODO(weiliang): support Word32Rol opcode in TurboFan.
2813  Int32Matcher m(right);
2814  if (m.HasValue()) {
2815  return Binop(wasm::kExprI32Ror, left,
2816  mcgraph()->Int32Constant(32 - m.Value()));
2817  } else {
2818  return Binop(wasm::kExprI32Ror, left,
2819  Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2820  }
2821 }
2822 
2823 Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2824  // Implement Rol by Ror since TurboFan does not have Rol opcode.
2825  // TODO(weiliang): support Word64Rol opcode in TurboFan.
2826  Int64Matcher m(right);
2827  if (m.HasValue()) {
2828  return Binop(wasm::kExprI64Ror, left,
2829  mcgraph()->Int64Constant(64 - m.Value()));
2830  } else {
2831  return Binop(wasm::kExprI64Ror, left,
2832  Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2833  }
2834 }
2835 
2836 Node* WasmGraphBuilder::Invert(Node* node) {
2837  return Unop(wasm::kExprI32Eqz, node);
2838 }
2839 
2840 bool CanCover(Node* value, IrOpcode::Value opcode) {
2841  if (value->opcode() != opcode) return false;
2842  bool first = true;
2843  for (Edge const edge : value->use_edges()) {
2844  if (NodeProperties::IsControlEdge(edge)) continue;
2845  if (NodeProperties::IsEffectEdge(edge)) continue;
2846  DCHECK(NodeProperties::IsValueEdge(edge));
2847  if (!first) return false;
2848  first = false;
2849  }
2850  return true;
2851 }
2852 
2853 Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
2854  if (mcgraph()->machine()->Is64()) {
2855  value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2856  }
2857  return value;
2858 }
2859 
2860 Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2861  value = BuildChangeInt32ToIntPtr(value);
2862  return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2863  BuildSmiShiftBitsConstant());
2864 }
2865 
2866 Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2867  return graph()->NewNode(mcgraph()->machine()->WordShl(),
2868  Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2869 }
2870 
2871 Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2872  return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2873 }
2874 
2875 Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2876  value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2877  BuildSmiShiftBitsConstant());
2878  if (mcgraph()->machine()->Is64()) {
2879  value =
2880  graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2881  }
2882  return value;
2883 }
2884 
2885 void WasmGraphBuilder::InitInstanceCache(
2886  WasmInstanceCacheNodes* instance_cache) {
2887  DCHECK_NOT_NULL(instance_node_);
2888 
2889  // Load the memory start.
2890  instance_cache->mem_start = SetEffect(graph()->NewNode(
2891  mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2892  mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2893  Effect(), Control()));
2894 
2895  // Load the memory size.
2896  instance_cache->mem_size = SetEffect(graph()->NewNode(
2897  mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2898  mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2899  Effect(), Control()));
2900 
2901  if (untrusted_code_mitigations_) {
2902  // Load the memory mask.
2903  instance_cache->mem_mask =
2904  LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr());
2905  } else {
2906  // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2907  instance_cache->mem_mask = nullptr;
2908  }
2909 }
2910 
2911 void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2912  WasmInstanceCacheNodes* instance_cache, Node* control) {
2913 #define INTRODUCE_PHI(field, rep) \
2914  instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2915  instance_cache->field, control);
2916 
2917  INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2918  INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
2919  if (untrusted_code_mitigations_) {
2920  INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation());
2921  }
2922 
2923 #undef INTRODUCE_PHI
2924 }
2925 
2926 void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2927  WasmInstanceCacheNodes* from,
2928  Node* merge) {
2929 #define INTRODUCE_PHI(field, rep) \
2930  if (to->field != from->field) { \
2931  Node* vals[] = {to->field, from->field, merge}; \
2932  to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2933  }
2934 
2935  INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2936  INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2937  if (untrusted_code_mitigations_) {
2938  INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2939  }
2940 
2941 #undef INTRODUCE_PHI
2942 }
2943 
2944 void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2945  WasmInstanceCacheNodes* from,
2946  Node* merge) {
2947  to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2948  merge, to->mem_size, from->mem_size);
2949  to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2950  merge, to->mem_start, from->mem_start);
2951  if (untrusted_code_mitigations_) {
2952  to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2953  merge, to->mem_mask, from->mem_mask);
2954  }
2955 }
2956 
2957 Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
2958  Node* merge, Node* tnode,
2959  Node* fnode) {
2960  if (IsPhiWithMerge(tnode, merge)) {
2961  AppendToPhi(tnode, fnode);
2962  } else if (tnode != fnode) {
2963  uint32_t count = merge->InputCount();
2964  // + 1 for the merge node.
2965  Node** vals = Buffer(count + 1);
2966  for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
2967  vals[count - 1] = fnode;
2968  vals[count] = merge;
2969  return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
2970  vals);
2971  }
2972  return tnode;
2973 }
2974 
2975 Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
2976  Node* fnode) {
2977  if (IsPhiWithMerge(tnode, merge)) {
2978  AppendToPhi(tnode, fnode);
2979  } else if (tnode != fnode) {
2980  uint32_t count = merge->InputCount();
2981  Node** effects = Buffer(count);
2982  for (uint32_t j = 0; j < count - 1; j++) {
2983  effects[j] = tnode;
2984  }
2985  effects[count - 1] = fnode;
2986  tnode = EffectPhi(count, effects, merge);
2987  }
2988  return tnode;
2989 }
2990 
2991 void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
2992  const wasm::WasmGlobal& global,
2993  Node** base_node,
2994  Node** offset_node) {
2995  DCHECK_NOT_NULL(instance_node_);
2996  if (global.mutability && global.imported) {
2997  if (imported_mutable_globals_ == nullptr) {
2998  // Load imported_mutable_globals_ from the instance object at runtime.
2999  imported_mutable_globals_ = graph()->NewNode(
3000  mcgraph()->machine()->Load(MachineType::UintPtr()),
3001  instance_node_.get(),
3002  mcgraph()->Int32Constant(
3003  WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
3004  graph()->start(), graph()->start());
3005  }
3006  *base_node = SetEffect(graph()->NewNode(
3007  mcgraph()->machine()->Load(MachineType::UintPtr()),
3008  imported_mutable_globals_.get(),
3009  mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
3010  Control()));
3011  *offset_node = mcgraph()->Int32Constant(0);
3012  } else {
3013  if (globals_start_ == nullptr) {
3014  // Load globals_start from the instance object at runtime.
3015  // TODO(wasm): we currently generate only one load of the {globals_start}
3016  // start per graph, which means it can be placed anywhere by the
3017  // scheduler. This is legal because the globals_start should never change.
3018  // However, in some cases (e.g. if the instance object is already in a
3019  // register), it is slightly more efficient to reload this value from the
3020  // instance object. Since this depends on register allocation, it is not
3021  // possible to express in the graph, and would essentially constitute a
3022  // "mem2reg" optimization in TurboFan.
3023  globals_start_ = graph()->NewNode(
3024  mcgraph()->machine()->Load(MachineType::UintPtr()),
3025  instance_node_.get(),
3026  mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
3027  graph()->start(), graph()->start());
3028  }
3029  *base_node = globals_start_.get();
3030  *offset_node = mcgraph()->Int32Constant(global.offset);
3031 
3032  if (mem_type == MachineType::Simd128() && global.offset != 0) {
3033  // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
3034  *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
3035  *offset_node);
3036  *offset_node = mcgraph()->Int32Constant(0);
3037  }
3038  }
3039 }
3040 
3041 Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
3042  DCHECK_NOT_NULL(instance_cache_);
3043  Node* mem_start = instance_cache_->mem_start;
3044  DCHECK_NOT_NULL(mem_start);
3045  if (offset == 0) return mem_start;
3046  return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
3047  mcgraph()->IntPtrConstant(offset));
3048 }
3049 
3050 Node* WasmGraphBuilder::CurrentMemoryPages() {
3051  // CurrentMemoryPages can not be called from asm.js.
3052  DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
3053  DCHECK_NOT_NULL(instance_cache_);
3054  Node* mem_size = instance_cache_->mem_size;
3055  DCHECK_NOT_NULL(mem_size);
3056  Node* result =
3057  graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
3058  mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
3059  if (mcgraph()->machine()->Is64()) {
3060  result =
3061  graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
3062  }
3063  return result;
3064 }
3065 
3066 Node* WasmGraphBuilder::BuildLoadBuiltinFromInstance(int builtin_index) {
3067  DCHECK(Builtins::IsBuiltinId(builtin_index));
3068  Node* isolate_root = LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
3069  return LOAD_TAGGED_POINTER(isolate_root,
3070  IsolateData::builtin_slot_offset(builtin_index));
3071 }
3072 
3073 // Only call this function for code which is not reused across instantiations,
3074 // as we do not patch the embedded js_context.
3075 Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f,
3076  Node* js_context,
3077  Node** parameters,
3078  int parameter_count) {
3079  const Runtime::Function* fun = Runtime::FunctionForId(f);
3080  auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3081  mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
3082  CallDescriptor::kNoFlags);
3083  // The CEntryStub is loaded from the instance_node so that generated code is
3084  // Isolate independent. At the moment this is only done for CEntryStub(1).
3085  DCHECK_EQ(1, fun->result_size);
3086  Node* centry_stub =
3087  LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
3088  // At the moment we only allow 4 parameters. If more parameters are needed,
3089  // increase this constant accordingly.
3090  static const int kMaxParams = 4;
3091  DCHECK_GE(kMaxParams, parameter_count);
3092  Node* inputs[kMaxParams + 6];
3093  int count = 0;
3094  inputs[count++] = centry_stub;
3095  for (int i = 0; i < parameter_count; i++) {
3096  inputs[count++] = parameters[i];
3097  }
3098  inputs[count++] =
3099  mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
3100  inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity
3101  inputs[count++] = js_context; // js_context
3102  inputs[count++] = Effect();
3103  inputs[count++] = Control();
3104 
3105  return SetEffect(mcgraph()->graph()->NewNode(
3106  mcgraph()->common()->Call(call_descriptor), count, inputs));
3107 }
3108 
3109 Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
3110  Node** parameters,
3111  int parameter_count) {
3112  return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3113  parameter_count);
3114 }
3115 
3116 Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3117  MachineType mem_type =
3118  wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3119  Node* base = nullptr;
3120  Node* offset = nullptr;
3121  GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3122  &offset);
3123  Node* load = SetEffect(graph()->NewNode(mcgraph()->machine()->Load(mem_type),
3124  base, offset, Effect(), Control()));
3125 #if defined(V8_TARGET_BIG_ENDIAN)
3126  load = BuildChangeEndiannessLoad(load, mem_type,
3127  env_->module->globals[index].type);
3128 #endif
3129  return load;
3130 }
3131 
3132 Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3133  MachineType mem_type =
3134  wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3135  Node* base = nullptr;
3136  Node* offset = nullptr;
3137  GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3138  &offset);
3139  const Operator* op = mcgraph()->machine()->Store(
3140  StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3141 #if defined(V8_TARGET_BIG_ENDIAN)
3142  val = BuildChangeEndiannessStore(val, mem_type.representation(),
3143  env_->module->globals[index].type);
3144 #endif
3145  return SetEffect(
3146  graph()->NewNode(op, base, offset, val, Effect(), Control()));
3147 }
3148 
3149 Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3150  uint8_t access_size, Node* index, uint32_t offset,
3151  wasm::WasmCodePosition position) {
3152  // Atomic operations access the memory, need to be bound checked till
3153  // TrapHandlers are enabled on atomic operations
3154  index =
3155  BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
3156  Node* effective_address =
3157  graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(offset),
3158  Uint32ToUintptr(index));
3159  // Unlike regular memory accesses, unaligned memory accesses for atomic
3160  // operations should trap
3161  // Access sizes are in powers of two, calculate mod without using division
3162  Node* cond =
3163  graph()->NewNode(mcgraph()->machine()->WordAnd(), effective_address,
3164  IntPtrConstant(access_size - 1));
3165  TrapIfFalse(wasm::kTrapUnalignedAccess,
3166  graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
3167  mcgraph()->Int32Constant(0)),
3168  position);
3169  return index;
3170 }
3171 
3172 Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3173  uint32_t offset,
3174  wasm::WasmCodePosition position,
3175  EnforceBoundsCheck enforce_check) {
3176  DCHECK_LE(1, access_size);
3177  index = Uint32ToUintptr(index);
3178  if (FLAG_wasm_no_bounds_checks) return index;
3179 
3180  if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3181  return index;
3182  }
3183 
3184  const bool statically_oob = access_size > env_->max_memory_size ||
3185  offset > env_->max_memory_size - access_size;
3186  if (statically_oob) {
3187  // The access will be out of bounds, even for the largest memory.
3188  TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3189  return mcgraph()->IntPtrConstant(0);
3190  }
3191  uint64_t end_offset = uint64_t{offset} + access_size - 1u;
3192  Node* end_offset_node = IntPtrConstant(end_offset);
3193 
3194  // The accessed memory is [index + offset, index + end_offset].
3195  // Check that the last read byte (at {index + end_offset}) is in bounds.
3196  // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3197  // compute {effective_size} as {mem_size - end_offset)}.
3198  // {effective_size} is >= 1 if condition 1) holds.
3199  // 2) Check that {index + end_offset < mem_size} by
3200  // - computing {effective_size} as {mem_size - end_offset} and
3201  // - checking that {index < effective_size}.
3202 
3203  auto m = mcgraph()->machine();
3204  Node* mem_size = instance_cache_->mem_size;
3205  if (end_offset >= env_->min_memory_size) {
3206  // The end offset is larger than the smallest memory.
3207  // Dynamically check the end offset against the dynamic memory size.
3208  Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
3209  TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3210  } else {
3211  // The end offset is smaller than the smallest memory, so only one check is
3212  // required. Check to see if the index is also a constant.
3213  UintPtrMatcher match(index);
3214  if (match.HasValue()) {
3215  uintptr_t index_val = match.Value();
3216  if (index_val < env_->min_memory_size - end_offset) {
3217  // The input index is a constant and everything is statically within
3218  // bounds of the smallest possible memory.
3219  return index;
3220  }
3221  }
3222  }
3223 
3224  // This produces a positive number, since {end_offset < min_size <= mem_size}.
3225  Node* effective_size =
3226  graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
3227 
3228  // Introduce the actual bounds check.
3229  Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
3230  TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3231 
3232  if (untrusted_code_mitigations_) {
3233  // In the fallthrough case, condition the index with the memory mask.
3234  Node* mem_mask = instance_cache_->mem_mask;
3235  DCHECK_NOT_NULL(mem_mask);
3236  index = graph()->NewNode(m->WordAnd(), index, mem_mask);
3237  }
3238  return index;
3239 }
3240 
3241 const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3242  wasm::ValueType type) {
3243  int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3244  MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3245  if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3246  wasm::ValueTypes::MachineRepresentationFor(type))) {
3247  return mcgraph()->machine()->Load(mach_type);
3248  }
3249  return mcgraph()->machine()->UnalignedLoad(mach_type);
3250 }
3251 
3252 const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3253  wasm::ValueType type) {
3254  int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3255  MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3256  if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3257  StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3258  return mcgraph()->machine()->Store(store_rep);
3259  }
3260  UnalignedStoreRepresentation store_rep(rep);
3261  return mcgraph()->machine()->UnalignedStore(store_rep);
3262 }
3263 
3264 Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3265  MachineRepresentation rep,
3266  Node* index, uint32_t offset,
3267  wasm::WasmCodePosition position) {
3268  int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3269  Node* info = graph()->NewNode(
3270  mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3271 
3272  Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3273  Int32Constant(offset), index);
3274  auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3275  SetEffect(graph()->NewNode(
3276  mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3277  info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
3278  };
3279  // Store address, is_store, and mem_rep.
3280  store(offsetof(wasm::MemoryTracingInfo, address),
3281  MachineRepresentation::kWord32, address);
3282  store(offsetof(wasm::MemoryTracingInfo, is_store),
3283  MachineRepresentation::kWord8,
3284  mcgraph()->Int32Constant(is_store ? 1 : 0));
3285  store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3286  MachineRepresentation::kWord8,
3287  mcgraph()->Int32Constant(static_cast<int>(rep)));
3288 
3289  Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3290  SetSourcePosition(call, position);
3291  return call;
3292 }
3293 
3294 Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3295  Node* index, uint32_t offset,
3296  uint32_t alignment,
3297  wasm::WasmCodePosition position) {
3298  Node* load;
3299 
3300  // Wasm semantics throw on OOB. Introduce explicit bounds check and
3301  // conditioning when not using the trap handler.
3302  index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3303  position, kCanOmitBoundsCheck);
3304 
3305  if (memtype.representation() == MachineRepresentation::kWord8 ||
3306  mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3307  if (use_trap_handler()) {
3308  load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3309  MemBuffer(offset), index, Effect(), Control());
3310  SetSourcePosition(load, position);
3311  } else {
3312  load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3313  MemBuffer(offset), index, Effect(), Control());
3314  }
3315  } else {
3316  // TODO(eholk): Support unaligned loads with trap handlers.
3317  DCHECK(!use_trap_handler());
3318  load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3319  MemBuffer(offset), index, Effect(), Control());
3320  }
3321 
3322  SetEffect(load);
3323 
3324 #if defined(V8_TARGET_BIG_ENDIAN)
3325  load = BuildChangeEndiannessLoad(load, memtype, type);
3326 #endif
3327 
3328  if (type == wasm::kWasmI64 &&
3329  ElementSizeInBytes(memtype.representation()) < 8) {
3330  // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3331  if (memtype.IsSigned()) {
3332  // sign extend
3333  load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3334  } else {
3335  // zero extend
3336  load =
3337  graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3338  }
3339  }
3340 
3341  if (FLAG_trace_wasm_memory) {
3342  TraceMemoryOperation(false, memtype.representation(), index, offset,
3343  position);
3344  }
3345 
3346  return load;
3347 }
3348 
3349 Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3350  uint32_t offset, uint32_t alignment, Node* val,
3351  wasm::WasmCodePosition position,
3352  wasm::ValueType type) {
3353  Node* store;
3354 
3355  index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3356  position, kCanOmitBoundsCheck);
3357 
3358 #if defined(V8_TARGET_BIG_ENDIAN)
3359  val = BuildChangeEndiannessStore(val, mem_rep, type);
3360 #endif
3361 
3362  if (mem_rep == MachineRepresentation::kWord8 ||
3363  mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3364  if (use_trap_handler()) {
3365  store =
3366  graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3367  MemBuffer(offset), index, val, Effect(), Control());
3368  SetSourcePosition(store, position);
3369  } else {
3370  StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3371  store =
3372  graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3373  index, val, Effect(), Control());
3374  }
3375  } else {
3376  // TODO(eholk): Support unaligned stores with trap handlers.
3377  DCHECK(!use_trap_handler());
3378  UnalignedStoreRepresentation rep(mem_rep);
3379  store =
3380  graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3381  MemBuffer(offset), index, val, Effect(), Control());
3382  }
3383 
3384  SetEffect(store);
3385 
3386  if (FLAG_trace_wasm_memory) {
3387  TraceMemoryOperation(true, mem_rep, index, offset, position);
3388  }
3389 
3390  return store;
3391 }
3392 
3393 namespace {
3394 Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3395  switch (rep) {
3396  case MachineRepresentation::kWord8:
3397  case MachineRepresentation::kWord16:
3398  case MachineRepresentation::kWord32:
3399  return mcgraph->Int32Constant(0);
3400  case MachineRepresentation::kWord64:
3401  return mcgraph->Int64Constant(0);
3402  case MachineRepresentation::kFloat32:
3403  return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3404  case MachineRepresentation::kFloat64:
3405  return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3406  default:
3407  UNREACHABLE();
3408  }
3409 }
3410 } // namespace
3411 
3412 Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3413  DCHECK_NOT_NULL(instance_cache_);
3414  Node* mem_start = instance_cache_->mem_start;
3415  Node* mem_size = instance_cache_->mem_size;
3416  DCHECK_NOT_NULL(mem_start);
3417  DCHECK_NOT_NULL(mem_size);
3418 
3419  // Asm.js semantics are defined in terms of typed arrays, hence OOB
3420  // reads return {undefined} coerced to the result type (0 for integers, NaN
3421  // for float and double).
3422  // Note that we check against the memory size ignoring the size of the
3423  // stored value, which is conservative if misaligned. Technically, asm.js
3424  // should never have misaligned accesses.
3425  index = Uint32ToUintptr(index);
3426  Diamond bounds_check(
3427  graph(), mcgraph()->common(),
3428  graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
3429  BranchHint::kTrue);
3430  bounds_check.Chain(Control());
3431 
3432  if (untrusted_code_mitigations_) {
3433  // Condition the index with the memory mask.
3434  Node* mem_mask = instance_cache_->mem_mask;
3435  DCHECK_NOT_NULL(mem_mask);
3436  index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
3437  }
3438 
3439  Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3440  index, Effect(), bounds_check.if_true);
3441  SetEffect(bounds_check.EffectPhi(load, Effect()));
3442  SetControl(bounds_check.merge);
3443  return bounds_check.Phi(type.representation(), load,
3444  GetAsmJsOOBValue(type.representation(), mcgraph()));
3445 }
3446 
3447 Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3448  if (mcgraph()->machine()->Is32()) return node;
3449  // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3450  Uint32Matcher matcher(node);
3451  if (matcher.HasValue()) {
3452  uintptr_t value = matcher.Value();
3453  return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3454  }
3455  return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3456 }
3457 
3458 Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3459  Node* val) {
3460  DCHECK_NOT_NULL(instance_cache_);
3461  Node* mem_start = instance_cache_->mem_start;
3462  Node* mem_size = instance_cache_->mem_size;
3463  DCHECK_NOT_NULL(mem_start);
3464  DCHECK_NOT_NULL(mem_size);
3465 
3466  // Asm.js semantics are to ignore OOB writes.
3467  // Note that we check against the memory size ignoring the size of the
3468  // stored value, which is conservative if misaligned. Technically, asm.js
3469  // should never have misaligned accesses.
3470  Diamond bounds_check(
3471  graph(), mcgraph()->common(),
3472  graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3473  BranchHint::kTrue);
3474  bounds_check.Chain(Control());
3475 
3476  if (untrusted_code_mitigations_) {
3477  // Condition the index with the memory mask.
3478  Node* mem_mask = instance_cache_->mem_mask;
3479  DCHECK_NOT_NULL(mem_mask);
3480  index =
3481  graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3482  }
3483 
3484  index = Uint32ToUintptr(index);
3485  const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3486  type.representation(), WriteBarrierKind::kNoWriteBarrier));
3487  Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
3488  bounds_check.if_true);
3489  SetEffect(bounds_check.EffectPhi(store, Effect()));
3490  SetControl(bounds_check.merge);
3491  return val;
3492 }
3493 
3494 void WasmGraphBuilder::PrintDebugName(Node* node) {
3495  PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3496 }
3497 
3498 Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3499 
3500 namespace {
3501 Signature<MachineRepresentation>* CreateMachineSignature(
3502  Zone* zone, wasm::FunctionSig* sig) {
3503  Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3504  sig->parameter_count());
3505  for (auto ret : sig->returns()) {
3506  builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3507  }
3508 
3509  for (auto param : sig->parameters()) {
3510  builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3511  }
3512  return builder.Build();
3513 }
3514 } // namespace
3515 
3516 void WasmGraphBuilder::LowerInt64() {
3517  if (mcgraph()->machine()->Is64()) return;
3518  Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3519  mcgraph()->zone(),
3520  CreateMachineSignature(mcgraph()->zone(), sig_));
3521  r.LowerGraph();
3522 }
3523 
3524 void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3525  SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3526  .LowerGraph();
3527 }
3528 
3529 void WasmGraphBuilder::SetSourcePosition(Node* node,
3530  wasm::WasmCodePosition position) {
3531  DCHECK_NE(position, wasm::kNoCodePosition);
3532  if (source_position_table_)
3533  source_position_table_->SetSourcePosition(node, SourcePosition(position));
3534 }
3535 
3536 Node* WasmGraphBuilder::S128Zero() {
3537  has_simd_ = true;
3538  return graph()->NewNode(mcgraph()->machine()->S128Zero());
3539 }
3540 
3541 Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3542  has_simd_ = true;
3543  switch (opcode) {
3544  case wasm::kExprF32x4Splat:
3545  return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3546  case wasm::kExprF32x4SConvertI32x4:
3547  return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3548  inputs[0]);
3549  case wasm::kExprF32x4UConvertI32x4:
3550  return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3551  inputs[0]);
3552  case wasm::kExprF32x4Abs:
3553  return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3554  case wasm::kExprF32x4Neg:
3555  return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3556  case wasm::kExprF32x4RecipApprox:
3557  return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3558  inputs[0]);
3559  case wasm::kExprF32x4RecipSqrtApprox:
3560  return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3561  inputs[0]);
3562  case wasm::kExprF32x4Add:
3563  return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3564  inputs[1]);
3565  case wasm::kExprF32x4AddHoriz:
3566  return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3567  inputs[1]);
3568  case wasm::kExprF32x4Sub:
3569  return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3570  inputs[1]);
3571  case wasm::kExprF32x4Mul:
3572  return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3573  inputs[1]);
3574  case wasm::kExprF32x4Min:
3575  return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3576  inputs[1]);
3577  case wasm::kExprF32x4Max:
3578  return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3579  inputs[1]);
3580  case wasm::kExprF32x4Eq:
3581  return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3582  inputs[1]);
3583  case wasm::kExprF32x4Ne:
3584  return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3585  inputs[1]);
3586  case wasm::kExprF32x4Lt:
3587  return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3588  inputs[1]);
3589  case wasm::kExprF32x4Le:
3590  return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3591  inputs[1]);
3592  case wasm::kExprF32x4Gt:
3593  return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3594  inputs[0]);
3595  case wasm::kExprF32x4Ge:
3596  return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3597  inputs[0]);
3598  case wasm::kExprI32x4Splat:
3599  return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3600  case wasm::kExprI32x4SConvertF32x4:
3601  return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3602  inputs[0]);
3603  case wasm::kExprI32x4UConvertF32x4:
3604  return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3605  inputs[0]);
3606  case wasm::kExprI32x4SConvertI16x8Low:
3607  return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3608  inputs[0]);
3609  case wasm::kExprI32x4SConvertI16x8High:
3610  return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3611  inputs[0]);
3612  case wasm::kExprI32x4Neg:
3613  return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3614  case wasm::kExprI32x4Add:
3615  return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3616  inputs[1]);
3617  case wasm::kExprI32x4AddHoriz:
3618  return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3619  inputs[1]);
3620  case wasm::kExprI32x4Sub:
3621  return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3622  inputs[1]);
3623  case wasm::kExprI32x4Mul:
3624  return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3625  inputs[1]);
3626  case wasm::kExprI32x4MinS:
3627  return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3628  inputs[1]);
3629  case wasm::kExprI32x4MaxS:
3630  return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3631  inputs[1]);
3632  case wasm::kExprI32x4Eq:
3633  return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3634  inputs[1]);
3635  case wasm::kExprI32x4Ne:
3636  return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3637  inputs[1]);
3638  case wasm::kExprI32x4LtS:
3639  return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3640  inputs[0]);
3641  case wasm::kExprI32x4LeS:
3642  return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3643  inputs[0]);
3644  case wasm::kExprI32x4GtS:
3645  return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3646  inputs[1]);
3647  case wasm::kExprI32x4GeS:
3648  return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3649  inputs[1]);
3650  case wasm::kExprI32x4UConvertI16x8Low:
3651  return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3652  inputs[0]);
3653  case wasm::kExprI32x4UConvertI16x8High:
3654  return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3655  inputs[0]);
3656  case wasm::kExprI32x4MinU:
3657  return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3658  inputs[1]);
3659  case wasm::kExprI32x4MaxU:
3660  return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3661  inputs[1]);
3662  case wasm::kExprI32x4LtU:
3663  return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3664  inputs[0]);
3665  case wasm::kExprI32x4LeU:
3666  return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3667  inputs[0]);
3668  case wasm::kExprI32x4GtU:
3669  return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3670  inputs[1]);
3671  case wasm::kExprI32x4GeU:
3672  return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3673  inputs[1]);
3674  case wasm::kExprI16x8Splat:
3675  return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3676  case wasm::kExprI16x8SConvertI8x16Low:
3677  return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3678  inputs[0]);
3679  case wasm::kExprI16x8SConvertI8x16High:
3680  return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3681  inputs[0]);
3682  case wasm::kExprI16x8Neg:
3683  return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3684  case wasm::kExprI16x8SConvertI32x4:
3685  return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3686  inputs[0], inputs[1]);
3687  case wasm::kExprI16x8Add:
3688  return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3689  inputs[1]);
3690  case wasm::kExprI16x8AddSaturateS:
3691  return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3692  inputs[0], inputs[1]);
3693  case wasm::kExprI16x8AddHoriz:
3694  return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3695  inputs[1]);
3696  case wasm::kExprI16x8Sub:
3697  return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3698  inputs[1]);
3699  case wasm::kExprI16x8SubSaturateS:
3700  return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3701  inputs[0], inputs[1]);
3702  case wasm::kExprI16x8Mul:
3703  return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3704  inputs[1]);
3705  case wasm::kExprI16x8MinS:
3706  return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3707  inputs[1]);
3708  case wasm::kExprI16x8MaxS:
3709  return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3710  inputs[1]);
3711  case wasm::kExprI16x8Eq:
3712  return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3713  inputs[1]);
3714  case wasm::kExprI16x8Ne:
3715  return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3716  inputs[1]);
3717  case wasm::kExprI16x8LtS:
3718  return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3719  inputs[0]);
3720  case wasm::kExprI16x8LeS:
3721  return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3722  inputs[0]);
3723  case wasm::kExprI16x8GtS:
3724  return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3725  inputs[1]);
3726  case wasm::kExprI16x8GeS:
3727  return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3728  inputs[1]);
3729  case wasm::kExprI16x8UConvertI8x16Low:
3730  return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3731  inputs[0]);
3732  case wasm::kExprI16x8UConvertI8x16High:
3733  return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3734  inputs[0]);
3735  case wasm::kExprI16x8UConvertI32x4:
3736  return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3737  inputs[0], inputs[1]);
3738  case wasm::kExprI16x8AddSaturateU:
3739  return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3740  inputs[0], inputs[1]);
3741  case wasm::kExprI16x8SubSaturateU:
3742  return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3743  inputs[0], inputs[1]);
3744  case wasm::kExprI16x8MinU:
3745  return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3746  inputs[1]);
3747  case wasm::kExprI16x8MaxU:
3748  return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3749  inputs[1]);
3750  case wasm::kExprI16x8LtU:
3751  return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3752  inputs[0]);
3753  case wasm::kExprI16x8LeU:
3754  return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3755  inputs[0]);
3756  case wasm::kExprI16x8GtU:
3757  return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3758  inputs[1]);
3759  case wasm::kExprI16x8GeU:
3760  return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3761  inputs[1]);
3762  case wasm::kExprI8x16Splat:
3763  return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3764  case wasm::kExprI8x16Neg:
3765  return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3766  case wasm::kExprI8x16SConvertI16x8:
3767  return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3768  inputs[0], inputs[1]);
3769  case wasm::kExprI8x16Add:
3770  return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3771  inputs[1]);
3772  case wasm::kExprI8x16AddSaturateS:
3773  return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3774  inputs[0], inputs[1]);
3775  case wasm::kExprI8x16Sub:
3776  return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3777  inputs[1]);
3778  case wasm::kExprI8x16SubSaturateS:
3779  return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3780  inputs[0], inputs[1]);
3781  case wasm::kExprI8x16Mul:
3782  return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3783  inputs[1]);
3784  case wasm::kExprI8x16MinS:
3785  return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3786  inputs[1]);
3787  case wasm::kExprI8x16MaxS:
3788  return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3789  inputs[1]);
3790  case wasm::kExprI8x16Eq:
3791  return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3792  inputs[1]);
3793  case wasm::kExprI8x16Ne:
3794  return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3795  inputs[1]);
3796  case wasm::kExprI8x16LtS:
3797  return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3798  inputs[0]);
3799  case wasm::kExprI8x16LeS:
3800  return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3801  inputs[0]);
3802  case wasm::kExprI8x16GtS:
3803  return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3804  inputs[1]);
3805  case wasm::kExprI8x16GeS:
3806  return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3807  inputs[1]);
3808  case wasm::kExprI8x16UConvertI16x8:
3809  return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3810  inputs[0], inputs[1]);
3811  case wasm::kExprI8x16AddSaturateU:
3812  return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3813  inputs[0], inputs[1]);
3814  case wasm::kExprI8x16SubSaturateU:
3815  return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3816  inputs[0], inputs[1]);
3817  case wasm::kExprI8x16MinU:
3818  return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3819  inputs[1]);
3820  case wasm::kExprI8x16MaxU:
3821  return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
3822  inputs[1]);
3823  case wasm::kExprI8x16LtU:
3824  return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
3825  inputs[0]);
3826  case wasm::kExprI8x16LeU:
3827  return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
3828  inputs[0]);
3829  case wasm::kExprI8x16GtU:
3830  return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
3831  inputs[1]);
3832  case wasm::kExprI8x16GeU:
3833  return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
3834  inputs[1]);
3835  case wasm::kExprS128And:
3836  return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
3837  inputs[1]);
3838  case wasm::kExprS128Or:
3839  return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
3840  inputs[1]);
3841  case wasm::kExprS128Xor:
3842  return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
3843  inputs[1]);
3844  case wasm::kExprS128Not:
3845  return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
3846  case wasm::kExprS128Select:
3847  return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[0],
3848  inputs[1], inputs[2]);
3849  case wasm::kExprS1x4AnyTrue:
3850  return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
3851  case wasm::kExprS1x4AllTrue:
3852  return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
3853  case wasm::kExprS1x8AnyTrue:
3854  return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
3855  case wasm::kExprS1x8AllTrue:
3856  return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
3857  case wasm::kExprS1x16AnyTrue:
3858  return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
3859  case wasm::kExprS1x16AllTrue:
3860  return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
3861  default:
3862  FATAL_UNSUPPORTED_OPCODE(opcode);
3863  }
3864 }
3865 
3866 Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
3867  Node* const* inputs) {
3868  has_simd_ = true;
3869  switch (opcode) {
3870  case wasm::kExprF32x4ExtractLane:
3871  return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
3872  inputs[0]);
3873  case wasm::kExprF32x4ReplaceLane:
3874  return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
3875  inputs[0], inputs[1]);
3876  case wasm::kExprI32x4ExtractLane:
3877  return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
3878  inputs[0]);
3879  case wasm::kExprI32x4ReplaceLane:
3880  return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
3881  inputs[0], inputs[1]);
3882  case wasm::kExprI16x8ExtractLane:
3883  return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
3884  inputs[0]);
3885  case wasm::kExprI16x8ReplaceLane:
3886  return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
3887  inputs[0], inputs[1]);
3888  case wasm::kExprI8x16ExtractLane:
3889  return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
3890  inputs[0]);
3891  case wasm::kExprI8x16ReplaceLane:
3892  return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
3893  inputs[0], inputs[1]);
3894  default:
3895  FATAL_UNSUPPORTED_OPCODE(opcode);
3896  }
3897 }
3898 
3899 Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
3900  Node* const* inputs) {
3901  has_simd_ = true;
3902  switch (opcode) {
3903  case wasm::kExprI32x4Shl:
3904  return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
3905  case wasm::kExprI32x4ShrS:
3906  return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
3907  inputs[0]);
3908  case wasm::kExprI32x4ShrU:
3909  return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
3910  inputs[0]);
3911  case wasm::kExprI16x8Shl:
3912  return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
3913  case wasm::kExprI16x8ShrS:
3914  return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
3915  inputs[0]);
3916  case wasm::kExprI16x8ShrU:
3917  return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
3918  inputs[0]);
3919  case wasm::kExprI8x16Shl:
3920  return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
3921  case wasm::kExprI8x16ShrS:
3922  return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
3923  inputs[0]);
3924  case wasm::kExprI8x16ShrU:
3925  return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
3926  inputs[0]);
3927  default:
3928  FATAL_UNSUPPORTED_OPCODE(opcode);
3929  }
3930 }
3931 
3932 Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
3933  Node* const* inputs) {
3934  has_simd_ = true;
3935  return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
3936  inputs[0], inputs[1]);
3937 }
3938 
3939 #define ATOMIC_BINOP_LIST(V) \
3940  V(I32AtomicAdd, Add, Uint32, Word32) \
3941  V(I64AtomicAdd, Add, Uint64, Word64) \
3942  V(I32AtomicAdd8U, Add, Uint8, Word32) \
3943  V(I32AtomicAdd16U, Add, Uint16, Word32) \
3944  V(I64AtomicAdd8U, Add, Uint8, Word64) \
3945  V(I64AtomicAdd16U, Add, Uint16, Word64) \
3946  V(I64AtomicAdd32U, Add, Uint32, Word64) \
3947  V(I32AtomicSub, Sub, Uint32, Word32) \
3948  V(I64AtomicSub, Sub, Uint64, Word64) \
3949  V(I32AtomicSub8U, Sub, Uint8, Word32) \
3950  V(I32AtomicSub16U, Sub, Uint16, Word32) \
3951  V(I64AtomicSub8U, Sub, Uint8, Word64) \
3952  V(I64AtomicSub16U, Sub, Uint16, Word64) \
3953  V(I64AtomicSub32U, Sub, Uint32, Word64) \
3954  V(I32AtomicAnd, And, Uint32, Word32) \
3955  V(I64AtomicAnd, And, Uint64, Word64) \
3956  V(I32AtomicAnd8U, And, Uint8, Word32) \
3957  V(I64AtomicAnd16U, And, Uint16, Word64) \
3958  V(I32AtomicAnd16U, And, Uint16, Word32) \
3959  V(I64AtomicAnd8U, And, Uint8, Word64) \
3960  V(I64AtomicAnd32U, And, Uint32, Word64) \
3961  V(I32AtomicOr, Or, Uint32, Word32) \
3962  V(I64AtomicOr, Or, Uint64, Word64) \
3963  V(I32AtomicOr8U, Or, Uint8, Word32) \
3964  V(I32AtomicOr16U, Or, Uint16, Word32) \
3965  V(I64AtomicOr8U, Or, Uint8, Word64) \
3966  V(I64AtomicOr16U, Or, Uint16, Word64) \
3967  V(I64AtomicOr32U, Or, Uint32, Word64) \
3968  V(I32AtomicXor, Xor, Uint32, Word32) \
3969  V(I64AtomicXor, Xor, Uint64, Word64) \
3970  V(I32AtomicXor8U, Xor, Uint8, Word32) \
3971  V(I32AtomicXor16U, Xor, Uint16, Word32) \
3972  V(I64AtomicXor8U, Xor, Uint8, Word64) \
3973  V(I64AtomicXor16U, Xor, Uint16, Word64) \
3974  V(I64AtomicXor32U, Xor, Uint32, Word64) \
3975  V(I32AtomicExchange, Exchange, Uint32, Word32) \
3976  V(I64AtomicExchange, Exchange, Uint64, Word64) \
3977  V(I32AtomicExchange8U, Exchange, Uint8, Word32) \
3978  V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
3979  V(I64AtomicExchange8U, Exchange, Uint8, Word64) \
3980  V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
3981  V(I64AtomicExchange32U, Exchange, Uint32, Word64)
3982 
3983 #define ATOMIC_CMP_EXCHG_LIST(V) \
3984  V(I32AtomicCompareExchange, Uint32, Word32) \
3985  V(I64AtomicCompareExchange, Uint64, Word64) \
3986  V(I32AtomicCompareExchange8U, Uint8, Word32) \
3987  V(I32AtomicCompareExchange16U, Uint16, Word32) \
3988  V(I64AtomicCompareExchange8U, Uint8, Word64) \
3989  V(I64AtomicCompareExchange16U, Uint16, Word64) \
3990  V(I64AtomicCompareExchange32U, Uint32, Word64)
3991 
3992 #define ATOMIC_LOAD_LIST(V) \
3993  V(I32AtomicLoad, Uint32, Word32) \
3994  V(I64AtomicLoad, Uint64, Word64) \
3995  V(I32AtomicLoad8U, Uint8, Word32) \
3996  V(I32AtomicLoad16U, Uint16, Word32) \
3997  V(I64AtomicLoad8U, Uint8, Word64) \
3998  V(I64AtomicLoad16U, Uint16, Word64) \
3999  V(I64AtomicLoad32U, Uint32, Word64)
4000 
4001 #define ATOMIC_STORE_LIST(V) \
4002  V(I32AtomicStore, Uint32, kWord32, Word32) \
4003  V(I64AtomicStore, Uint64, kWord64, Word64) \
4004  V(I32AtomicStore8U, Uint8, kWord8, Word32) \
4005  V(I32AtomicStore16U, Uint16, kWord16, Word32) \
4006  V(I64AtomicStore8U, Uint8, kWord8, Word64) \
4007  V(I64AtomicStore16U, Uint16, kWord16, Word64) \
4008  V(I64AtomicStore32U, Uint32, kWord32, Word64)
4009 
4010 Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
4011  uint32_t alignment, uint32_t offset,
4012  wasm::WasmCodePosition position) {
4013  Node* node;
4014  switch (opcode) {
4015 #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \
4016  case wasm::kExpr##Name: { \
4017  Node* index = CheckBoundsAndAlignment( \
4018  wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4019  position); \
4020  node = graph()->NewNode( \
4021  mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
4022  MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4023  break; \
4024  }
4025  ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
4026 #undef BUILD_ATOMIC_BINOP
4027 
4028 #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \
4029  case wasm::kExpr##Name: { \
4030  Node* index = CheckBoundsAndAlignment( \
4031  wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4032  position); \
4033  node = graph()->NewNode( \
4034  mcgraph()->machine()->Prefix##AtomicCompareExchange( \
4035  MachineType::Type()), \
4036  MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
4037  break; \
4038  }
4039  ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
4040 #undef BUILD_ATOMIC_CMP_EXCHG
4041 
4042 #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \
4043  case wasm::kExpr##Name: { \
4044  Node* index = CheckBoundsAndAlignment( \
4045  wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4046  position); \
4047  node = graph()->NewNode( \
4048  mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \
4049  MemBuffer(offset), index, Effect(), Control()); \
4050  break; \
4051  }
4052  ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
4053 #undef BUILD_ATOMIC_LOAD_OP
4054 
4055 #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \
4056  case wasm::kExpr##Name: { \
4057  Node* index = CheckBoundsAndAlignment( \
4058  wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4059  position); \
4060  node = graph()->NewNode( \
4061  mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
4062  MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4063  break; \
4064  }
4065  ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
4066 #undef BUILD_ATOMIC_STORE_OP
4067  case wasm::kExprAtomicWake: {
4068  Node* index = CheckBoundsAndAlignment(
4069  wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4070  position);
4071  // Now that we've bounds-checked, compute the effective address.
4072  Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4073  Uint32Constant(offset), index);
4074  WasmAtomicWakeDescriptor interface_descriptor;
4075  auto call_descriptor = Linkage::GetStubCallDescriptor(
4076  mcgraph()->zone(), interface_descriptor,
4077  interface_descriptor.GetStackParameterCount(),
4078  CallDescriptor::kNoFlags, Operator::kNoProperties,
4079  StubCallMode::kCallWasmRuntimeStub);
4080  Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4081  wasm::WasmCode::kWasmAtomicWake, RelocInfo::WASM_STUB_CALL);
4082  node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4083  call_target, address, inputs[1], Effect(),
4084  Control());
4085  break;
4086  }
4087 
4088  case wasm::kExprI32AtomicWait: {
4089  Node* index = CheckBoundsAndAlignment(
4090  wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4091  position);
4092  // Now that we've bounds-checked, compute the effective address.
4093  Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4094  Uint32Constant(offset), index);
4095  Node* timeout;
4096  if (mcgraph()->machine()->Is32()) {
4097  timeout = BuildF64SConvertI64(inputs[2]);
4098  } else {
4099  timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4100  inputs[2]);
4101  }
4102  WasmI32AtomicWaitDescriptor interface_descriptor;
4103  auto call_descriptor = Linkage::GetStubCallDescriptor(
4104  mcgraph()->zone(), interface_descriptor,
4105  interface_descriptor.GetStackParameterCount(),
4106  CallDescriptor::kNoFlags, Operator::kNoProperties,
4107  StubCallMode::kCallWasmRuntimeStub);
4108  Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4109  wasm::WasmCode::kWasmI32AtomicWait, RelocInfo::WASM_STUB_CALL);
4110  node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4111  call_target, address, inputs[1], timeout,
4112  Effect(), Control());
4113  break;
4114  }
4115 
4116  default:
4117  FATAL_UNSUPPORTED_OPCODE(opcode);
4118  }
4119  return SetEffect(node);
4120 }
4121 
4122 #undef ATOMIC_BINOP_LIST
4123 #undef ATOMIC_CMP_EXCHG_LIST
4124 #undef ATOMIC_LOAD_LIST
4125 #undef ATOMIC_STORE_LIST
4126 
4127 class WasmDecorator final : public GraphDecorator {
4128  public:
4129  explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
4130  : origins_(origins), decoder_(decoder) {}
4131 
4132  void Decorate(Node* node) final {
4133  origins_->SetNodeOrigin(
4134  node, NodeOrigin("wasm graph creation", "n/a",
4135  NodeOrigin::kWasmBytecode, decoder_->position()));
4136  }
4137 
4138  private:
4139  compiler::NodeOriginTable* origins_;
4140  wasm::Decoder* decoder_;
4141 };
4142 
4143 void WasmGraphBuilder::AddBytecodePositionDecorator(
4144  NodeOriginTable* node_origins, wasm::Decoder* decoder) {
4145  DCHECK_NULL(decorator_);
4146  decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
4147  graph()->AddDecorator(decorator_);
4148 }
4149 
4150 void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
4151  DCHECK_NOT_NULL(decorator_);
4152  graph()->RemoveDecorator(decorator_);
4153  decorator_ = nullptr;
4154 }
4155 
4156 namespace {
4157 bool must_record_function_compilation(Isolate* isolate) {
4158  return isolate->logger()->is_listening_to_code_events() ||
4159  isolate->is_profiling();
4160 }
4161 
4162 PRINTF_FORMAT(4, 5)
4163 void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
4164  Isolate* isolate, Handle<Code> code,
4165  const char* format, ...) {
4166  DCHECK(must_record_function_compilation(isolate));
4167 
4168  ScopedVector<char> buffer(128);
4169  va_list arguments;
4170  va_start(arguments, format);
4171  int len = VSNPrintF(buffer, format, arguments);
4172  CHECK_LT(0, len);
4173  va_end(arguments);
4174  Handle<String> name_str =
4175  isolate->factory()->NewStringFromAsciiChecked(buffer.start());
4176  PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
4177 }
4178 
4179 class WasmWrapperGraphBuilder : public WasmGraphBuilder {
4180  public:
4181  WasmWrapperGraphBuilder(Zone* zone, JSGraph* jsgraph, wasm::FunctionSig* sig,
4182  compiler::SourcePositionTable* spt,
4183  StubCallMode stub_mode)
4184  : WasmGraphBuilder(nullptr, zone, jsgraph, sig, spt),
4185  isolate_(jsgraph->isolate()),
4186  jsgraph_(jsgraph),
4187  stub_mode_(stub_mode) {}
4188 
4189  Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
4190  MachineOperatorBuilder* machine = mcgraph()->machine();
4191  CommonOperatorBuilder* common = mcgraph()->common();
4192  Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4193  ? mcgraph()->RelocatableIntPtrConstant(
4194  wasm::WasmCode::kWasmAllocateHeapNumber,
4195  RelocInfo::WASM_STUB_CALL)
4196  : jsgraph()->HeapConstant(
4197  BUILTIN_CODE(isolate_, AllocateHeapNumber));
4198  if (!allocate_heap_number_operator_.is_set()) {
4199  auto call_descriptor = Linkage::GetStubCallDescriptor(
4200  mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
4201  CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
4202  allocate_heap_number_operator_.set(common->Call(call_descriptor));
4203  }
4204  Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
4205  target, Effect(), control);
4206  SetEffect(
4207  graph()->NewNode(machine->Store(StoreRepresentation(
4208  MachineRepresentation::kFloat64, kNoWriteBarrier)),
4209  heap_number, BuildHeapNumberValueIndexConstant(),
4210  value, heap_number, control));
4211  return heap_number;
4212  }
4213 
4214  Node* BuildChangeSmiToFloat64(Node* value) {
4215  return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4216  BuildChangeSmiToInt32(value));
4217  }
4218 
4219  Node* BuildTestHeapObject(Node* value) {
4220  return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4221  mcgraph()->IntPtrConstant(kHeapObjectTag));
4222  }
4223 
4224  Node* BuildLoadHeapNumberValue(Node* value) {
4225  return SetEffect(graph()->NewNode(
4226  mcgraph()->machine()->Load(MachineType::Float64()), value,
4227  BuildHeapNumberValueIndexConstant(), Effect(), Control()));
4228  }
4229 
4230  Node* BuildHeapNumberValueIndexConstant() {
4231  return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4232  }
4233 
4234  Node* BuildChangeInt32ToTagged(Node* value) {
4235  MachineOperatorBuilder* machine = mcgraph()->machine();
4236  CommonOperatorBuilder* common = mcgraph()->common();
4237 
4238  if (SmiValuesAre32Bits()) {
4239  return BuildChangeInt32ToSmi(value);
4240  }
4241  DCHECK(SmiValuesAre31Bits());
4242 
4243  Node* effect = Effect();
4244  Node* control = Control();
4245  Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4246  graph()->start());
4247 
4248  Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4249  Node* branch =
4250  graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4251 
4252  Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4253  Node* vtrue = BuildAllocateHeapNumberWithValue(
4254  graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4255  Node* etrue = Effect();
4256 
4257  Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4258  Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4259  vfalse = BuildChangeInt32ToIntPtr(vfalse);
4260 
4261  Node* merge =
4262  SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
4263  SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
4264  return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4265  vtrue, vfalse, merge);
4266  }
4267 
4268  Node* BuildChangeFloat64ToTagged(Node* value) {
4269  MachineOperatorBuilder* machine = mcgraph()->machine();
4270  CommonOperatorBuilder* common = mcgraph()->common();
4271 
4272  // Check several conditions:
4273  // i32?
4274  // ├─ true: zero?
4275  // │ ├─ true: negative?
4276  // │ │ ├─ true: box
4277  // │ │ └─ false: potentially Smi
4278  // │ └─ false: potentially Smi
4279  // └─ false: box
4280  // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
4281  // still need to check whether the value fits in a Smi.
4282 
4283  Node* effect = Effect();
4284  Node* control = Control();
4285  Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4286  Node* check_i32 = graph()->NewNode(
4287  machine->Float64Equal(), value,
4288  graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4289  Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
4290 
4291  Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
4292  Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
4293 
4294  // We only need to check for -0 if the {value} can potentially contain -0.
4295  Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4296  mcgraph()->Int32Constant(0));
4297  Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4298  check_zero, if_i32);
4299 
4300  Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4301  Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
4302 
4303  // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4304  Node* check_negative = graph()->NewNode(
4305  machine->Int32LessThan(),
4306  graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4307  mcgraph()->Int32Constant(0));
4308  Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4309  check_negative, if_zero);
4310 
4311  Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4312  Node* if_not_negative =
4313  graph()->NewNode(common->IfFalse(), branch_negative);
4314 
4315  // We need to create a box for negative 0.
4316  Node* if_smi =
4317  graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
4318  Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
4319 
4320  // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4321  // 32-bit machines we need to deal with potential overflow and fallback to
4322  // boxing.
4323  Node* vsmi;
4324  if (SmiValuesAre32Bits()) {
4325  vsmi = BuildChangeInt32ToSmi(value32);
4326  } else {
4327  DCHECK(SmiValuesAre31Bits());
4328  Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4329  value32, if_smi);
4330 
4331  Node* check_ovf =
4332  graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4333  Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4334  check_ovf, if_smi);
4335 
4336  Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4337  if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4338 
4339  if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4340  vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4341  vsmi = BuildChangeInt32ToIntPtr(vsmi);
4342  }
4343 
4344  // Allocate the box for the {value}.
4345  Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4346  Node* ebox = Effect();
4347 
4348  Node* merge =
4349  SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
4350  SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
4351  return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4352  vsmi, vbox, merge);
4353  }
4354 
4355  int AddArgumentNodes(Node** args, int pos, int param_count,
4356  wasm::FunctionSig* sig) {
4357  // Convert wasm numbers to JS values.
4358  for (int i = 0; i < param_count; ++i) {
4359  Node* param =
4360  Param(i + 1); // Start from index 1 to drop the instance_node.
4361  args[pos++] = ToJS(param, sig->GetParam(i));
4362  }
4363  return pos;
4364  }
4365 
4366  Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4367  auto call_descriptor = Linkage::GetStubCallDescriptor(
4368  mcgraph()->zone(), TypeConversionDescriptor{}, 0,
4369  CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
4370  Node* stub_code =
4371  (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4372  ? mcgraph()->RelocatableIntPtrConstant(
4373  wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
4374  : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
4375 
4376  Node* result = SetEffect(
4377  graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4378  node, js_context, Effect(), Control()));
4379 
4380  SetSourcePosition(result, 1);
4381 
4382  return result;
4383  }
4384 
4385  Node* BuildChangeTaggedToFloat64(Node* value) {
4386  MachineOperatorBuilder* machine = mcgraph()->machine();
4387  CommonOperatorBuilder* common = mcgraph()->common();
4388 
4389  // Implement the following decision tree:
4390  // heap object?
4391  // ├─ true: undefined?
4392  // │ ├─ true: f64 const
4393  // │ └─ false: load heap number value
4394  // └─ false: smi to float64
4395 
4396  Node* check_heap_object = BuildTestHeapObject(value);
4397  Diamond is_heap_object(graph(), common, check_heap_object,
4398  BranchHint::kFalse);
4399  is_heap_object.Chain(Control());
4400 
4401  SetControl(is_heap_object.if_true);
4402  Node* orig_effect = Effect();
4403 
4404  Node* undefined_node =
4405  LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4406  Node* check_undefined =
4407  graph()->NewNode(machine->WordEqual(), value, undefined_node);
4408  Node* effect_tagged = Effect();
4409 
4410  Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
4411  is_undefined.Nest(is_heap_object, true);
4412 
4413  SetControl(is_undefined.if_false);
4414  Node* vheap_number = BuildLoadHeapNumberValue(value);
4415  Node* effect_undefined = Effect();
4416 
4417  SetControl(is_undefined.merge);
4418  Node* vundefined =
4419  mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4420  Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
4421  vundefined, vheap_number);
4422 
4423  effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
4424 
4425  // If input is Smi: just convert to float64.
4426  Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4427 
4428  SetControl(is_heap_object.merge);
4429  SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
4430  return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
4431  vfrom_smi);
4432  }
4433 
4434  Node* ToJS(Node* node, wasm::ValueType type) {
4435  switch (type) {
4436  case wasm::kWasmI32:
4437  return BuildChangeInt32ToTagged(node);
4438  case wasm::kWasmS128:
4439  case wasm::kWasmI64:
4440  UNREACHABLE();
4441  case wasm::kWasmF32:
4442  node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4443  node);
4444  return BuildChangeFloat64ToTagged(node);
4445  case wasm::kWasmF64:
4446  return BuildChangeFloat64ToTagged(node);
4447  case wasm::kWasmAnyRef:
4448  return node;
4449  default:
4450  UNREACHABLE();
4451  }
4452  }
4453 
4454  Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4455  DCHECK_NE(wasm::kWasmStmt, type);
4456 
4457  // The parameter is of type AnyRef, we take it as is.
4458  if (type == wasm::kWasmAnyRef) {
4459  return node;
4460  }
4461 
4462  // Do a JavaScript ToNumber.
4463  Node* num = BuildJavaScriptToNumber(node, js_context);
4464 
4465  // Change representation.
4466  num = BuildChangeTaggedToFloat64(num);
4467 
4468  switch (type) {
4469  case wasm::kWasmI32: {
4470  num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4471  num);
4472  break;
4473  }
4474  case wasm::kWasmS128:
4475  case wasm::kWasmI64:
4476  UNREACHABLE();
4477  case wasm::kWasmF32:
4478  num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4479  num);
4480  break;
4481  case wasm::kWasmF64:
4482  break;
4483  default:
4484  UNREACHABLE();
4485  }
4486  return num;
4487  }
4488 
4489  void BuildModifyThreadInWasmFlag(bool new_value) {
4490  if (!trap_handler::IsTrapHandlerEnabled()) return;
4491  Node* isolate_root =
4492  LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
4493 
4494  Node* thread_in_wasm_flag_address =
4495  LOAD_RAW(isolate_root, Isolate::thread_in_wasm_flag_address_offset(),
4496  MachineType::Pointer());
4497 
4498  SetEffect(graph()->NewNode(
4499  mcgraph()->machine()->Store(StoreRepresentation(
4500  MachineRepresentation::kWord32, kNoWriteBarrier)),
4501  thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
4502  mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
4503  }
4504 
4505  Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
4506  Node* shared = SetEffect(graph()->NewNode(
4507  jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4508  jsgraph()->Int32Constant(
4509  wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()),
4510  Effect(), Control()));
4511  return SetEffect(graph()->NewNode(
4512  jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4513  jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4514  kHeapObjectTag),
4515  Effect(), Control()));
4516  }
4517 
4518  Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
4519  return SetEffect(graph()->NewNode(
4520  jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4521  jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4522  kHeapObjectTag),
4523  Effect(), Control()));
4524  }
4525 
4526  Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
4527  Node* function_index_smi = SetEffect(graph()->NewNode(
4528  jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4529  jsgraph()->Int32Constant(
4530  WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
4531  Effect(), Control()));
4532  Node* function_index = BuildChangeSmiToInt32(function_index_smi);
4533  return function_index;
4534  }
4535 
4536  Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
4537  Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
4538  jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4539  jsgraph()->Int32Constant(
4540  WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
4541  Effect(), Control()));
4542  Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
4543  return jump_table_offset;
4544  }
4545 
4546  void BuildJSToWasmWrapper(bool is_import) {
4547  const int wasm_count = static_cast<int>(sig_->parameter_count());
4548 
4549  // Build the start and the JS parameter nodes.
4550  SetEffect(SetControl(Start(wasm_count + 5)));
4551 
4552  // Create the js_closure and js_context parameters.
4553  Node* js_closure =
4554  graph()->NewNode(jsgraph()->common()->Parameter(
4555  Linkage::kJSCallClosureParamIndex, "%closure"),
4556  graph()->start());
4557  Node* js_context = graph()->NewNode(
4558  mcgraph()->common()->Parameter(
4559  Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
4560  graph()->start());
4561 
4562  // Create the instance_node node to pass as parameter. It is loaded from
4563  // an actual reference to an instance or a placeholder reference,
4564  // called {WasmExportedFunction} via the {WasmExportedFunctionData}
4565  // structure.
4566  Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
4567  instance_node_.set(
4568  BuildLoadInstanceFromExportedFunctionData(function_data));
4569 
4570  if (!wasm::IsJSCompatibleSignature(sig_)) {
4571  // Throw a TypeError. Use the js_context of the calling javascript
4572  // function (passed as a parameter), such that the generated code is
4573  // js_context independent.
4574  BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4575  nullptr, 0);
4576  Return(jsgraph()->SmiConstant(0));
4577  return;
4578  }
4579 
4580  const int args_count = wasm_count + 1; // +1 for wasm_code.
4581  Node** args = Buffer(args_count);
4582  Node** rets;
4583 
4584  // Convert JS parameters to wasm numbers.
4585  for (int i = 0; i < wasm_count; ++i) {
4586  Node* param = Param(i + 1);
4587  Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
4588  args[i + 1] = wasm_param;
4589  }
4590 
4591  // Set the ThreadInWasm flag before we do the actual call.
4592  BuildModifyThreadInWasmFlag(true);
4593 
4594  if (is_import) {
4595  // Call to an imported function.
4596  // Load function index from {WasmExportedFunctionData}.
4597  Node* function_index =
4598  BuildLoadFunctionIndexFromExportedFunctionData(function_data);
4599  BuildImportCall(sig_, args, &rets, wasm::kNoCodePosition, function_index);
4600  } else {
4601  // Call to a wasm function defined in this module.
4602  // The call target is the jump table slot for that function.
4603  Node* jump_table_start =
4604  LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
4605  Node* jump_table_offset =
4606  BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
4607  Node* jump_table_slot = graph()->NewNode(
4608  mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
4609  args[0] = jump_table_slot;
4610 
4611  BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
4612  kNoRetpoline);
4613  }
4614 
4615  // Clear the ThreadInWasm flag.
4616  BuildModifyThreadInWasmFlag(false);
4617 
4618  Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
4619  : ToJS(rets[0], sig_->GetReturn());
4620  Return(jsval);
4621  }
4622 
4623  bool BuildWasmImportCallWrapper(WasmImportCallKind kind) {
4624  int wasm_count = static_cast<int>(sig_->parameter_count());
4625 
4626  // Build the start and the parameter nodes.
4627  SetEffect(SetControl(Start(wasm_count + 4)));
4628 
4629  instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4630 
4631  Node* native_context =
4632  LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
4633 
4634  if (kind == WasmImportCallKind::kRuntimeTypeError) {
4635  // =======================================================================
4636  // === Runtime TypeError =================================================
4637  // =======================================================================
4638  BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
4639  native_context, nullptr, 0);
4640  // We don't need to return a value here, as the runtime call will not
4641  // return anyway (the c entry stub will trigger stack unwinding).
4642  ReturnVoid();
4643  return false;
4644  }
4645 
4646  // The callable is passed as the last parameter, after WASM arguments.
4647  Node* callable_node = Param(wasm_count + 1);
4648 
4649  Node* undefined_node =
4650  LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4651 
4652  Node* call = nullptr;
4653  bool sloppy_receiver = true;
4654 
4655  BuildModifyThreadInWasmFlag(false); // exiting WASM via call.
4656 
4657  switch (kind) {
4658  // =======================================================================
4659  // === JS Functions with matching arity ==================================
4660  // =======================================================================
4661  case WasmImportCallKind::kJSFunctionArityMatch:
4662  sloppy_receiver = false;
4663  V8_FALLTHROUGH; // fallthru
4664  case WasmImportCallKind::kJSFunctionArityMatchSloppy: {
4665  Node** args = Buffer(wasm_count + 9);
4666  int pos = 0;
4667  Node* function_context = SetEffect(graph()->NewNode(
4668  mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4669  callable_node,
4670  mcgraph()->Int32Constant(
4671  wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
4672  Effect(), Control()));
4673  args[pos++] = callable_node; // target callable.
4674  // Receiver.
4675  if (sloppy_receiver) {
4676  Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
4677  native_context, Context::GLOBAL_PROXY_INDEX);
4678  args[pos++] = global_proxy;
4679  } else {
4680  args[pos++] = undefined_node;
4681  }
4682 
4683  auto call_descriptor = Linkage::GetJSCallDescriptor(
4684  graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
4685 
4686  // Convert wasm numbers to JS values.
4687  pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4688 
4689  args[pos++] = undefined_node; // new target
4690  args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4691  args[pos++] = function_context;
4692  args[pos++] = Effect();
4693  args[pos++] = Control();
4694 
4695  call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4696  args);
4697  break;
4698  }
4699  // =======================================================================
4700  // === JS Functions with arguments adapter ===============================
4701  // =======================================================================
4702  case WasmImportCallKind::kJSFunctionArityMismatch:
4703  sloppy_receiver = false;
4704  V8_FALLTHROUGH; // fallthru
4705  case WasmImportCallKind::kJSFunctionArityMismatchSloppy: {
4706  Node** args = Buffer(wasm_count + 9);
4707  int pos = 0;
4708  Node* function_context = SetEffect(graph()->NewNode(
4709  mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4710  callable_node,
4711  mcgraph()->Int32Constant(
4712  wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
4713  Effect(), Control()));
4714  args[pos++] =
4715  BuildLoadBuiltinFromInstance(Builtins::kArgumentsAdaptorTrampoline);
4716  args[pos++] = callable_node; // target callable
4717  args[pos++] = undefined_node; // new target
4718  args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4719 
4720  // Load shared function info, and then the formal parameter count.
4721  Node* shared_function_info = SetEffect(graph()->NewNode(
4722  mcgraph()->machine()->Load(MachineType::TaggedPointer()),
4723  callable_node,
4724  mcgraph()->Int32Constant(
4725  wasm::ObjectAccess::
4726  SharedFunctionInfoOffsetInTaggedJSFunction()),
4727  Effect(), Control()));
4728  Node* formal_param_count = SetEffect(graph()->NewNode(
4729  mcgraph()->machine()->Load(MachineType::Uint16()),
4730  shared_function_info,
4731  mcgraph()->Int32Constant(
4732  wasm::ObjectAccess::
4733  FormalParameterCountOffsetInSharedFunctionInfo()),
4734  Effect(), Control()));
4735  args[pos++] = formal_param_count;
4736 
4737  // Receiver.
4738  if (sloppy_receiver) {
4739  Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
4740  native_context, Context::GLOBAL_PROXY_INDEX);
4741  args[pos++] = global_proxy;
4742  } else {
4743  args[pos++] = undefined_node;
4744  }
4745 
4746 #ifdef V8_TARGET_ARCH_IA32
4747  // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative
4748  // call instead once builtins are embedded in every build configuration.
4749  CallDescriptor::Flags flags = CallDescriptor::kAllowCallThroughSlot;
4750 #else
4751  CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4752 #endif
4753  auto call_descriptor = Linkage::GetStubCallDescriptor(
4754  mcgraph()->zone(), ArgumentsAdaptorDescriptor{}, 1 + wasm_count,
4755  flags, Operator::kNoProperties);
4756 
4757  // Convert wasm numbers to JS values.
4758  pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4759  args[pos++] = function_context;
4760  args[pos++] = Effect();
4761  args[pos++] = Control();
4762  call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4763  args);
4764  break;
4765  }
4766  // =======================================================================
4767  // === General case of unknown callable ==================================
4768  // =======================================================================
4769  case WasmImportCallKind::kUseCallBuiltin: {
4770  Node** args = Buffer(wasm_count + 9);
4771  int pos = 0;
4772  args[pos++] = mcgraph()->RelocatableIntPtrConstant(
4773  wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
4774  args[pos++] = callable_node;
4775  args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
4776  args[pos++] = undefined_node; // receiver
4777 
4778  auto call_descriptor = Linkage::GetStubCallDescriptor(
4779  graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
4780  CallDescriptor::kNoFlags, Operator::kNoProperties,
4781  StubCallMode::kCallWasmRuntimeStub);
4782 
4783  // Convert wasm numbers to JS values.
4784  pos = AddArgumentNodes(args, pos, wasm_count, sig_);
4785 
4786  // The native_context is sufficient here, because all kind of callables
4787  // which depend on the context provide their own context. The context
4788  // here is only needed if the target is a constructor to throw a
4789  // TypeError, if the target is a native function, or if the target is a
4790  // callable JSObject, which can only be constructed by the runtime.
4791  args[pos++] = native_context;
4792  args[pos++] = Effect();
4793  args[pos++] = Control();
4794 
4795  call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
4796  args);
4797  break;
4798  }
4799  default:
4800  UNREACHABLE();
4801  }
4802  DCHECK_NOT_NULL(call);
4803 
4804  SetEffect(call);
4805  SetSourcePosition(call, 0);
4806 
4807  // Convert the return value back.
4808  Node* val = sig_->return_count() == 0
4809  ? mcgraph()->Int32Constant(0)
4810  : FromJS(call, native_context, sig_->GetReturn());
4811 
4812  BuildModifyThreadInWasmFlag(true); // reentering WASM upon return.
4813 
4814  Return(val);
4815  return true;
4816  }
4817 
4818  void BuildWasmInterpreterEntry(uint32_t func_index) {
4819  int param_count = static_cast<int>(sig_->parameter_count());
4820 
4821  // Build the start and the parameter nodes.
4822  SetEffect(SetControl(Start(param_count + 3)));
4823 
4824  // Create the instance_node from the passed parameter.
4825  instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
4826 
4827  // Compute size for the argument buffer.
4828  int args_size_bytes = 0;
4829  for (wasm::ValueType type : sig_->parameters()) {
4830  args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
4831  }
4832 
4833  // The return value is also passed via this buffer:
4834  DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
4835  // TODO(wasm): Handle multi-value returns.
4836  DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
4837  int return_size_bytes =
4838  sig_->return_count() == 0
4839  ? 0
4840  : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
4841 
4842  // Get a stack slot for the arguments.
4843  Node* arg_buffer =
4844  args_size_bytes == 0 && return_size_bytes == 0
4845  ? mcgraph()->IntPtrConstant(0)
4846  : graph()->NewNode(mcgraph()->machine()->StackSlot(
4847  std::max(args_size_bytes, return_size_bytes), 8));
4848 
4849  // Now store all our arguments to the buffer.
4850  int offset = 0;
4851 
4852  for (int i = 0; i < param_count; ++i) {
4853  wasm::ValueType type = sig_->GetParam(i);
4854  // Start from the parameter with index 1 to drop the instance_node.
4855  SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
4856  Int32Constant(offset), Param(i + 1), Effect(),
4857  Control()));
4858  offset += wasm::ValueTypes::ElementSizeInBytes(type);
4859  }
4860  DCHECK_EQ(args_size_bytes, offset);
4861 
4862  // We are passing the raw arg_buffer here. To the GC and other parts, it
4863  // looks like a Smi (lowest bit not set). In the runtime function however,
4864  // don't call Smi::value on it, but just cast it to a byte pointer.
4865  Node* parameters[] = {jsgraph()->SmiConstant(func_index), arg_buffer};
4866  BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
4867  arraysize(parameters));
4868 
4869  // Read back the return value.
4870  if (sig_->return_count() == 0) {
4871  Return(Int32Constant(0));
4872  } else {
4873  // TODO(wasm): Implement multi-return.
4874  DCHECK_EQ(1, sig_->return_count());
4875  MachineType load_rep =
4876  wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
4877  Node* val = SetEffect(
4878  graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
4879  Int32Constant(0), Effect(), Control()));
4880  Return(val);
4881  }
4882 
4883  if (ContainsInt64(sig_)) LowerInt64();
4884  }
4885 
4886  void BuildCWasmEntry() {
4887  // Build the start and the JS parameter nodes.
4888  SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
4889 
4890  // Create parameter nodes (offset by 1 for the receiver parameter).
4891  Node* code_entry = Param(CWasmEntryParameters::kCodeEntry + 1);
4892  Node* object_ref_node = Param(CWasmEntryParameters::kObjectRef + 1);
4893  Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
4894 
4895  int wasm_arg_count = static_cast<int>(sig_->parameter_count());
4896  int arg_count =
4897  wasm_arg_count + 4; // code, object_ref_node, control, effect
4898  Node** args = Buffer(arg_count);
4899 
4900  int pos = 0;
4901  args[pos++] = code_entry;
4902  args[pos++] = object_ref_node;
4903 
4904  int offset = 0;
4905  for (wasm::ValueType type : sig_->parameters()) {
4906  Node* arg_load = SetEffect(
4907  graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
4908  Int32Constant(offset), Effect(), Control()));
4909  args[pos++] = arg_load;
4910  offset += wasm::ValueTypes::ElementSizeInBytes(type);
4911  }
4912 
4913  args[pos++] = Effect();
4914  args[pos++] = Control();
4915  DCHECK_EQ(arg_count, pos);
4916 
4917  // Call the wasm code.
4918  auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
4919 
4920  Node* call = SetEffect(graph()->NewNode(
4921  mcgraph()->common()->Call(call_descriptor), arg_count, args));
4922 
4923  // Store the return value.
4924  DCHECK_GE(1, sig_->return_count());
4925  if (sig_->return_count() == 1) {
4926  StoreRepresentation store_rep(
4927  wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
4928  kNoWriteBarrier);
4929  SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
4930  arg_buffer, Int32Constant(0), call, Effect(),
4931  Control()));
4932  }
4933  Return(jsgraph()->SmiConstant(0));
4934 
4935  if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
4936  MachineRepresentation sig_reps[] = {
4937  MachineRepresentation::kWord32, // return value
4938  MachineRepresentation::kTagged, // receiver
4939  MachineRepresentation::kTagged, // arg0 (code)
4940  MachineRepresentation::kTagged // arg1 (buffer)
4941  };
4942  Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
4943  Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
4944  mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
4945  r.LowerGraph();
4946  }
4947  }
4948 
4949  JSGraph* jsgraph() { return jsgraph_; }
4950 
4951  private:
4952  Isolate* const isolate_;
4953  JSGraph* jsgraph_;
4954  StubCallMode stub_mode_;
4955  SetOncePointer<const Operator> allocate_heap_number_operator_;
4956 };
4957 
4958 void AppendSignature(char* buffer, size_t max_name_len,
4959  wasm::FunctionSig* sig) {
4960  size_t name_len = strlen(buffer);
4961  auto append_name_char = [&](char c) {
4962  if (name_len + 1 < max_name_len) buffer[name_len++] = c;
4963  };
4964  for (wasm::ValueType t : sig->parameters()) {
4965  append_name_char(wasm::ValueTypes::ShortNameOf(t));
4966  }
4967  append_name_char(':');
4968  for (wasm::ValueType t : sig->returns()) {
4969  append_name_char(wasm::ValueTypes::ShortNameOf(t));
4970  }
4971  buffer[name_len] = '\0';
4972 }
4973 
4974 } // namespace
4975 
4976 MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
4977  wasm::FunctionSig* sig,
4978  bool is_import) {
4979  TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
4980  "CompileJSToWasmWrapper");
4981  //----------------------------------------------------------------------------
4982  // Create the Graph.
4983  //----------------------------------------------------------------------------
4984  Zone zone(isolate->allocator(), ZONE_NAME);
4985  Graph graph(&zone);
4986  CommonOperatorBuilder common(&zone);
4987  MachineOperatorBuilder machine(
4988  &zone, MachineType::PointerRepresentation(),
4989  InstructionSelector::SupportedMachineOperatorFlags(),
4990  InstructionSelector::AlignmentRequirements());
4991  JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
4992 
4993  Node* control = nullptr;
4994  Node* effect = nullptr;
4995 
4996  WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
4997  StubCallMode::kCallOnHeapBuiltin);
4998  builder.set_control_ptr(&control);
4999  builder.set_effect_ptr(&effect);
5000  builder.BuildJSToWasmWrapper(is_import);
5001 
5002  //----------------------------------------------------------------------------
5003  // Run the compilation pipeline.
5004  //----------------------------------------------------------------------------
5005  static constexpr size_t kMaxNameLen = 128;
5006  char debug_name[kMaxNameLen] = "js_to_wasm:";
5007  AppendSignature(debug_name, kMaxNameLen, sig);
5008 
5009  // Schedule and compile to machine code.
5010  int params = static_cast<int>(sig->parameter_count());
5011  CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5012  &zone, false, params + 1, CallDescriptor::kNoFlags);
5013 
5014  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5015  isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
5016  WasmAssemblerOptions());
5017  Handle<Code> code;
5018  if (!maybe_code.ToHandle(&code)) {
5019  return maybe_code;
5020  }
5021 #ifdef ENABLE_DISASSEMBLER
5022  if (FLAG_print_opt_code) {
5023  CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5024  OFStream os(tracing_scope.file());
5025  code->Disassemble(debug_name, os);
5026  }
5027 #endif
5028 
5029  if (must_record_function_compilation(isolate)) {
5030  RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
5031  debug_name);
5032  }
5033 
5034  return code;
5035 }
5036 
5037 WasmImportCallKind GetWasmImportCallKind(Handle<JSReceiver> target,
5038  wasm::FunctionSig* expected_sig) {
5039  if (WasmExportedFunction::IsWasmExportedFunction(*target)) {
5040  auto imported_function = WasmExportedFunction::cast(*target);
5041  wasm::FunctionSig* imported_sig =
5042  imported_function->instance()
5043  ->module()
5044  ->functions[imported_function->function_index()]
5045  .sig;
5046  if (*imported_sig != *expected_sig) {
5047  return WasmImportCallKind::kLinkError;
5048  }
5049  return WasmImportCallKind::kWasmToWasm;
5050  }
5051  // Assuming we are calling to JS, check whether this would be a runtime error.
5052  if (!wasm::IsJSCompatibleSignature(expected_sig)) {
5053  return WasmImportCallKind::kRuntimeTypeError;
5054  }
5055  // For JavaScript calls, determine whether the target has an arity match
5056  // and whether it has a sloppy receiver.
5057  if (target->IsJSFunction()) {
5058  Handle<JSFunction> function = Handle<JSFunction>::cast(target);
5059  SharedFunctionInfo* shared = function->shared();
5060 
5061 // Check for math intrinsics.
5062 #define COMPARE_SIG_FOR_BUILTIN(name) \
5063  { \
5064  wasm::FunctionSig* sig = wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
5065  if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name); \
5066  DCHECK_NOT_NULL(sig); \
5067  if (*expected_sig == *sig) return WasmImportCallKind::k##name; \
5068  }
5069 #define COMPARE_SIG_FOR_BUILTIN_F64(name) \
5070  case Builtins::kMath##name: \
5071  COMPARE_SIG_FOR_BUILTIN(F64##name); \
5072  break;
5073 #define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
5074  case Builtins::kMath##name: \
5075  COMPARE_SIG_FOR_BUILTIN(F64##name); \
5076  COMPARE_SIG_FOR_BUILTIN(F32##name); \
5077  break;
5078 
5079  if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
5080  switch (shared->builtin_id()) {
5081  COMPARE_SIG_FOR_BUILTIN_F64(Acos);
5082  COMPARE_SIG_FOR_BUILTIN_F64(Asin);
5083  COMPARE_SIG_FOR_BUILTIN_F64(Atan);
5084  COMPARE_SIG_FOR_BUILTIN_F64(Cos);
5085  COMPARE_SIG_FOR_BUILTIN_F64(Sin);
5086  COMPARE_SIG_FOR_BUILTIN_F64(Tan);
5087  COMPARE_SIG_FOR_BUILTIN_F64(Exp);
5088  COMPARE_SIG_FOR_BUILTIN_F64(Log);
5089  COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
5090  //===========================================================
5091  // TODO(8505): Math.pow for wasm does not match JS.
5092  // COMPARE_SIG_FOR_BUILTIN_F64(Pow);
5093  //===========================================================
5094  COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
5095  COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
5096  COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
5097  COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
5098  COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
5099  COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
5100  case Builtins::kMathFround:
5101  COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
5102  break;
5103  default:
5104  break;
5105  }
5106  }
5107 
5108 #undef COMPARE_SIG_FOR_BUILTIN
5109 #undef COMPARE_SIG_FOR_BUILTIN_F64
5110 #undef COMPARE_SIG_FOR_BUILTIN_F32_F64
5111 
5112  if (IsClassConstructor(shared->kind())) {
5113  // Class constructor will throw anyway.
5114  return WasmImportCallKind::kUseCallBuiltin;
5115  }
5116  bool sloppy = is_sloppy(shared->language_mode()) && !shared->native();
5117  if (shared->internal_formal_parameter_count() ==
5118  expected_sig->parameter_count()) {
5119  return sloppy ? WasmImportCallKind::kJSFunctionArityMatchSloppy
5120  : WasmImportCallKind::kJSFunctionArityMatch;
5121  }
5122  return sloppy ? WasmImportCallKind::kJSFunctionArityMismatchSloppy
5123  : WasmImportCallKind::kJSFunctionArityMismatch;
5124  }
5125  // Unknown case. Use the call builtin.
5126  return WasmImportCallKind::kUseCallBuiltin;
5127 }
5128 
5129 wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
5130  const char** name_ptr) {
5131 #define CASE(name) \
5132  case WasmImportCallKind::k##name: \
5133  *name_ptr = "WasmMathIntrinsic:" #name; \
5134  return wasm::kExpr##name
5135  switch (kind) {
5136  CASE(F64Acos);
5137  CASE(F64Asin);
5138  CASE(F64Atan);
5139  CASE(F64Cos);
5140  CASE(F64Sin);
5141  CASE(F64Tan);
5142  CASE(F64Exp);
5143  CASE(F64Log);
5144  CASE(F64Atan2);
5145  CASE(F64Pow);
5146  CASE(F64Ceil);
5147  CASE(F64Floor);
5148  CASE(F64Sqrt);
5149  CASE(F64Min);
5150  CASE(F64Max);
5151  CASE(F64Abs);
5152  CASE(F32Min);
5153  CASE(F32Max);
5154  CASE(F32Abs);
5155  CASE(F32Ceil);
5156  CASE(F32Floor);
5157  CASE(F32Sqrt);
5158  CASE(F32ConvertF64);
5159  default:
5160  UNREACHABLE();
5161  return wasm::kExprUnreachable;
5162  }
5163 #undef CASE
5164 }
5165 
5166 wasm::WasmCode* CompileWasmMathIntrinsic(Isolate* isolate,
5167  wasm::NativeModule* native_module,
5168  WasmImportCallKind kind,
5169  wasm::FunctionSig* sig) {
5170  DCHECK_EQ(1, sig->return_count());
5171 
5172  TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5173  "CompileWasmMathIntrinsic");
5174 
5175  Zone zone(isolate->allocator(), ZONE_NAME);
5176 
5177  // Compile a WASM function with a single bytecode and let TurboFan
5178  // generate either inlined machine code or a call to a helper.
5179  SourcePositionTable* source_positions = nullptr;
5180  MachineGraph* mcgraph = new (&zone) MachineGraph(
5181  new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5182  new (&zone) MachineOperatorBuilder(
5183  &zone, MachineType::PointerRepresentation(),
5184  InstructionSelector::SupportedMachineOperatorFlags(),
5185  InstructionSelector::AlignmentRequirements()));
5186 
5187  wasm::CompilationEnv env(
5188  native_module->module(), wasm::UseTrapHandler::kNoTrapHandler,
5189  wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
5190  wasm::LowerSimd::kNoLowerSimd);
5191 
5192  WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
5193  source_positions);
5194 
5195  // Set up the graph start.
5196  Node* start = builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
5197  Node* effect = start;
5198  Node* control = start;
5199  builder.set_effect_ptr(&effect);
5200  builder.set_control_ptr(&control);
5201  builder.set_instance_node(builder.Param(wasm::kWasmInstanceParameterIndex));
5202 
5203  // Generate either a unop or a binop.
5204  Node* result = nullptr;
5205  const char* debug_name = "WasmMathIntrinsic";
5206  auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
5207  switch (sig->parameter_count()) {
5208  case 1:
5209  result = builder.Unop(opcode, builder.Param(1));
5210  break;
5211  case 2:
5212  result = builder.Binop(opcode, builder.Param(1), builder.Param(2));
5213  break;
5214  default:
5215  UNREACHABLE();
5216  break;
5217  }
5218 
5219  builder.Return(result);
5220 
5221  // Run the compiler pipeline to generate machine code.
5222  auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
5223  if (mcgraph->machine()->Is32()) {
5224  call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5225  }
5226 
5227  wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5228  isolate->wasm_engine(), call_descriptor, mcgraph, Code::WASM_FUNCTION,
5229  wasm::WasmCode::kFunction, debug_name, AssemblerOptions::Default(isolate),
5230  native_module, source_positions);
5231  CHECK_NOT_NULL(wasm_code);
5232  // TODO(titzer): add counters for math intrinsic code size / allocation
5233 
5234  return wasm_code;
5235 }
5236 
5237 wasm::WasmCode* CompileWasmImportCallWrapper(Isolate* isolate,
5238  wasm::NativeModule* native_module,
5239  WasmImportCallKind kind,
5240  wasm::FunctionSig* sig,
5241  bool source_positions) {
5242  DCHECK_NE(WasmImportCallKind::kLinkError, kind);
5243  DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind);
5244 
5245  // Check for math intrinsics first.
5246  if (FLAG_wasm_math_intrinsics &&
5247  kind >= WasmImportCallKind::kFirstMathIntrinsic &&
5248  kind <= WasmImportCallKind::kLastMathIntrinsic) {
5249  return CompileWasmMathIntrinsic(isolate, native_module, kind, sig);
5250  }
5251 
5252  TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5253  "CompileWasmImportCallWrapper");
5254  //----------------------------------------------------------------------------
5255  // Create the Graph
5256  //----------------------------------------------------------------------------
5257  Zone zone(isolate->allocator(), ZONE_NAME);
5258  Graph graph(&zone);
5259  CommonOperatorBuilder common(&zone);
5260  MachineOperatorBuilder machine(
5261  &zone, MachineType::PointerRepresentation(),
5262  InstructionSelector::SupportedMachineOperatorFlags(),
5263  InstructionSelector::AlignmentRequirements());
5264  JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5265 
5266  Node* control = nullptr;
5267  Node* effect = nullptr;
5268 
5269  SourcePositionTable* source_position_table =
5270  source_positions ? new (&zone) SourcePositionTable(&graph) : nullptr;
5271 
5272  WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, source_position_table,
5273  StubCallMode::kCallWasmRuntimeStub);
5274  builder.set_control_ptr(&control);
5275  builder.set_effect_ptr(&effect);
5276  builder.BuildWasmImportCallWrapper(kind);
5277 
5278  const char* func_name = "wasm-to-js";
5279 
5280  // Schedule and compile to machine code.
5281  CallDescriptor* incoming =
5282  GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline,
5283  WasmGraphBuilder::kExtraCallableParam);
5284  if (machine.Is32()) {
5285  incoming = GetI32WasmCallDescriptor(&zone, incoming);
5286  }
5287  wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5288  isolate->wasm_engine(), incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
5289  wasm::WasmCode::kWasmToJsWrapper, func_name,
5290  AssemblerOptions::Default(isolate), native_module, source_position_table);
5291  CHECK_NOT_NULL(wasm_code);
5292 
5293  return wasm_code;
5294 }
5295 
5296 wasm::WasmCode* CompileWasmInterpreterEntry(Isolate* isolate,
5297  wasm::NativeModule* native_module,
5298  uint32_t func_index,
5299  wasm::FunctionSig* sig) {
5300  //----------------------------------------------------------------------------
5301  // Create the Graph
5302  //----------------------------------------------------------------------------
5303  Zone zone(isolate->allocator(), ZONE_NAME);
5304  Graph graph(&zone);
5305  CommonOperatorBuilder common(&zone);
5306  MachineOperatorBuilder machine(
5307  &zone, MachineType::PointerRepresentation(),
5308  InstructionSelector::SupportedMachineOperatorFlags(),
5309  InstructionSelector::AlignmentRequirements());
5310  JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5311 
5312  Node* control = nullptr;
5313  Node* effect = nullptr;
5314 
5315  WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5316  StubCallMode::kCallWasmRuntimeStub);
5317  builder.set_control_ptr(&control);
5318  builder.set_effect_ptr(&effect);
5319  builder.BuildWasmInterpreterEntry(func_index);
5320 
5321  // Schedule and compile to machine code.
5322  CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
5323  if (machine.Is32()) {
5324  incoming = GetI32WasmCallDescriptor(&zone, incoming);
5325  }
5326 
5327  EmbeddedVector<char, 32> func_name;
5328  func_name.Truncate(
5329  SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
5330 
5331  wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5332  isolate->wasm_engine(), incoming, &jsgraph, Code::WASM_INTERPRETER_ENTRY,
5333  wasm::WasmCode::kInterpreterEntry, func_name.start(),
5334  AssemblerOptions::Default(isolate), native_module);
5335  CHECK_NOT_NULL(wasm_code);
5336 
5337  return wasm_code;
5338 }
5339 
5340 MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
5341  Zone zone(isolate->allocator(), ZONE_NAME);
5342  Graph graph(&zone);
5343  CommonOperatorBuilder common(&zone);
5344  MachineOperatorBuilder machine(
5345  &zone, MachineType::PointerRepresentation(),
5346  InstructionSelector::SupportedMachineOperatorFlags(),
5347  InstructionSelector::AlignmentRequirements());
5348  JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5349 
5350  Node* control = nullptr;
5351  Node* effect = nullptr;
5352 
5353  WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5354  StubCallMode::kCallOnHeapBuiltin);
5355  builder.set_control_ptr(&control);
5356  builder.set_effect_ptr(&effect);
5357  builder.BuildCWasmEntry();
5358 
5359  // Schedule and compile to machine code.
5360  CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5361  &zone, false, CWasmEntryParameters::kNumParameters + 1,
5362  CallDescriptor::kNoFlags);
5363 
5364  // Build a name in the form "c-wasm-entry:<params>:<returns>".
5365  static constexpr size_t kMaxNameLen = 128;
5366  char debug_name[kMaxNameLen] = "c-wasm-entry:";
5367  AppendSignature(debug_name, kMaxNameLen, sig);
5368 
5369  MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5370  isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
5371  AssemblerOptions::Default(isolate));
5372  Handle<Code> code;
5373  if (!maybe_code.ToHandle(&code)) {
5374  return maybe_code;
5375  }
5376 #ifdef ENABLE_DISASSEMBLER
5377  if (FLAG_print_opt_code) {
5378  CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5379  OFStream os(tracing_scope.file());
5380  code->Disassemble(debug_name, os);
5381  }
5382 #endif
5383 
5384  return code;
5385 }
5386 
5387 TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
5388  wasm::WasmCompilationUnit* wasm_unit)
5389  : wasm_unit_(wasm_unit) {}
5390 
5391 // Clears unique_ptrs, but (part of) the type is forward declared in the header.
5392 TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5393 
5394 bool TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5395  wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
5396  wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph,
5397  NodeOriginTable* node_origins, SourcePositionTable* source_positions) {
5398  base::ElapsedTimer decode_timer;
5399  if (FLAG_trace_wasm_decode_time) {
5400  decode_timer.Start();
5401  }
5402 
5403  // Create a TF graph during decoding.
5404  WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
5405  source_positions);
5406  wasm::VoidResult graph_construction_result = wasm::BuildTFGraph(
5407  wasm_unit_->wasm_engine_->allocator(),
5408  wasm_unit_->native_module_->enabled_features(), env->module, &builder,
5409  detected, func_body, node_origins);
5410  if (graph_construction_result.failed()) {
5411  if (FLAG_trace_wasm_compiler) {
5412  StdoutStream{} << "Compilation failed: "
5413  << graph_construction_result.error_msg() << std::endl;
5414  }
5415  wasm_unit_->native_module()->compilation_state()->SetError(
5416  wasm_unit_->func_index_, std::move(graph_construction_result));
5417  return false;
5418  }
5419 
5420  builder.LowerInt64();
5421 
5422  if (builder.has_simd() &&
5423  (!CpuFeatures::SupportsWasmSimd128() || env->lower_simd)) {
5424  SimdScalarLowering(mcgraph,
5425  CreateMachineSignature(mcgraph->zone(), func_body.sig))
5426  .LowerGraph();
5427  }
5428 
5429  if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5430  wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5431  PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(), func_body,
5432  env->module, wasm::kPrintLocals);
5433  }
5434  if (FLAG_trace_wasm_decode_time) {
5435  *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5436  }
5437  return true;
5438 }
5439 
5440 namespace {
5441 Vector<const char> GetDebugName(Zone* zone, int index) {
5442  // TODO(herhut): Use name from module if available.
5443  constexpr int kBufferLength = 24;
5444 
5445  EmbeddedVector<char, kBufferLength> name_vector;
5446  int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
5447  DCHECK(name_len > 0 && name_len < name_vector.length());
5448 
5449  char* index_name = zone->NewArray<char>(name_len);
5450  memcpy(index_name, name_vector.start(), name_len);
5451  return Vector<const char>(index_name, name_len);
5452 }
5453 } // namespace
5454 
5455 void TurbofanWasmCompilationUnit::ExecuteCompilation(
5456  wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
5457  Counters* counters, wasm::WasmFeatures* detected) {
5458  TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5459  "ExecuteTurbofanCompilation");
5460  double decode_ms = 0;
5461  size_t node_count = 0;
5462 
5463  Zone zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5464  MachineGraph* mcgraph = new (&zone) MachineGraph(
5465  new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5466  new (&zone) MachineOperatorBuilder(
5467  &zone, MachineType::PointerRepresentation(),
5468  InstructionSelector::SupportedMachineOperatorFlags(),
5469  InstructionSelector::AlignmentRequirements()));
5470 
5471  OptimizedCompilationInfo info(GetDebugName(&zone, wasm_unit_->func_index_),
5472  &zone, Code::WASM_FUNCTION);
5473  if (env->runtime_exception_support) {
5474  info.SetWasmRuntimeExceptionSupport();
5475  }
5476 
5477  if (info.trace_turbo_json_enabled()) {
5478  TurboCfgFile tcf;
5479  tcf << AsC1VCompilation(&info);
5480  }
5481 
5482  NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
5483  ? new (&zone)
5484  NodeOriginTable(mcgraph->graph())
5485  : nullptr;
5486  SourcePositionTable* source_positions =
5487  new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
5488  if (!BuildGraphForWasmFunction(env, func_body, detected, &decode_ms, mcgraph,
5489  node_origins, source_positions)) {
5490  // Compilation failed.
5491  return;
5492  }
5493 
5494  if (node_origins) {
5495  node_origins->AddDecorator();
5496  }
5497 
5498  base::ElapsedTimer pipeline_timer;
5499  if (FLAG_trace_wasm_decode_time) {
5500  node_count = mcgraph->graph()->NodeCount();
5501  pipeline_timer.Start();
5502  }
5503 
5504  // Run the compiler pipeline to generate machine code.
5505  auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
5506  if (mcgraph->machine()->Is32()) {
5507  call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5508  }
5509 
5510  if (wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmFunction(
5511  &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
5512  source_positions, node_origins, func_body, wasm_unit_->native_module_,
5513  wasm_unit_->func_index_)) {
5514  wasm_unit_->SetResult(wasm_code, counters);
5515  }
5516  if (FLAG_trace_wasm_decode_time) {
5517  double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
5518  PrintF(
5519  "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
5520  "%0.3f ms pipeline\n",
5521  static_cast<unsigned>(func_body.end - func_body.start), decode_ms,
5522  node_count, pipeline_ms);
5523  }
5524  // TODO(bradnelson): Improve histogram handling of size_t.
5525  counters->wasm_compile_function_peak_memory_bytes()->AddSample(
5526  static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
5527 }
5528 
5529 namespace {
5530 // Helper for allocating either an GP or FP reg, or the next stack slot.
5531 class LinkageLocationAllocator {
5532  public:
5533  template <size_t kNumGpRegs, size_t kNumFpRegs>
5534  constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
5535  const DoubleRegister (&fp)[kNumFpRegs])
5536  : allocator_(wasm::LinkageAllocator(gp, fp)) {}
5537 
5538  LinkageLocation Next(MachineRepresentation rep) {
5539  MachineType type = MachineType::TypeForRepresentation(rep);
5540  if (IsFloatingPoint(rep)) {
5541  if (allocator_.CanAllocateFP(rep)) {
5542  int reg_code = allocator_.NextFpReg(rep);
5543  return LinkageLocation::ForRegister(reg_code, type);
5544  }
5545  } else if (allocator_.CanAllocateGP()) {
5546  int reg_code = allocator_.NextGpReg();
5547  return LinkageLocation::ForRegister(reg_code, type);
5548  }
5549  // Cannot use register; use stack slot.
5550  int index = -1 - allocator_.NextStackSlot(rep);
5551  return LinkageLocation::ForCallerFrameSlot(index, type);
5552  }
5553 
5554  void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
5555  int NumStackSlots() const { return allocator_.NumStackSlots(); }
5556 
5557  private:
5558  wasm::LinkageAllocator allocator_;
5559 };
5560 } // namespace
5561 
5562 // General code uses the above configuration data.
5563 CallDescriptor* GetWasmCallDescriptor(
5564  Zone* zone, wasm::FunctionSig* fsig,
5565  WasmGraphBuilder::UseRetpoline use_retpoline,
5566  WasmGraphBuilder::ExtraCallableParam extra_callable_param) {
5567  // The extra here is to accomodate the instance object as first parameter
5568  // and, in the case of an import wrapper, the additional callable.
5569  int extra_params = extra_callable_param ? 2 : 1;
5570  LocationSignature::Builder locations(zone, fsig->return_count(),
5571  fsig->parameter_count() + extra_params);
5572 
5573  // Add register and/or stack parameter(s).
5574  LinkageLocationAllocator params(wasm::kGpParamRegisters,
5575  wasm::kFpParamRegisters);
5576 
5577  // The instance object.
5578  locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
5579 
5580  const int parameter_count = static_cast<int>(fsig->parameter_count());
5581  for (int i = 0; i < parameter_count; i++) {
5582  MachineRepresentation param =
5583  wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
5584  auto l = params.Next(param);
5585  locations.AddParam(l);
5586  }
5587 
5588  // Import call wrappers have an additional (implicit) parameter, the callable.
5589  // For consistency with JS, we use the JSFunction register.
5590  if (extra_callable_param) {
5591  locations.AddParam(LinkageLocation::ForRegister(
5592  kJSFunctionRegister.code(), MachineType::TaggedPointer()));
5593  }
5594 
5595  // Add return location(s).
5596  LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5597  wasm::kFpReturnRegisters);
5598 
5599  int parameter_slots = params.NumStackSlots();
5600  if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
5601 
5602  rets.SetStackOffset(parameter_slots);
5603 
5604  const int return_count = static_cast<int>(locations.return_count_);
5605  for (int i = 0; i < return_count; i++) {
5606  MachineRepresentation ret =
5607  wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
5608  auto l = rets.Next(ret);
5609  locations.AddReturn(l);
5610  }
5611 
5612  const RegList kCalleeSaveRegisters = 0;
5613  const RegList kCalleeSaveFPRegisters = 0;
5614 
5615  // The target for wasm calls is always a code object.
5616  MachineType target_type = MachineType::Pointer();
5617  LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
5618 
5619  CallDescriptor::Kind kind = extra_callable_param
5620  ? CallDescriptor::kCallWasmImportWrapper
5621  : CallDescriptor::kCallWasmFunction;
5622 
5623  CallDescriptor::Flags flags =
5624  use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
5625  return new (zone) CallDescriptor( // --
5626  kind, // kind
5627  target_type, // target MachineType
5628  target_loc, // target location
5629  locations.Build(), // location_sig
5630  parameter_slots, // stack_parameter_count
5631  compiler::Operator::kNoProperties, // properties
5632  kCalleeSaveRegisters, // callee-saved registers
5633  kCalleeSaveFPRegisters, // callee-saved fp regs
5634  flags, // flags
5635  "wasm-call", // debug name
5636  0, // allocatable registers
5637  rets.NumStackSlots() - parameter_slots); // stack_return_count
5638 }
5639 
5640 namespace {
5641 CallDescriptor* ReplaceTypeInCallDescriptorWith(
5642  Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
5643  MachineType input_type, MachineRepresentation output_type) {
5644  size_t parameter_count = call_descriptor->ParameterCount();
5645  size_t return_count = call_descriptor->ReturnCount();
5646  for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5647  if (call_descriptor->GetParameterType(i) == input_type) {
5648  parameter_count += num_replacements - 1;
5649  }
5650  }
5651  for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5652  if (call_descriptor->GetReturnType(i) == input_type) {
5653  return_count += num_replacements - 1;
5654  }
5655  }
5656  if (parameter_count == call_descriptor->ParameterCount() &&
5657  return_count == call_descriptor->ReturnCount()) {
5658  return call_descriptor;
5659  }
5660 
5661  LocationSignature::Builder locations(zone, return_count, parameter_count);
5662 
5663  LinkageLocationAllocator params(wasm::kGpParamRegisters,
5664  wasm::kFpParamRegisters);
5665  for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
5666  if (call_descriptor->GetParameterType(i) == input_type) {
5667  for (size_t j = 0; j < num_replacements; j++) {
5668  locations.AddParam(params.Next(output_type));
5669  }
5670  } else {
5671  locations.AddParam(
5672  params.Next(call_descriptor->GetParameterType(i).representation()));
5673  }
5674  }
5675 
5676  LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
5677  wasm::kFpReturnRegisters);
5678  rets.SetStackOffset(params.NumStackSlots());
5679  for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
5680  if (call_descriptor->GetReturnType(i) == input_type) {
5681  for (size_t j = 0; j < num_replacements; j++) {
5682  locations.AddReturn(rets.Next(output_type));
5683  }
5684  } else {
5685  locations.AddReturn(
5686  rets.Next(call_descriptor->GetReturnType(i).representation()));
5687  }
5688  }
5689 
5690  return new (zone) CallDescriptor( // --
5691  call_descriptor->kind(), // kind
5692  call_descriptor->GetInputType(0), // target MachineType
5693  call_descriptor->GetInputLocation(0), // target location
5694  locations.Build(), // location_sig
5695  params.NumStackSlots(), // stack_parameter_count
5696  call_descriptor->properties(), // properties
5697  call_descriptor->CalleeSavedRegisters(), // callee-saved registers
5698  call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
5699  call_descriptor->flags(), // flags
5700  call_descriptor->debug_name(), // debug name
5701  call_descriptor->AllocatableRegisters(), // allocatable registers
5702  rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
5703 }
5704 } // namespace
5705 
5706 CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
5707  CallDescriptor* call_descriptor) {
5708  return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
5709  MachineType::Int64(),
5710  MachineRepresentation::kWord32);
5711 }
5712 
5713 CallDescriptor* GetI32WasmCallDescriptorForSimd(
5714  Zone* zone, CallDescriptor* call_descriptor) {
5715  return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
5716  MachineType::Simd128(),
5717  MachineRepresentation::kWord32);
5718 }
5719 
5720 AssemblerOptions WasmAssemblerOptions() {
5721  AssemblerOptions options;
5722  options.record_reloc_info_for_serialization = true;
5723  options.enable_root_array_delta_access = false;
5724  return options;
5725 }
5726 
5727 #undef WASM_64
5728 #undef FATAL_UNSUPPORTED_OPCODE
5729 #undef WASM_INSTANCE_OBJECT_SIZE
5730 #undef WASM_INSTANCE_OBJECT_OFFSET
5731 #undef LOAD_RAW
5732 #undef LOAD_INSTANCE_FIELD
5733 #undef LOAD_TAGGED_POINTER
5734 #undef LOAD_TAGGED_ANY
5735 #undef LOAD_FIXED_ARRAY_SLOT
5736 #undef LOAD_FIXED_ARRAY_SLOT_SMI
5737 #undef LOAD_FIXED_ARRAY_SLOT_PTR
5738 #undef LOAD_FIXED_ARRAY_SLOT_ANY
5739 #undef STORE_FIXED_ARRAY_SLOT_SMI
5740 #undef STORE_FIXED_ARRAY_SLOT_ANY
5741 
5742 } // namespace compiler
5743 } // namespace internal
5744 } // namespace v8
int64_t
uintptr_t
v8
Definition: libplatform.h:13
uint32_t