embedded-data.cc

// Copyright 2018 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/snapshot/embedded-data.h"

#include "src/assembler-inl.h"
#include "src/callable.h"
#include "src/macro-assembler.h"
#include "src/objects-inl.h"
#include "src/snapshot/snapshot.h"

namespace v8 {
namespace internal {

// static
bool InstructionStream::PcIsOffHeap(Isolate* isolate, Address pc) {
  if (FLAG_embedded_builtins) {
    const Address start = reinterpret_cast<Address>(isolate->embedded_blob());
    return IsInRange(pc, start, start + isolate->embedded_blob_size());
  } else {
    return false;
  }
}

// static
Code InstructionStream::TryLookupCode(Isolate* isolate, Address address) {
  if (!PcIsOffHeap(isolate, address)) return Code();

  EmbeddedData d = EmbeddedData::FromBlob();
  if (address < d.InstructionStartOfBuiltin(0)) return Code();

  // Note: Addresses within the padding section between builtins (i.e. within
  // start + size <= address < start + padded_size) are interpreted as belonging
  // to the preceding builtin.

  int l = 0, r = Builtins::builtin_count;
  while (l < r) {
    const int mid = (l + r) / 2;
    Address start = d.InstructionStartOfBuiltin(mid);
    Address end = start + d.PaddedInstructionSizeOfBuiltin(mid);

    if (address < start) {
      r = mid;
    } else if (address >= end) {
      l = mid + 1;
    } else {
      return isolate->builtins()->builtin(mid);
    }
  }

  UNREACHABLE();
}

// static
void InstructionStream::CreateOffHeapInstructionStream(Isolate* isolate,
                                                       uint8_t** data,
                                                       uint32_t* size) {
  EmbeddedData d = EmbeddedData::FromIsolate(isolate);

  v8::PageAllocator* page_allocator = v8::internal::GetPlatformPageAllocator();
  const uint32_t page_size =
      static_cast<uint32_t>(page_allocator->AllocatePageSize());
  const uint32_t allocated_size = RoundUp(d.size(), page_size);

  uint8_t* allocated_bytes = static_cast<uint8_t*>(
      AllocatePages(page_allocator, isolate->heap()->GetRandomMmapAddr(),
                    allocated_size, page_size, PageAllocator::kReadWrite));
  CHECK_NOT_NULL(allocated_bytes);

  std::memcpy(allocated_bytes, d.data(), d.size());
  CHECK(SetPermissions(page_allocator, allocated_bytes, allocated_size,
                       PageAllocator::kReadExecute));

  *data = allocated_bytes;
  *size = d.size();

  d.Dispose();
}

// static
void InstructionStream::FreeOffHeapInstructionStream(uint8_t* data,
                                                     uint32_t size) {
  v8::PageAllocator* page_allocator = v8::internal::GetPlatformPageAllocator();
  const uint32_t page_size =
      static_cast<uint32_t>(page_allocator->AllocatePageSize());
  CHECK(FreePages(page_allocator, data, RoundUp(size, page_size)));
}

namespace {

bool BuiltinAliasesOffHeapTrampolineRegister(Isolate* isolate, Code code) {
  DCHECK(Builtins::IsIsolateIndependent(code->builtin_index()));
  switch (Builtins::KindOf(code->builtin_index())) {
    case Builtins::CPP:
    case Builtins::TFC:
    case Builtins::TFH:
    case Builtins::TFJ:
    case Builtins::TFS:
      break;

    // Bytecode handlers will only ever be used by the interpreter and so there
    // will never be a need to use trampolines with them.
    case Builtins::BCH:
    case Builtins::API:
    case Builtins::ASM:
      // TODO(jgruber): Extend checks to remaining kinds.
      return false;
  }

  Callable callable = Builtins::CallableFor(
      isolate, static_cast<Builtins::Name>(code->builtin_index()));
  CallInterfaceDescriptor descriptor = callable.descriptor();

  if (descriptor.ContextRegister() == kOffHeapTrampolineRegister) {
    return true;
  }

  for (int i = 0; i < descriptor.GetRegisterParameterCount(); i++) {
    Register reg = descriptor.GetRegisterParameter(i);
    if (reg == kOffHeapTrampolineRegister) return true;
  }

  return false;
}

void FinalizeEmbeddedCodeTargets(Isolate* isolate, EmbeddedData* blob) {
  static const int kRelocMask =
      RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
      RelocInfo::ModeMask(RelocInfo::RELATIVE_CODE_TARGET);

  for (int i = 0; i < Builtins::builtin_count; i++) {
    if (!Builtins::IsIsolateIndependent(i)) continue;

    Code code = isolate->builtins()->builtin(i);
    RelocIterator on_heap_it(code, kRelocMask);
    RelocIterator off_heap_it(blob, code, kRelocMask);

#if defined(V8_TARGET_ARCH_X64) || defined(V8_TARGET_ARCH_ARM64) || \
    defined(V8_TARGET_ARCH_ARM) || defined(V8_TARGET_ARCH_MIPS) ||  \
    defined(V8_TARGET_ARCH_IA32)
    // On these platforms we emit relative builtin-to-builtin
    // jumps for isolate independent builtins in the snapshot. This fixes up the
    // relative jumps to the right offsets in the snapshot.
    // See also: Code::IsIsolateIndependent.
    while (!on_heap_it.done()) {
      DCHECK(!off_heap_it.done());

      RelocInfo* rinfo = on_heap_it.rinfo();
      DCHECK_EQ(rinfo->rmode(), off_heap_it.rinfo()->rmode());
      Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
      CHECK(Builtins::IsIsolateIndependentBuiltin(target));

      // Do not emit write-barrier for off-heap writes.
      off_heap_it.rinfo()->set_target_address(
          blob->InstructionStartOfBuiltin(target->builtin_index()),
          SKIP_WRITE_BARRIER);

      on_heap_it.next();
      off_heap_it.next();
    }
    DCHECK(off_heap_it.done());
#else
    // Architectures other than x64 and arm/arm64 do not use pc-relative calls
    // and thus must not contain embedded code targets. Instead, we use an
    // indirection through the root register.
    CHECK(on_heap_it.done());
    CHECK(off_heap_it.done());
#endif  // defined(V8_TARGET_ARCH_X64) || defined(V8_TARGET_ARCH_ARM64)
  }
}

}  // namespace

// static
EmbeddedData EmbeddedData::FromIsolate(Isolate* isolate) {
  Builtins* builtins = isolate->builtins();

  // Store instruction stream lengths and offsets.
  std::vector<struct Metadata> metadata(kTableSize);

  bool saw_unsafe_builtin = false;
  uint32_t raw_data_size = 0;
  for (int i = 0; i < Builtins::builtin_count; i++) {
    Code code = builtins->builtin(i);

    if (Builtins::IsIsolateIndependent(i)) {
      // Sanity-check that the given builtin is isolate-independent and does not
      // use the trampoline register in its calling convention.
      if (!code->IsIsolateIndependent(isolate)) {
        saw_unsafe_builtin = true;
        fprintf(stderr, "%s is not isolate-independent.\n", Builtins::name(i));
      }
      if (Builtins::IsWasmRuntimeStub(i) &&
          RelocInfo::RequiresRelocation(code)) {
        // Wasm additionally requires that its runtime stubs must be
        // individually PIC (i.e. we must be able to copy each stub outside the
        // embedded area without relocations). In particular, that means
        // pc-relative calls to other builtins are disallowed.
        saw_unsafe_builtin = true;
        fprintf(stderr, "%s is a wasm runtime stub but needs relocation.\n",
                Builtins::name(i));
      }
      if (BuiltinAliasesOffHeapTrampolineRegister(isolate, code)) {
        saw_unsafe_builtin = true;
        fprintf(stderr, "%s aliases the off-heap trampoline register.\n",
                Builtins::name(i));
      }

      uint32_t length = static_cast<uint32_t>(code->raw_instruction_size());

      DCHECK_EQ(0, raw_data_size % kCodeAlignment);
      metadata[i].instructions_offset = raw_data_size;
      metadata[i].instructions_length = length;

      // Align the start of each instruction stream.
      raw_data_size += PadAndAlign(length);
    } else {
      metadata[i].instructions_offset = raw_data_size;
    }
  }
  CHECK_WITH_MSG(
      !saw_unsafe_builtin,
      "One or more builtins marked as isolate-independent either contains "
      "isolate-dependent code or aliases the off-heap trampoline register. "
      "If in doubt, ask jgruber@");

  const uint32_t blob_size = RawDataOffset() + raw_data_size;
  uint8_t* const blob = new uint8_t[blob_size];
  uint8_t* const raw_data_start = blob + RawDataOffset();

  // Initially zap the entire blob, effectively padding the alignment area
  // between two builtins with int3's (on x64/ia32).
  ZapCode(reinterpret_cast<Address>(blob), blob_size);

  // Write the metadata tables.
  DCHECK_EQ(MetadataSize(), sizeof(metadata[0]) * metadata.size());
  std::memcpy(blob + MetadataOffset(), metadata.data(), MetadataSize());

  // Write the raw data section.
  for (int i = 0; i < Builtins::builtin_count; i++) {
    if (!Builtins::IsIsolateIndependent(i)) continue;
    Code code = builtins->builtin(i);
    uint32_t offset = metadata[i].instructions_offset;
    uint8_t* dst = raw_data_start + offset;
    DCHECK_LE(RawDataOffset() + offset + code->raw_instruction_size(),
              blob_size);
    std::memcpy(dst, reinterpret_cast<uint8_t*>(code->raw_instruction_start()),
                code->raw_instruction_size());
  }

  EmbeddedData d(blob, blob_size);

  // Fix up call targets that point to other embedded builtins.
  FinalizeEmbeddedCodeTargets(isolate, &d);

  // Hash the blob and store the result.
  STATIC_ASSERT(HashSize() == kSizetSize);
  const size_t hash = d.CreateHash();
  std::memcpy(blob + HashOffset(), &hash, HashSize());

  DCHECK_EQ(hash, d.CreateHash());
  DCHECK_EQ(hash, d.Hash());

  if (FLAG_serialization_statistics) d.PrintStatistics();

  return d;
}

Address EmbeddedData::InstructionStartOfBuiltin(int i) const {
  DCHECK(Builtins::IsBuiltinId(i));
  const struct Metadata* metadata = Metadata();
  const uint8_t* result = RawData() + metadata[i].instructions_offset;
  DCHECK_LE(result, data_ + size_);
  DCHECK_IMPLIES(result == data_ + size_, InstructionSizeOfBuiltin(i) == 0);
  return reinterpret_cast<Address>(result);
}

uint32_t EmbeddedData::InstructionSizeOfBuiltin(int i) const {
  DCHECK(Builtins::IsBuiltinId(i));
  const struct Metadata* metadata = Metadata();
  return metadata[i].instructions_length;
}

size_t EmbeddedData::CreateHash() const {
  STATIC_ASSERT(HashOffset() == 0);
  STATIC_ASSERT(HashSize() == kSizetSize);
  return base::hash_range(data_ + HashSize(), data_ + size_);
}

void EmbeddedData::PrintStatistics() const {
  DCHECK(FLAG_serialization_statistics);

  constexpr int kCount = Builtins::builtin_count;

  int embedded_count = 0;
  int instruction_size = 0;
  int sizes[kCount];
  for (int i = 0; i < kCount; i++) {
    if (!Builtins::IsIsolateIndependent(i)) continue;
    const int size = InstructionSizeOfBuiltin(i);
    instruction_size += size;
    sizes[embedded_count] = size;
    embedded_count++;
  }

  // Sort for percentiles.
  std::sort(&sizes[0], &sizes[embedded_count]);

  const int k50th = embedded_count * 0.5;
  const int k75th = embedded_count * 0.75;
  const int k90th = embedded_count * 0.90;
  const int k99th = embedded_count * 0.99;

  const int metadata_size = static_cast<int>(HashSize() + MetadataSize());

  PrintF("EmbeddedData:\n");
  PrintF("  Total size:                         %d\n",
         static_cast<int>(size()));
  PrintF("  Metadata size:                      %d\n", metadata_size);
  PrintF("  Instruction size:                   %d\n", instruction_size);
  PrintF("  Padding:                            %d\n",
         static_cast<int>(size() - metadata_size - instruction_size));
  PrintF("  Embedded builtin count:             %d\n", embedded_count);
  PrintF("  Instruction size (50th percentile): %d\n", sizes[k50th]);
  PrintF("  Instruction size (75th percentile): %d\n", sizes[k75th]);
  PrintF("  Instruction size (90th percentile): %d\n", sizes[k90th]);
  PrintF("  Instruction size (99th percentile): %d\n", sizes[k99th]);
  PrintF("\n");
}

}  // namespace internal
}  // namespace v8