5 #ifndef V8_HEAP_HEAP_INL_H_ 6 #define V8_HEAP_HEAP_INL_H_ 13 #include "src/heap/heap-write-barrier.h" 14 #include "src/heap/heap.h" 16 #include "src/base/atomic-utils.h" 17 #include "src/base/platform/platform.h" 18 #include "src/feedback-vector.h" 22 #include "src/heap/spaces-inl.h" 23 #include "src/isolate-data.h" 24 #include "src/isolate.h" 27 #include "src/objects-inl.h" 28 #include "src/objects/allocation-site-inl.h" 29 #include "src/objects/api-callbacks-inl.h" 30 #include "src/objects/descriptor-array.h" 31 #include "src/objects/literal-objects-inl.h" 32 #include "src/objects/scope-info.h" 33 #include "src/objects/script-inl.h" 34 #include "src/profiler/heap-profiler.h" 35 #include "src/string-hasher.h" 36 #include "src/zone/zone-list-inl.h" 42 #include "src/heap/heap-write-barrier-inl.h" 47 AllocationSpace AllocationResult::RetrySpace() {
49 return static_cast<AllocationSpace
>(Smi::ToInt(object_));
52 HeapObject* AllocationResult::ToObjectChecked() {
54 return HeapObject::cast(object_);
57 Isolate* Heap::isolate() {
58 return reinterpret_cast<Isolate*
>(
59 reinterpret_cast<intptr_t
>(
this) -
60 reinterpret_cast<size_t>(reinterpret_cast<Isolate*>(16)->heap()) + 16);
63 int64_t Heap::external_memory() {
64 return isolate()->isolate_data()->external_memory_;
67 void Heap::update_external_memory(
int64_t delta) {
68 isolate()->isolate_data()->external_memory_ += delta;
71 void Heap::update_external_memory_concurrently_freed(intptr_t freed) {
72 external_memory_concurrently_freed_ += freed;
75 void Heap::account_external_memory_concurrently_freed() {
76 isolate()->isolate_data()->external_memory_ -=
77 external_memory_concurrently_freed_;
78 external_memory_concurrently_freed_ = 0;
81 RootsTable& Heap::roots_table() {
return isolate()->roots_table(); }
84 #define ROOT_ACCESSOR(Type, name, CamelName) \ 86 return std::remove_pointer<Type>::type::cast( \ 87 roots_table()[RootIndex::k##CamelName]); \ 89 MUTABLE_ROOT_LIST(ROOT_ACCESSOR)
92 #define ROOT_ACCESSOR(type, name, CamelName) \ 93 void Heap::set_##name(type value) { \ 96 DCHECK_IMPLIES(deserialization_complete(), \ 97 !RootsTable::IsImmortalImmovable(RootIndex::k##CamelName)); \ 98 DCHECK_IMPLIES(RootsTable::IsImmortalImmovable(RootIndex::k##CamelName), \ 99 IsImmovable(HeapObject::cast(value))); \ 100 roots_table()[RootIndex::k##CamelName] = value; \ 102 ROOT_LIST(ROOT_ACCESSOR)
105 void Heap::SetRootCodeStubs(SimpleNumberDictionary value) {
106 roots_table()[RootIndex::kCodeStubs] = value;
109 void Heap::SetRootMaterializedObjects(FixedArray objects) {
110 roots_table()[RootIndex::kMaterializedObjects] = objects;
113 void Heap::SetRootScriptList(Object* value) {
114 roots_table()[RootIndex::kScriptList] = value;
117 void Heap::SetRootStringTable(StringTable value) {
118 roots_table()[RootIndex::kStringTable] = value;
121 void Heap::SetRootNoScriptSharedFunctionInfos(Object* value) {
122 roots_table()[RootIndex::kNoScriptSharedFunctionInfos] = value;
125 void Heap::SetMessageListeners(TemplateList value) {
126 roots_table()[RootIndex::kMessageListeners] = value;
129 PagedSpace* Heap::paged_space(
int idx) {
130 DCHECK_NE(idx, LO_SPACE);
131 DCHECK_NE(idx, NEW_SPACE);
132 DCHECK_NE(idx, CODE_LO_SPACE);
133 DCHECK_NE(idx, NEW_LO_SPACE);
134 return static_cast<PagedSpace*
>(space_[idx]);
137 Space* Heap::space(
int idx) {
return space_[idx]; }
139 Address* Heap::NewSpaceAllocationTopAddress() {
140 return new_space_->allocation_top_address();
143 Address* Heap::NewSpaceAllocationLimitAddress() {
144 return new_space_->allocation_limit_address();
147 Address* Heap::OldSpaceAllocationTopAddress() {
148 return old_space_->allocation_top_address();
151 Address* Heap::OldSpaceAllocationLimitAddress() {
152 return old_space_->allocation_limit_address();
155 void Heap::UpdateNewSpaceAllocationCounter() {
156 new_space_allocation_counter_ = NewSpaceAllocationCounter();
159 size_t Heap::NewSpaceAllocationCounter() {
160 return new_space_allocation_counter_ + new_space()->AllocatedSinceLastGC();
163 AllocationResult Heap::AllocateRaw(
int size_in_bytes, AllocationSpace space,
164 AllocationAlignment alignment) {
165 DCHECK(AllowHandleAllocation::IsAllowed());
166 DCHECK(AllowHeapAllocation::IsAllowed());
167 DCHECK(gc_state_ == NOT_IN_GC);
168 #ifdef V8_ENABLE_ALLOCATION_TIMEOUT 169 if (FLAG_random_gc_interval > 0 || FLAG_gc_interval >= 0) {
170 if (!always_allocate() && Heap::allocation_timeout_-- <= 0) {
171 return AllocationResult::Retry(space);
176 IncrementObjectCounters();
179 bool large_object = size_in_bytes > kMaxRegularHeapObjectSize;
181 HeapObject*
object =
nullptr;
182 AllocationResult allocation;
183 if (NEW_SPACE == space) {
186 space = FLAG_young_generation_large_objects ? NEW_LO_SPACE : LO_SPACE;
188 allocation = new_space_->AllocateRaw(size_in_bytes, alignment);
189 if (allocation.To(&
object)) {
190 OnAllocationEvent(
object, size_in_bytes);
197 if (OLD_SPACE == space) {
199 allocation = lo_space_->AllocateRaw(size_in_bytes);
201 allocation = old_space_->AllocateRaw(size_in_bytes, alignment);
203 }
else if (CODE_SPACE == space) {
204 if (size_in_bytes <= code_space()->AreaSize()) {
205 allocation = code_space_->AllocateRawUnaligned(size_in_bytes);
207 allocation = code_lo_space_->AllocateRaw(size_in_bytes);
209 }
else if (LO_SPACE == space) {
210 DCHECK(large_object);
211 allocation = lo_space_->AllocateRaw(size_in_bytes);
212 }
else if (NEW_LO_SPACE == space) {
213 DCHECK(FLAG_young_generation_large_objects);
214 allocation = new_lo_space_->AllocateRaw(size_in_bytes);
215 }
else if (CODE_LO_SPACE == space) {
216 allocation = code_lo_space_->AllocateRaw(size_in_bytes);
217 }
else if (MAP_SPACE == space) {
218 allocation = map_space_->AllocateRawUnaligned(size_in_bytes);
219 }
else if (RO_SPACE == space) {
220 #ifdef V8_USE_SNAPSHOT 221 DCHECK(isolate_->serializer_enabled());
223 DCHECK(!large_object);
224 DCHECK(CanAllocateInReadOnlySpace());
225 allocation = read_only_space_->AllocateRaw(size_in_bytes, alignment);
231 if (allocation.To(&
object)) {
232 if (space == CODE_SPACE) {
235 UnprotectAndRegisterMemoryChunk(
object);
236 ZapCodeObject(object->address(), size_in_bytes);
238 OnAllocationEvent(
object, size_in_bytes);
244 void Heap::OnAllocationEvent(HeapObject*
object,
int size_in_bytes) {
245 for (
auto& tracker : allocation_trackers_) {
246 tracker->AllocationEvent(object->address(), size_in_bytes);
249 if (FLAG_verify_predictable) {
250 ++allocations_count_;
252 MonotonicallyIncreasingTimeInMs();
254 UpdateAllocationsHash(
object);
255 UpdateAllocationsHash(size_in_bytes);
257 if (allocations_count_ % FLAG_dump_allocations_digest_at_alloc == 0) {
258 PrintAllocationsHash();
260 }
else if (FLAG_fuzzer_gc_analysis) {
261 ++allocations_count_;
262 }
else if (FLAG_trace_allocation_stack_interval > 0) {
263 ++allocations_count_;
264 if (allocations_count_ % FLAG_trace_allocation_stack_interval == 0) {
265 isolate()->PrintStack(stdout, Isolate::kPrintStackConcise);
271 void Heap::OnMoveEvent(HeapObject* target, HeapObject* source,
273 HeapProfiler* heap_profiler = isolate_->heap_profiler();
274 if (heap_profiler->is_tracking_object_moves()) {
275 heap_profiler->ObjectMoveEvent(source->address(), target->address(),
278 for (
auto& tracker : allocation_trackers_) {
279 tracker->MoveEvent(source->address(), target->address(), size_in_bytes);
281 if (target->IsSharedFunctionInfo()) {
282 LOG_CODE_EVENT(isolate_, SharedFunctionInfoMoveEvent(source->address(),
286 if (FLAG_verify_predictable) {
287 ++allocations_count_;
289 MonotonicallyIncreasingTimeInMs();
291 UpdateAllocationsHash(source);
292 UpdateAllocationsHash(target);
293 UpdateAllocationsHash(size_in_bytes);
295 if (allocations_count_ % FLAG_dump_allocations_digest_at_alloc == 0) {
296 PrintAllocationsHash();
298 }
else if (FLAG_fuzzer_gc_analysis) {
299 ++allocations_count_;
303 bool Heap::CanAllocateInReadOnlySpace() {
304 return !deserialization_complete_ &&
305 (isolate()->serializer_enabled() ||
306 !isolate()->initialized_from_snapshot());
309 void Heap::UpdateAllocationsHash(HeapObject*
object) {
310 Address object_address =
object->address();
311 MemoryChunk* memory_chunk = MemoryChunk::FromAddress(object_address);
312 AllocationSpace allocation_space = memory_chunk->owner()->identity();
314 STATIC_ASSERT(kSpaceTagSize + kPageSizeBits <= 32);
316 static_cast<uint32_t>(object_address - memory_chunk->address()) |
317 (static_cast<uint32_t>(allocation_space) << kPageSizeBits);
319 UpdateAllocationsHash(value);
323 void Heap::UpdateAllocationsHash(
uint32_t value) {
324 uint16_t c1 =
static_cast<uint16_t
>(value);
325 uint16_t c2 =
static_cast<uint16_t
>(value >> 16);
326 raw_allocations_hash_ =
327 StringHasher::AddCharacterCore(raw_allocations_hash_, c1);
328 raw_allocations_hash_ =
329 StringHasher::AddCharacterCore(raw_allocations_hash_, c2);
332 void Heap::RegisterExternalString(String
string) {
333 DCHECK(string->IsExternalString());
334 DCHECK(!string->IsThinString());
335 external_string_table_.AddString(
string);
338 void Heap::UpdateExternalString(String
string,
size_t old_payload,
339 size_t new_payload) {
340 DCHECK(string->IsExternalString());
341 Page* page = Page::FromHeapObject(
string);
343 if (old_payload > new_payload)
344 page->DecrementExternalBackingStoreBytes(
345 ExternalBackingStoreType::kExternalString, old_payload - new_payload);
347 page->IncrementExternalBackingStoreBytes(
348 ExternalBackingStoreType::kExternalString, new_payload - old_payload);
351 void Heap::FinalizeExternalString(String
string) {
352 DCHECK(string->IsExternalString());
353 Page* page = Page::FromHeapObject(
string);
354 ExternalString ext_string = ExternalString::cast(
string);
356 page->DecrementExternalBackingStoreBytes(
357 ExternalBackingStoreType::kExternalString,
358 ext_string->ExternalPayloadSize());
362 string->address() + ExternalString::kResourceOffset);
365 if (*resource_addr !=
nullptr) {
367 *resource_addr =
nullptr;
371 Address Heap::NewSpaceTop() {
return new_space_->top(); }
374 bool Heap::InNewSpace(Object*
object) {
375 DCHECK(!HasWeakHeapObjectTag(
object));
376 return object->IsHeapObject() && InNewSpace(HeapObject::cast(
object));
380 bool Heap::InNewSpace(MaybeObject
object) {
381 HeapObject* heap_object;
382 return object->GetHeapObject(&heap_object) && InNewSpace(heap_object);
386 bool Heap::InNewSpace(HeapObject* heap_object) {
388 bool result = MemoryChunk::FromHeapObject(heap_object)->InNewSpace();
394 Heap* heap = Heap::FromWritableHeapObject(heap_object);
395 DCHECK(heap->gc_state_ != NOT_IN_GC || InToSpace(heap_object));
402 bool Heap::InNewSpace(HeapObjectPtr heap_object) {
403 bool result = MemoryChunk::FromHeapObject(heap_object)->InNewSpace();
409 Heap* heap = Heap::FromWritableHeapObject(&heap_object);
410 DCHECK(heap->gc_state_ != NOT_IN_GC || InToSpace(heap_object));
417 bool Heap::InFromSpace(Object*
object) {
418 DCHECK(!HasWeakHeapObjectTag(
object));
419 return object->IsHeapObject() && InFromSpace(HeapObject::cast(
object));
423 bool Heap::InFromSpace(MaybeObject
object) {
424 HeapObject* heap_object;
425 return object->GetHeapObject(&heap_object) && InFromSpace(heap_object);
429 bool Heap::InFromSpace(HeapObject* heap_object) {
430 return MemoryChunk::FromHeapObject(heap_object)
431 ->IsFlagSet(Page::IN_FROM_SPACE);
435 bool Heap::InToSpace(Object*
object) {
436 DCHECK(!HasWeakHeapObjectTag(
object));
437 return object->IsHeapObject() && InToSpace(HeapObject::cast(
object));
441 bool Heap::InToSpace(MaybeObject
object) {
442 HeapObject* heap_object;
443 return object->GetHeapObject(&heap_object) && InToSpace(heap_object);
447 bool Heap::InToSpace(HeapObject* heap_object) {
448 return MemoryChunk::FromHeapObject(heap_object)->IsFlagSet(Page::IN_TO_SPACE);
452 bool Heap::InToSpace(HeapObjectPtr heap_object) {
453 return MemoryChunk::FromHeapObject(heap_object)->IsFlagSet(Page::IN_TO_SPACE);
456 bool Heap::InOldSpace(Object*
object) {
return old_space_->Contains(
object); }
458 bool Heap::InReadOnlySpace(Object*
object) {
459 return read_only_space_->Contains(
object);
463 Heap* Heap::FromWritableHeapObject(
const HeapObject* obj) {
464 MemoryChunk* chunk = MemoryChunk::FromHeapObject(obj);
468 SLOW_DCHECK(chunk->owner()->identity() != RO_SPACE ||
469 static_cast<ReadOnlySpace*
>(chunk->owner())->writable());
470 Heap* heap = chunk->heap();
471 SLOW_DCHECK(heap !=
nullptr);
476 Heap* Heap::FromWritableHeapObject(
const HeapObjectPtr* obj) {
477 MemoryChunk* chunk = MemoryChunk::FromHeapObject(*obj);
481 SLOW_DCHECK(chunk->owner()->identity() != RO_SPACE ||
482 static_cast<ReadOnlySpace*
>(chunk->owner())->writable());
483 Heap* heap = chunk->heap();
484 SLOW_DCHECK(heap !=
nullptr);
488 bool Heap::ShouldBePromoted(Address old_address) {
489 Page* page = Page::FromAddress(old_address);
490 Address age_mark = new_space_->age_mark();
491 return page->IsFlagSet(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK) &&
492 (!page->ContainsLimit(age_mark) || old_address < age_mark);
495 void Heap::CopyBlock(Address dst, Address src,
int byte_size) {
496 CopyWords(dst, src, static_cast<size_t>(byte_size / kPointerSize));
499 template <Heap::FindMementoMode mode>
500 AllocationMemento* Heap::FindAllocationMemento(Map map, HeapObject*
object) {
501 Address object_address =
object->address();
502 Address memento_address = object_address +
object->SizeFromMap(map);
503 Address last_memento_word_address = memento_address + kPointerSize;
505 if (!Page::OnSamePage(object_address, last_memento_word_address)) {
508 HeapObject* candidate = HeapObject::FromAddress(memento_address);
509 Map candidate_map = candidate->map();
513 MSAN_MEMORY_IS_INITIALIZED(&candidate_map,
sizeof(candidate_map));
514 if (candidate_map != ReadOnlyRoots(
this).allocation_memento_map()) {
520 Page* object_page = Page::FromAddress(object_address);
521 if (object_page->IsFlagSet(Page::NEW_SPACE_BELOW_AGE_MARK)) {
523 reinterpret_cast<SemiSpace*
>(object_page->owner())->age_mark();
524 if (!object_page->Contains(age_mark)) {
528 if (object_address < age_mark) {
533 AllocationMemento* memento_candidate = AllocationMemento::cast(candidate);
540 return memento_candidate;
541 case Heap::kForRuntime:
542 if (memento_candidate ==
nullptr)
return nullptr;
547 DCHECK(memento_address == top ||
548 memento_address + HeapObject::kHeaderSize <= top ||
549 !Page::OnSamePage(memento_address, top - 1));
550 if ((memento_address != top) && memento_candidate->IsValid()) {
551 return memento_candidate;
560 void Heap::UpdateAllocationSite(Map map, HeapObject*
object,
561 PretenuringFeedbackMap* pretenuring_feedback) {
562 DCHECK_NE(pretenuring_feedback, &global_pretenuring_feedback_);
564 InFromSpace(
object) ||
565 (InToSpace(
object) && Page::FromAddress(object->address())
566 ->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION)) ||
567 (!InNewSpace(
object) && Page::FromAddress(object->address())
568 ->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION)));
569 if (!FLAG_allocation_site_pretenuring ||
570 !AllocationSite::CanTrack(map->instance_type()))
572 AllocationMemento* memento_candidate =
573 FindAllocationMemento<kForGC>(map, object);
574 if (memento_candidate ==
nullptr)
return;
579 Address key = memento_candidate->GetAllocationSiteUnchecked();
580 (*pretenuring_feedback)[
reinterpret_cast<AllocationSite*
>(key)]++;
583 void Heap::ExternalStringTable::AddString(String
string) {
584 DCHECK(string->IsExternalString());
585 DCHECK(!Contains(
string));
587 if (InNewSpace(
string)) {
588 new_space_strings_.push_back(
string);
590 old_space_strings_.push_back(
string);
594 Oddball* Heap::ToBoolean(
bool condition) {
595 ReadOnlyRoots roots(
this);
596 return condition ? roots.true_value() : roots.false_value();
599 uint64_t Heap::HashSeed() {
601 ReadOnlyRoots(
this).hash_seed()->copy_out(0, reinterpret_cast<byte*>(&seed),
603 DCHECK(FLAG_randomize_hashes || seed == 0);
607 int Heap::NextScriptId() {
608 int last_id = last_script_id()->value();
609 if (last_id == Smi::kMaxValue) last_id = v8::UnboundScript::kNoScriptId;
611 set_last_script_id(Smi::FromInt(last_id));
615 int Heap::NextDebuggingId() {
616 int last_id = last_debugging_id()->value();
617 if (last_id == DebugInfo::DebuggingIdBits::kMax) {
618 last_id = DebugInfo::kNoDebuggingId;
621 set_last_debugging_id(Smi::FromInt(last_id));
625 int Heap::GetNextTemplateSerialNumber() {
626 int next_serial_number = next_template_serial_number()->value() + 1;
627 set_next_template_serial_number(Smi::FromInt(next_serial_number));
628 return next_serial_number;
631 int Heap::MaxNumberToStringCacheSize()
const {
635 size_t number_string_cache_size = max_semi_space_size_ / 512;
636 number_string_cache_size =
637 Max(static_cast<size_t>(kInitialNumberStringCacheSize * 2),
638 Min<size_t>(0x4000u, number_string_cache_size));
641 return static_cast<int>(number_string_cache_size * 2);
644 void Heap::IncrementExternalBackingStoreBytes(ExternalBackingStoreType type,
646 base::CheckedIncrement(&backing_store_bytes_, amount);
651 bool Heap::IsWithinLargeObject(Address address) {
652 if (new_lo_space()->FindPage(address) || lo_space()->FindPage(address) ||
653 code_lo_space()->FindPage(address))
658 void Heap::DecrementExternalBackingStoreBytes(ExternalBackingStoreType type,
660 base::CheckedDecrement(&backing_store_bytes_, amount);
663 AlwaysAllocateScope::AlwaysAllocateScope(Isolate* isolate)
664 : heap_(isolate->heap()) {
665 heap_->always_allocate_scope_count_++;
668 AlwaysAllocateScope::~AlwaysAllocateScope() {
669 heap_->always_allocate_scope_count_--;
672 CodeSpaceMemoryModificationScope::CodeSpaceMemoryModificationScope(Heap* heap)
674 if (heap_->write_protect_code_memory()) {
675 heap_->increment_code_space_memory_modification_scope_depth();
676 heap_->code_space()->SetReadAndWritable();
677 LargePage* page = heap_->code_lo_space()->first_page();
678 while (page !=
nullptr) {
679 DCHECK(page->IsFlagSet(MemoryChunk::IS_EXECUTABLE));
680 CHECK(heap_->memory_allocator()->IsMemoryChunkExecutable(page));
681 page->SetReadAndWritable();
682 page = page->next_page();
687 CodeSpaceMemoryModificationScope::~CodeSpaceMemoryModificationScope() {
688 if (heap_->write_protect_code_memory()) {
689 heap_->decrement_code_space_memory_modification_scope_depth();
690 heap_->code_space()->SetReadAndExecutable();
691 LargePage* page = heap_->code_lo_space()->first_page();
692 while (page !=
nullptr) {
693 DCHECK(page->IsFlagSet(MemoryChunk::IS_EXECUTABLE));
694 CHECK(heap_->memory_allocator()->IsMemoryChunkExecutable(page));
695 page->SetReadAndExecutable();
696 page = page->next_page();
701 CodePageCollectionMemoryModificationScope::
702 CodePageCollectionMemoryModificationScope(Heap* heap)
704 if (heap_->write_protect_code_memory() &&
705 !heap_->code_space_memory_modification_scope_depth()) {
706 heap_->EnableUnprotectedMemoryChunksRegistry();
710 CodePageCollectionMemoryModificationScope::
711 ~CodePageCollectionMemoryModificationScope() {
712 if (heap_->write_protect_code_memory() &&
713 !heap_->code_space_memory_modification_scope_depth()) {
714 heap_->ProtectUnprotectedMemoryChunks();
715 heap_->DisableUnprotectedMemoryChunksRegistry();
719 CodePageMemoryModificationScope::CodePageMemoryModificationScope(
722 scope_active_(chunk_->heap()->write_protect_code_memory() &&
723 chunk_->IsFlagSet(MemoryChunk::IS_EXECUTABLE)) {
725 DCHECK(chunk_->owner()->identity() == CODE_SPACE ||
726 (chunk_->owner()->identity() == CODE_LO_SPACE));
727 chunk_->SetReadAndWritable();
731 CodePageMemoryModificationScope::~CodePageMemoryModificationScope() {
733 chunk_->SetReadAndExecutable();
740 #endif // V8_HEAP_HEAP_INL_H_