8364212: Shenandoah: Rework archived objects loading

Reviewed-by: wkemper, kdnilsen

Author: shipilev

src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp

@@ -34,6 +34,7 @@ class ShenandoahAllocRequest : StackObj {
  enum Type {
  _alloc_shared, // Allocate common, outside of TLAB
  _alloc_shared_gc, // Allocate common, outside of GCLAB/PLAB
+ _alloc_cds, // Allocate for CDS
  _alloc_tlab, // Allocate TLAB
  _alloc_gclab, // Allocate GCLAB
  _alloc_plab, // Allocate PLAB
@@ -46,6 +47,8 @@ class ShenandoahAllocRequest : StackObj {
  return "Shared";
  case _alloc_shared_gc:
  return "Shared GC";
+ case _alloc_cds:
+ return "CDS";
  case _alloc_tlab:
  return "TLAB";
  case _alloc_gclab:
@@ -121,6 +124,10 @@ class ShenandoahAllocRequest : StackObj {
  return ShenandoahAllocRequest(0, requested_size, _alloc_shared, ShenandoahAffiliation::YOUNG_GENERATION);
  }
 
+ static inline ShenandoahAllocRequest for_cds(size_t requested_size) {
+ return ShenandoahAllocRequest(0, requested_size, _alloc_cds, ShenandoahAffiliation::YOUNG_GENERATION);
+ }
+
  inline size_t size() const {
  return _requested_size;
  }
@@ -163,6 +170,7 @@ class ShenandoahAllocRequest : StackObj {
  switch (_alloc_type) {
  case _alloc_tlab:
  case _alloc_shared:
+ case _alloc_cds:
  return true;
  case _alloc_gclab:
  case _alloc_plab:
@@ -178,6 +186,7 @@ class ShenandoahAllocRequest : StackObj {
  switch (_alloc_type) {
  case _alloc_tlab:
  case _alloc_shared:
+ case _alloc_cds:
  return false;
  case _alloc_gclab:
  case _alloc_plab:
@@ -197,6 +206,7 @@ class ShenandoahAllocRequest : StackObj {
  return true;
  case _alloc_shared:
  case _alloc_shared_gc:
+ case _alloc_cds:
  return false;
  default:
  ShouldNotReachHere();

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -816,6 +816,7 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool&
  switch (req.type()) {
  case ShenandoahAllocRequest::_alloc_tlab:
  case ShenandoahAllocRequest::_alloc_shared:
+ case ShenandoahAllocRequest::_alloc_cds:
  return allocate_for_mutator(req, in_new_region);
  case ShenandoahAllocRequest::_alloc_gclab:
  case ShenandoahAllocRequest::_alloc_plab:
@@ -1169,8 +1170,8 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
  return result;
 }
 
-HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
- assert(req.is_mutator_alloc(), "All humongous allocations are performed by mutator");
+HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req, bool is_humongous) {
+ assert(req.is_mutator_alloc(), "All contiguous allocations are performed by mutator");
  shenandoah_assert_heaplocked();
 
  size_t words_size = req.size();
@@ -1244,10 +1245,16 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
  assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous");
  assert(r->is_empty(), "Should be empty");
 
- if (i == beg) {
- r->make_humongous_start();
+ r->set_affiliation(req.affiliation());
+
+ if (is_humongous) {
+ if (i == beg) {
+ r->make_humongous_start();
+ } else {
+ r->make_humongous_cont();
+ }
  } else {
- r->make_humongous_cont();
+ r->make_regular_allocation(req.affiliation());
  }
 
  // Trailing region may be non-full, record the remainder there
@@ -1257,21 +1264,31 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) {
  } else {
  used_words = ShenandoahHeapRegion::region_size_words();
  }
-
- r->set_affiliation(req.affiliation());
  r->set_update_watermark(r->bottom());
  r->set_top(r->bottom() + used_words);
  }
  generation->increase_affiliated_region_count(num);
 
- // retire_range_from_partition() will adjust bounds on Mutator free set if appropriate
- _partitions.retire_range_from_partition(ShenandoahFreeSetPartitionId::Mutator, beg, end);
-
- size_t total_humongous_size = ShenandoahHeapRegion::region_size_bytes() * num;
- _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, total_humongous_size);
+ size_t total_used = 0;
+ if (is_humongous) {
+ // Humongous allocation retires all regions at once: no allocation is possible anymore.
+ _partitions.retire_range_from_partition(ShenandoahFreeSetPartitionId::Mutator, beg, end);
+ total_used = ShenandoahHeapRegion::region_size_bytes() * num;
+ } else {
+ // Non-humongous allocation retires only the regions that cannot be used for allocation anymore.
+ for (idx_t i = beg; i <= end; i++) {
+ ShenandoahHeapRegion* r = _heap->get_region(i);
+ if (r->free() < PLAB::min_size() * HeapWordSize) {
+ _partitions.retire_from_partition(ShenandoahFreeSetPartitionId::Mutator, i, r->used());
+ }
+ total_used += r->used();
+ }
+ }
+ _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, total_used);
  _partitions.assert_bounds();
+
  req.set_actual_size(words_size);
- if (remainder != 0) {
+ if (remainder != 0 && is_humongous) {
  req.set_waste(ShenandoahHeapRegion::region_size_words() - remainder);
  }
  return _heap->get_region(beg)->bottom();
@@ -2060,7 +2077,10 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_
  case ShenandoahAllocRequest::_alloc_shared:
  case ShenandoahAllocRequest::_alloc_shared_gc:
  in_new_region = true;
- return allocate_contiguous(req);
+ return allocate_contiguous(req, /* is_humongous = */ true);
+ case ShenandoahAllocRequest::_alloc_cds:
+ in_new_region = true;
+ return allocate_contiguous(req, /* is_humongous = */ false);
  case ShenandoahAllocRequest::_alloc_plab:
  case ShenandoahAllocRequest::_alloc_gclab:
  case ShenandoahAllocRequest::_alloc_tlab:

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -345,7 +345,7 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
  // object. No other objects are packed into these regions.
  //
  // Precondition: ShenandoahHeapRegion::requires_humongous(req.size())
- HeapWord* allocate_contiguous(ShenandoahAllocRequest& req);
+ HeapWord* allocate_contiguous(ShenandoahAllocRequest& req, bool is_humongous);
 
  // Change region r from the Mutator partition to the GC's Collector or OldCollector partition. This requires that the
  // region is entirely empty.

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -2773,41 +2773,15 @@ bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
 
 HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
 #if INCLUDE_CDS_JAVA_HEAP
- // CDS wants a continuous memory range to load a bunch of objects.
- // This effectively bypasses normal allocation paths, and requires
- // a bit of massaging to unbreak GC invariants.
-
- ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
-
- // Easy case: a single regular region, no further adjustments needed.
- if (!ShenandoahHeapRegion::requires_humongous(size)) {
- return allocate_memory(req);
- }
-
- // Hard case: the requested size would cause a humongous allocation.
- // We need to make sure it looks like regular allocation to the rest of GC.
-
- // CDS code would guarantee no objects straddle multiple regions, as long as
- // regions are as large as MIN_GC_REGION_ALIGNMENT. It is impractical at this
- // point to deal with case when Shenandoah runs with smaller regions.
- // TODO: This check can be dropped once MIN_GC_REGION_ALIGNMENT agrees more with Shenandoah.
- if (ShenandoahHeapRegion::region_size_bytes() < ArchiveHeapWriter::MIN_GC_REGION_ALIGNMENT) {
- return nullptr;
- }
-
- HeapWord* mem = allocate_memory(req);
- size_t start_idx = heap_region_index_containing(mem);
- size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
-
- // Flip humongous -> regular.
- {
- ShenandoahHeapLocker locker(lock(), false);
- for (size_t c = start_idx; c < start_idx + num_regions; c++) {
- get_region(c)->make_regular_bypass();
- }
- }
+ // CDS wants a raw continuous memory range to load a bunch of objects itself.
+ // This is an unusual request, since all requested regions should be regular, not humongous.
+ //
+ // CDS would guarantee no objects straddle multiple regions, as long as regions are as large
+ // as MIN_GC_REGION_ALIGNMENT.
+ guarantee(ShenandoahHeapRegion::region_size_bytes() >= ArchiveHeapWriter::MIN_GC_REGION_ALIGNMENT, "Must be");
 
- return mem;
+ ShenandoahAllocRequest req = ShenandoahAllocRequest::for_cds(size);
+ return allocate_memory(req);
 #else
  assert(false, "Archive heap loader should not be available, should not be here");
  return nullptr;
@@ -2834,17 +2808,25 @@ void ShenandoahHeap::complete_loaded_archive_space(MemRegion archive_space) {
  "Archive space should be fully used: " PTR_FORMAT " " PTR_FORMAT,
  p2i(cur), p2i(end));
 
- // Region bounds are good.
- ShenandoahHeapRegion* begin_reg = heap_region_containing(start);
- ShenandoahHeapRegion* end_reg = heap_region_containing(end);
- assert(begin_reg->is_regular(), "Must be");
- assert(end_reg->is_regular(), "Must be");
- assert(begin_reg->bottom() == start,
- "Must agree: archive-space-start: " PTR_FORMAT ", begin-region-bottom: " PTR_FORMAT,
- p2i(start), p2i(begin_reg->bottom()));
- assert(end_reg->top() == end,
- "Must agree: archive-space-end: " PTR_FORMAT ", end-region-top: " PTR_FORMAT,
- p2i(end), p2i(end_reg->top()));
+ // All regions in contiguous space have good state.
+ size_t begin_reg_idx = heap_region_index_containing(start);
+ size_t end_reg_idx = heap_region_index_containing(end);
+
+ for (size_t idx = begin_reg_idx; idx <= end_reg_idx; idx++) {
+ ShenandoahHeapRegion* r = get_region(idx);
+ assert(r->is_regular(), "Must be regular");
+ assert(r->is_young(), "Must be young");
+ assert(idx == end_reg_idx || r->top() == r->end(),
+ "All regions except the last one should be full: " PTR_FORMAT " " PTR_FORMAT,
+ p2i(r->top()), p2i(r->end()));
+ assert(idx != begin_reg_idx || r->bottom() == start,
+ "Archive space start should be at the bottom of first region: " PTR_FORMAT " " PTR_FORMAT,
+ p2i(r->bottom()), p2i(start));
+ assert(idx != end_reg_idx || r->top() == end,
+ "Archive space end should be at the top of last region: " PTR_FORMAT " " PTR_FORMAT,
+ p2i(r->top()), p2i(end));
+ }
+
 #endif
 }
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -142,27 +142,17 @@ void ShenandoahHeapRegion::make_affiliated_maybe() {
 
 void ShenandoahHeapRegion::make_regular_bypass() {
  shenandoah_assert_heaplocked();
- assert (!Universe::is_fully_initialized() ||
- ShenandoahHeap::heap()->is_full_gc_in_progress() ||
+ assert (ShenandoahHeap::heap()->is_full_gc_in_progress() ||
  ShenandoahHeap::heap()->is_degenerated_gc_in_progress(),
- "Only for STW GC or when Universe is initializing (CDS)");
+ "Only for STW GC");
  reset_age();
- auto cur_state = state();
- switch (cur_state) {
+ switch (state()) {
  case _empty_uncommitted:
  do_commit();
  case _empty_committed:
  case _cset:
  case _humongous_start:
  case _humongous_cont:
- if (cur_state == _humongous_start || cur_state == _humongous_cont) {
- // CDS allocates chunks of the heap to fill with regular objects. The allocator
- // will dutifully track any waste in the unused portion of the last region. Once
- // CDS has finished initializing the objects, it will convert these regions to
- // regular regions. The 'waste' in the last region is no longer wasted at this point,
- // so we must stop treating it as such.
- decrement_humongous_waste();
- }
  set_state(_regular);
  return;
  case _pinned_cset:

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -112,6 +112,7 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(ShenandoahAllocRequest::
  switch (type) {
  case ShenandoahAllocRequest::_alloc_shared:
  case ShenandoahAllocRequest::_alloc_shared_gc:
+ case ShenandoahAllocRequest::_alloc_cds:
  // Counted implicitly by tlab/gclab allocs
  break;
  case ShenandoahAllocRequest::_alloc_tlab: