[GR-71654] [GR-72101] [GR-72102] Remove the Java heap allocation in AbstractCollectionPolicy.updateSizeParameters() + fixes and optimizations.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java

@@ -106,6 +106,12 @@ public void update(UnmodifiableEconomicMap<OptionKey<?>, Object> values) {
                 }
             }
         } while (!v.compareAndSet(expect, newMap));
+
+        UnmodifiableMapCursor<OptionKey<?>, Object> cursor = values.getEntries();
+        while (cursor.advance()) {
+            OptionKey<?> key = cursor.getKey();
+            key.afterValueUpdate();
+        }
     }
 
     @Override

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java

@@ -29,8 +29,8 @@
 import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.Isolates;
-import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.GCCause;
+import com.oracle.svm.core.thread.VMOperation;
 import com.oracle.svm.core.util.BasedOnJDKFile;
 import com.oracle.svm.core.util.TimeUtils;
 import com.oracle.svm.core.util.Timer;
@@ -108,7 +108,8 @@ class AdaptiveCollectionPolicy extends AbstractCollectionPolicy {
     private static final int ADAPTIVE_SIZE_POLICY_INITIALIZING_STEPS = ADAPTIVE_SIZE_POLICY_READY_THRESHOLD;
     /**
      * The minimum estimated decrease in {@link #gcCost()} in percent to decide in favor of
-     * expanding a space by 1% of the combined size of {@link #edenSize} and {@link #promoSize}.
+     * expanding a space by 1% of the combined size of {@link SizeParameters#getEdenSize} and
+     * {@link SizeParameters#getPromoSize}.
      */
     private static final double ADAPTIVE_SIZE_ESTIMATOR_MIN_TOTAL_SIZE_COST_TRADEOFF = 0.5;
     /** The effective number of most recent data points used by estimator (exponential decay). */
@@ -212,7 +213,7 @@ public boolean shouldCollectCompletely(boolean followingIncrementalCollection, b
          */
         UnsignedWord averagePromoted = UnsignedUtils.fromDouble(avgPromoted.getPaddedAverage());
         UnsignedWord promotionEstimate = UnsignedUtils.min(averagePromoted, youngUsed);
-        UnsignedWord oldFree = oldSize.subtract(oldUsed);
+        UnsignedWord oldFree = sizes.getOldSize().subtract(oldUsed);
         return promotionEstimate.aboveThan(oldFree);
     }
 
@@ -254,7 +255,7 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un
             targetSize = survivorLimit;
             decrTenuringThreshold = true;
         }
-        survivorSize = targetSize;
+        sizes.setSurvivorSize(targetSize);
 
         if (decrTenuringThreshold) {
             tenuringThreshold = Math.max(tenuringThreshold - 1, 1);
@@ -264,44 +265,47 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un
     }
 
     protected void computeEdenSpaceSize(@SuppressWarnings("unused") boolean completeCollection, @SuppressWarnings("unused") GCCause cause) {
+        UnsignedWord curEden = sizes.getEdenSize();
+        UnsignedWord curPromo = sizes.getPromoSize();
+
         boolean expansionReducesCost = true; // general assumption
         if (shouldUseEstimator(youngGenChangeForMinorThroughput, minorGcCost())) {
-            expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, edenSize, majorGcCost(), promoSize);
+            expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, curEden, majorGcCost(), curPromo);
             /*
              * Note that if the estimator thinks expanding does not lead to significant improvement,
              * shrink so to not get stuck in a supposed optimum and to keep collecting data points.
              */
         }
 
-        UnsignedWord desiredEdenSize = edenSize;
+        UnsignedWord desiredEdenSize = curEden;
         if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) {
             // from adjust_eden_for_throughput():
-            UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(edenSize);
+            UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(curEden);
             double scaleByRatio = minorGcCost() / gcCost();
             assert scaleByRatio >= 0 && scaleByRatio <= 1;
             UnsignedWord scaledEdenHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(edenHeapDelta));
 
             desiredEdenSize = alignUp(desiredEdenSize.add(scaledEdenHeapDelta));
-            desiredEdenSize = UnsignedUtils.max(desiredEdenSize, edenSize);
+            desiredEdenSize = UnsignedUtils.max(desiredEdenSize, curEden);
             youngGenChangeForMinorThroughput++;
         }
         if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) {
-            UnsignedWord desiredSum = edenSize.add(promoSize);
-            desiredEdenSize = adjustEdenForFootprint(edenSize, desiredSum);
+            UnsignedWord desiredSum = curEden.add(curPromo);
+            desiredEdenSize = adjustEdenForFootprint(curEden, desiredSum);
         }
         assert isAligned(desiredEdenSize);
         desiredEdenSize = minSpaceSize(desiredEdenSize);
 
-        UnsignedWord edenLimit = getMaximumEdenSize();
+        UnsignedWord edenLimit = computeEdenLimit();
         if (desiredEdenSize.aboveThan(edenLimit)) {
             /*
              * If the policy says to get a larger eden but is hitting the limit, don't decrease
              * eden. This can lead to a general drifting down of the eden size. Let the tenuring
              * calculation push more into the old gen.
              */
-            desiredEdenSize = UnsignedUtils.max(edenLimit, edenSize);
+            desiredEdenSize = UnsignedUtils.max(edenLimit, curEden);
         }
-        edenSize = desiredEdenSize;
+        sizes.setEdenSize(desiredEdenSize);
     }
 
     private static boolean shouldUseEstimator(long genChangeForThroughput, double cost) {
@@ -435,13 +439,13 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj
 
         if (completeCollection) {
             updateCollectionEndAverages(avgMajorGcCost, avgMajorPause, majorCostEstimator, avgMajorIntervalSeconds,
-                            cause, latestMajorMutatorIntervalNanos, timer.lastIntervalNanos(), promoSize);
+                            cause, latestMajorMutatorIntervalNanos, timer.lastIntervalNanos(), sizes.getPromoSize());
             majorCount++;
             minorCountSinceMajorCollection = 0;
 
         } else {
             updateCollectionEndAverages(avgMinorGcCost, avgMinorPause, minorCostEstimator, null,
-                            cause, latestMinorMutatorIntervalNanos, timer.lastIntervalNanos(), edenSize);
+                            cause, latestMinorMutatorIntervalNanos, timer.lastIntervalNanos(), sizes.getEdenSize());
             minorCount++;
             minorCountSinceMajorCollection++;
 
@@ -455,7 +459,7 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj
 
         GCAccounting accounting = GCImpl.getAccounting();
         UnsignedWord oldLive = accounting.getOldGenerationAfterChunkBytes();
-        oldSizeExceededInPreviousCollection = oldLive.aboveThan(oldSize);
+        oldSizeExceededInPreviousCollection = oldLive.aboveThan(sizes.getOldSize());
 
         if (!completeCollection) {
             /*
@@ -471,7 +475,7 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj
             UnsignedWord tenuredChunkBytes = accounting.getLastIncrementalCollectionPromotedChunkBytes();
             updateAverages(survivorOverflow, survivedChunkBytes, tenuredChunkBytes);
 
-            computeSurvivorSpaceSizeAndThreshold(survivorOverflow, sizes.maxSurvivorSize());
+            computeSurvivorSpaceSizeAndThreshold(survivorOverflow, sizes.getMaxSurvivorSize());
         }
         if (shouldUpdateStats(cause)) {
             computeEdenSpaceSize(completeCollection, cause);
@@ -483,51 +487,55 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj
     }
 
     private void computeOldGenSpaceSize(UnsignedWord oldLive) { // compute_old_gen_free_space
+        UnsignedWord curEden = sizes.getEdenSize();
+        UnsignedWord curPromo = sizes.getPromoSize();
+        UnsignedWord curMaxOld = sizes.getMaxOldSize();
+
         avgOldLive.sample(oldLive);
 
         // NOTE: if maxOldSize shrunk and difference is negative, unsigned conversion results in 0
-        UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.maxOldSize()) - avgOldLive.getAverage());
-        promoLimit = alignDown(UnsignedUtils.max(promoSize, promoLimit));
+        UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(curMaxOld) - avgOldLive.getAverage());
+        promoLimit = alignDown(UnsignedUtils.max(curPromo, promoLimit));
 
         boolean expansionReducesCost = true; // general assumption
         if (shouldUseEstimator(oldGenChangeForMajorThroughput, majorGcCost())) {
-            expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, promoSize, minorGcCost(), edenSize);
+            expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, curPromo, minorGcCost(), curEden);
             /*
              * Note that if the estimator thinks expanding does not lead to significant improvement,
              * shrink so to not get stuck in a supposed optimum and to keep collecting data points.
              */
         }
 
-        UnsignedWord desiredPromoSize = promoSize;
+        UnsignedWord desiredPromoSize = curPromo;
         if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) {
             // from adjust_promo_for_throughput():
-            UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(promoSize);
+            UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(curPromo);
             double scaleByRatio = majorGcCost() / gcCost();
             assert scaleByRatio >= 0 && scaleByRatio <= 1;
             UnsignedWord scaledPromoHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(promoHeapDelta));
 
-            desiredPromoSize = alignUp(promoSize.add(scaledPromoHeapDelta));
-            desiredPromoSize = UnsignedUtils.max(desiredPromoSize, promoSize);
+            desiredPromoSize = alignUp(curPromo.add(scaledPromoHeapDelta));
+            desiredPromoSize = UnsignedUtils.max(desiredPromoSize, curPromo);
             oldGenChangeForMajorThroughput++;
         }
         if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) {
-            UnsignedWord desiredSum = edenSize.add(promoSize);
-            desiredPromoSize = adjustPromoForFootprint(promoSize, desiredSum);
+            UnsignedWord desiredSum = curEden.add(curPromo);
+            desiredPromoSize = adjustPromoForFootprint(curPromo, desiredSum);
         }
         assert isAligned(desiredPromoSize);
         desiredPromoSize = minSpaceSize(desiredPromoSize);
 
         desiredPromoSize = UnsignedUtils.min(desiredPromoSize, promoLimit);
-        promoSize = desiredPromoSize;
+        sizes.setPromoSize(desiredPromoSize);
 
         // from PSOldGen::resize
         UnsignedWord desiredFreeSpace = calculatedOldFreeSizeInBytes();
         UnsignedWord desiredOldSize = alignUp(oldLive.add(desiredFreeSpace));
-        oldSize = UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), sizes.maxOldSize());
+        sizes.setOldSize(UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), curMaxOld));
     }
 
     UnsignedWord calculatedOldFreeSizeInBytes() {
-        return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(promoSize) + avgPromoted.getPaddedAverage());
+        return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.getPromoSize()) + avgPromoted.getPaddedAverage());
     }
 
     private static UnsignedWord adjustPromoForFootprint(UnsignedWord curPromo, UnsignedWord desiredSum) {
@@ -573,6 +581,11 @@ protected boolean shouldUpdateStats(GCCause cause) { // should_update_{eden,prom
         return cause == GenScavengeGCCause.OnAllocation || USE_ADAPTIVE_SIZE_POLICY_WITH_SYSTEM_GC;
     }
 
+    protected UnsignedWord computeEdenLimit() {
+        assert VMOperation.isGCInProgress() : "result wouldn't be consistent otherwise";
+        return alignDown(sizes.getMaxYoungSize().subtract(sizes.getSurvivorSize().multiply(2)));
+    }
+
     private void updateCollectionEndAverages(AdaptiveWeightedAverage costAverage, AdaptivePaddedAverage pauseAverage, ReciprocalLeastSquareFit costEstimator,
                     AdaptiveWeightedAverage intervalSeconds, GCCause cause, long mutatorNanos, long pauseNanos, UnsignedWord sizeBytes) {
         if (shouldUpdateStats(cause)) {
@@ -591,10 +604,4 @@ private void updateCollectionEndAverages(AdaptiveWeightedAverage costAverage, Ad
             costEstimator.sample(UnsignedUtils.toDouble(sizeBytes), cost);
         }
     }
-
-    @Override
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    protected long gcCount() {
-        return minorCount + majorCount;
-    }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java

@@ -228,4 +228,8 @@ default boolean isOutOfMemory(UnsignedWord usedBytes) {
     default void onMaximumHeapSizeExceeded() {
         throw OutOfMemoryUtil.heapSizeExceeded();
     }
+
+    @Uninterruptible(reason = "Tear-down in progress.")
+    default void tearDown() {
+    }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java

@@ -27,7 +27,6 @@
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.word.UnsignedWord;
 
-import com.oracle.svm.core.SubstrateGCOptions;
 import com.oracle.svm.core.heap.DynamicHeapSizeManager;
 
 /**
@@ -46,26 +45,26 @@ protected long getMaximumHeapSizeOptionValue() {
         if (ImageSingletons.contains(DynamicHeapSizeManager.class)) {
             return ImageSingletons.lookup(DynamicHeapSizeManager.class).maxHeapSize().rawValue();
         }
-
-        return SubstrateGCOptions.MaxHeapSize.getValue();
+        return super.getMaximumHeapSizeOptionValue();
     }
 
     @Override
     public boolean isOutOfMemory(UnsignedWord usedBytes) {
-        if (ImageSingletons.contains(DynamicHeapSizeManager.class)) {
-            return ImageSingletons.lookup(DynamicHeapSizeManager.class).outOfMemory(usedBytes);
+        boolean outOfMemory = super.isOutOfMemory(usedBytes);
+        if (ImageSingletons.contains(DynamicHeapSizeManager.class) && outOfMemory) {
+            outOfMemory = ImageSingletons.lookup(DynamicHeapSizeManager.class).outOfMemory(usedBytes);
+            if (!outOfMemory) {
+                /* No longer out-of-memory - update the heap size parameters to reflect that. */
+                GCImpl.getPolicy().updateSizeParameters();
+            }
         }
-
-        return super.isOutOfMemory(usedBytes);
+        return outOfMemory;
     }
 
     @Override
     public void onMaximumHeapSizeExceeded() {
         if (isOutOfMemory(HeapImpl.getAccounting().getUsedBytes())) {
             super.onMaximumHeapSizeExceeded();
-        } else {
-            /* No longer out-of-memory - update the heap size parameters to reflect that. */
-            GCImpl.getPolicy().updateSizeParameters();
         }
     }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -151,6 +151,11 @@ public final class GCImpl implements GC {
         }
     }
 
+    @Uninterruptible(reason = "Tear-down in progress.")
+    public void tearDown() {
+        policy.tearDown();
+    }
+
     @Override
     public String getName() {
         if (SubstrateOptions.useEpsilonGC()) {

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java

@@ -76,6 +76,7 @@
 import com.oracle.svm.core.metaspace.Metaspace;
 import com.oracle.svm.core.nodes.CFunctionEpilogueNode;
 import com.oracle.svm.core.nodes.CFunctionPrologueNode;
+import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey;
 import com.oracle.svm.core.option.RuntimeOptionKey;
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.thread.PlatformThreads;
@@ -211,7 +212,8 @@ public void walkObjects(ObjectVisitor visitor) {
     public boolean tearDown() {
         youngGeneration.tearDown();
         oldGeneration.tearDown();
-        getChunkProvider().tearDown();
+        chunkProvider.tearDown();
+        gcImpl.tearDown();
 
         if (Metaspace.isSupported()) {
             MetaspaceImpl.singleton().tearDown();
@@ -701,10 +703,19 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH
     }
 
     @Override
-    public void optionValueChanged(RuntimeOptionKey<?> key) {
-        if (!SubstrateUtil.HOSTED) {
-            GCImpl.getPolicy().updateSizeParameters();
+    public void optionValueChanged(NotifyGCRuntimeOptionKey<?> key) {
+        if (SubstrateUtil.HOSTED || isIrrelevantForGCPolicy(key)) {
+            return;
         }
+
+        GCImpl.getPolicy().updateSizeParameters();
+    }
+
+    /** For the GC policy, mainly heap-size-related GC options are relevant. */
+    private static boolean isIrrelevantForGCPolicy(RuntimeOptionKey<?> key) {
+        return key == SubstrateGCOptions.DisableExplicitGC ||
+                        key == SubstrateGCOptions.PrintGC ||
+                        key == SubstrateGCOptions.VerboseGC;
     }
 
     @Override
@@ -1008,7 +1019,7 @@ private long totalMemory() {
 
     @Substitute
     private long maxMemory() {
-        GCImpl.getPolicy().updateSizeParameters();
+        GCImpl.getPolicy().ensureSizeParametersInitialized();
         return GCImpl.getPolicy().getMaximumHeapSize().rawValue();
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java

@@ -124,6 +124,7 @@ public static UnsignedWord getMaximumHeapFree() {
         return Word.unsigned(SerialGCOptions.MaxHeapFree.getValue());
     }
 
+    @Fold
     public static int getHeapChunkHeaderPadding() {
         return SerialAndEpsilonGCOptions.HeapChunkHeaderPadding.getValue();
     }
@@ -160,15 +161,12 @@ public static UnsignedWord getLargeArrayThreshold() {
         return Word.unsigned(SerialAndEpsilonGCOptions.LargeArrayThreshold.getValue());
     }
 
-    /*
-     * Zapping
-     */
-
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Fold
     public static boolean getZapProducedHeapChunks() {
         return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue();
     }
 
+    @Fold
     public static boolean getZapConsumedHeapChunks() {
         return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapConsumedHeapChunks.getValue();
     }