diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java index 2b8c35a41b9b..31e5150650f1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java @@ -106,6 +106,12 @@ public void update(UnmodifiableEconomicMap, Object> values) { } } } while (!v.compareAndSet(expect, newMap)); + + UnmodifiableMapCursor, Object> cursor = values.getEntries(); + while (cursor.advance()) { + OptionKey key = cursor.getKey(); + key.afterValueUpdate(); + } } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index 9ff4478e676e..482b0fda5f38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -24,17 +24,22 @@ */ package com.oracle.svm.core.genscavenge; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.IsolateArgumentParser; import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.PhysicalMemory; -import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.locks.VMMutex; import com.oracle.svm.core.os.CommittedMemoryProvider; -import com.oracle.svm.core.thread.JavaSpinLockUtils; +import com.oracle.svm.core.stack.StackOverflowCheck; +import com.oracle.svm.core.thread.RecurringCallbackSupport; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.UnsignedUtils; @@ -42,13 +47,13 @@ import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.Word; -import jdk.internal.misc.Unsafe; /** * Note that a lot of methods in this class are final. Subclasses may only override certain methods * to avoid inconsistencies between the different heap size values. */ abstract class AbstractCollectionPolicy implements CollectionPolicy { + private static final VMMutex SIZES_MUTEX = new VMMutex("AbstractCollectionPolicy.sizes"); protected static final int MIN_SPACE_SIZE_AS_NUMBER_OF_ALIGNED_CHUNKS = 8; @@ -61,9 +66,6 @@ static int getMaxSurvivorSpaces(Integer userValue) { return (userValue != null) ? userValue : AbstractCollectionPolicy.MAX_TENURING_THRESHOLD; } - private static final Unsafe U = Unsafe.getUnsafe(); - private static final long SIZES_UPDATE_LOCK_OFFSET = U.objectFieldOffset(AbstractCollectionPolicy.class, "sizesUpdateLock"); - /* * Constants that can be made options if desirable. These are -XX options in HotSpot, refer to * their descriptions for details. The values are HotSpot defaults unless labeled otherwise. @@ -84,29 +86,27 @@ static int getMaxSurvivorSpaces(Integer userValue) { protected static final int NEW_RATIO = 2; protected final AdaptiveWeightedAverage avgYoungGenAlignedChunkFraction = new AdaptiveWeightedAverage(ADAPTIVE_TIME_WEIGHT); + protected final SizeParameters sizes = new SizeParameters(); private final int initialNewRatio; - protected UnsignedWord survivorSize; - protected UnsignedWord edenSize; - protected UnsignedWord promoSize; - protected UnsignedWord oldSize; protected int tenuringThreshold; - protected volatile SizeParameters sizes = null; - @SuppressWarnings("unused") private volatile int sizesUpdateLock; - protected AbstractCollectionPolicy(int initialNewRatio, int initialTenuringThreshold) { this.initialNewRatio = initialNewRatio; this.tenuringThreshold = UninterruptibleUtils.Math.clamp(initialTenuringThreshold, 1, HeapParameters.getMaxSurvivorSpaces() + 1); } + @Override + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + sizes.tearDown(); + } + @Override public boolean shouldCollectOnAllocation() { - if (sizes == null) { - return false; // updateSizeParameters() has never been called - } + guaranteeSizeParametersInitialized(); UnsignedWord edenUsed = HeapImpl.getAccounting().getEdenUsedBytes(); - return edenUsed.aboveOrEqual(edenSize); + return edenUsed.aboveOrEqual(sizes.getEdenSize()); } @Override @@ -147,155 +147,107 @@ static UnsignedWord minSpaceSize(UnsignedWord size) { @Override public void ensureSizeParametersInitialized() { - if (sizes == null) { + if (!sizes.isInitialized()) { updateSizeParameters(); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected void guaranteeSizeParametersInitialized() { - VMError.guarantee(sizes != null); - } - + VMError.guarantee(sizes.isInitialized()); + } + + /** + * (Re)computes and updates the {@link SizeParameters} used by this garbage collection policy to + * reflect the latest values derived from relevant heap size options and physical memory + * constraints. + *

+ * This method is thread-safe and may be invoked concurrently by multiple threads. It is called + * at least once during VM startup, and is called again if the value of a relevant runtime + * option changes. A mutex is used to guarantee consistency of the computed parameters and + * prevent lost updates (note that a CAS would not be sufficient to prevent lost updates because + * threads could overwrite more recent values with outdated values). + */ @Override public void updateSizeParameters() { - /* - * Read the old object before computing the new values. Otherwise, we risk reusing an - * outdated SizeParameters object. - */ - SizeParameters prevParams = sizes; - SizeParameters newParams = computeSizeParameters(prevParams); - if (prevParams != null && newParams.equal(prevParams)) { - return; // nothing to do - } - updateSizeParameters0(newParams, prevParams); - guaranteeSizeParametersInitialized(); // sanity - } - - @Uninterruptible(reason = "Holding the spin lock at a safepoint can result in deadlocks.") - private void updateSizeParameters0(SizeParameters newParams, SizeParameters prevParams) { - /* - * We use a primitive spin lock because at this point, the current thread might be unable to - * use a Java lock (e.g. no Thread object yet), and the critical section is short, so we do - * not want to suspend and wake up threads for it. - */ - JavaSpinLockUtils.lockNoTransition(this, SIZES_UPDATE_LOCK_OFFSET); + StackOverflowCheck.singleton().makeYellowZoneAvailable(); try { - if (sizes != prevParams) { - /* - * Some other thread beat us and we cannot tell if our values or their values are - * newer, so back off - any newer values will be applied eventually. - */ - return; - } - updateSizeParametersLocked(newParams, prevParams); - } finally { - JavaSpinLockUtils.unlock(this, SIZES_UPDATE_LOCK_OFFSET); - } - } + SIZES_MUTEX.lock(); + try { + assert RecurringCallbackSupport.isCallbackUnsupportedOrTimerSuspended() : "recurring callbacks could trigger recursive locking, which isn't supported"; - @Uninterruptible(reason = "Holding the spin lock at a safepoint can result in deadlocks. Updating the size parameters must be atomic with regard to garbage collection.") - private void updateSizeParametersLocked(SizeParameters newParams, SizeParameters prevParams) { - sizes = newParams; + RawSizeParameters newValuesOnStack = StackValue.get(RawSizeParameters.class); + computeSizeParameters(newValuesOnStack); - if (prevParams == null || gcCount() == 0) { - survivorSize = newParams.initialSurvivorSize; - edenSize = newParams.initialEdenSize; - oldSize = newParams.initialOldSize(); - promoSize = UnsignedUtils.min(edenSize, oldSize); + sizes.update(newValuesOnStack); + } finally { + SIZES_MUTEX.unlock(); + } + } finally { + StackOverflowCheck.singleton().protectYellowZone(); } - - /* - * NOTE: heap limits can change when options are updated at runtime or once the physical - * memory size becomes known. This means that we start off with sizes which can cause higher - * GC costs initially, and when shrinking the heap, that previously computed values such as - * GC costs and intervals and survived/promoted objects are likely no longer representative. - * - * We assume that such changes happen very early on and values then adapt reasonably quick, - * but we must still ensure that computations can handle it (for example, no overflows). - */ - survivorSize = UnsignedUtils.min(survivorSize, newParams.maxSurvivorSize()); - edenSize = UnsignedUtils.min(edenSize, getMaximumEdenSize()); - oldSize = UnsignedUtils.min(oldSize, newParams.maxOldSize()); - promoSize = UnsignedUtils.min(promoSize, newParams.maxOldSize()); } @Override public final UnsignedWord getInitialEdenSize() { - guaranteeSizeParametersInitialized(); - return sizes.initialEdenSize; + return sizes.getInitialEdenSize(); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public final UnsignedWord getMaximumEdenSize() { - guaranteeSizeParametersInitialized(); - return alignDown(sizes.maxYoungSize.subtract(survivorSize.multiply(2))); + return sizes.getMaxEdenSize(); } @Override public final UnsignedWord getMaximumHeapSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxHeapSize; + return sizes.getMaxHeapSize(); } @Override public final UnsignedWord getMaximumYoungGenerationSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxYoungSize; + return sizes.getMaxYoungSize(); } @Override public final UnsignedWord getInitialSurvivorSize() { - guaranteeSizeParametersInitialized(); - return sizes.initialSurvivorSize; + return sizes.getInitialSurvivorSize(); } @Override public final UnsignedWord getMaximumSurvivorSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxSurvivorSize(); + return sizes.getMaxSurvivorSize(); } @Override public final UnsignedWord getCurrentHeapCapacity() { - assert VMOperation.isGCInProgress() : "use only during GC"; - guaranteeSizeParametersInitialized(); - return edenSize.add(survivorSize).add(oldSize); + return sizes.getHeapSize(); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public final UnsignedWord getSurvivorSpacesCapacity() { - assert VMOperation.isGCInProgress() : "use only during GC"; - guaranteeSizeParametersInitialized(); - return survivorSize; + return sizes.getSurvivorSize(); } @Override - @Uninterruptible(reason = "Ensure reading a consistent value.") public final UnsignedWord getYoungGenerationCapacity() { - guaranteeSizeParametersInitialized(); - return edenSize.add(survivorSize); + return sizes.getYoungSize(); } @Override public final UnsignedWord getInitialOldSize() { - guaranteeSizeParametersInitialized(); - return sizes.initialOldSize(); + return sizes.getInitialOldSize(); } @Override public final UnsignedWord getMaximumOldSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxOldSize(); + return sizes.getMaxOldSize(); } @Override public final UnsignedWord getOldGenerationCapacity() { - guaranteeSizeParametersInitialized(); - return oldSize; + return sizes.getOldSize(); } @Override @@ -307,13 +259,13 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { * survivor spaces in a future collection. We could alternatively return * getCurrentHeapCapacity() to have chunks ready during full GCs as well. */ - UnsignedWord total = edenSize.add(HeapImpl.getAccounting().getSurvivorUsedBytes()); + UnsignedWord total = sizes.getEdenSize().add(HeapImpl.getAccounting().getSurvivorUsedBytes()); double alignedFraction = Math.min(1, Math.max(0, avgYoungGenAlignedChunkFraction.getAverage())); return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(total) * alignedFraction); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; @@ -321,6 +273,8 @@ public int getTenuringAge() { @Override public void onCollectionBegin(boolean completeCollection, long beginNanoTime) { + sizes.freeUnusedSizeParameters(); + // Capture the fraction of bytes in aligned chunks at the start to include all allocated // (also dead) objects, because we use it to reserve aligned chunks for future allocations UnsignedWord youngChunkBytes = GCImpl.getAccounting().getYoungChunkBytesBefore(); @@ -332,15 +286,13 @@ public void onCollectionBegin(boolean completeCollection, long beginNanoTime) { @Override public final UnsignedWord getMinimumHeapSize() { - return sizes.minHeapSize; + return sizes.getMinHeapSize(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract long gcCount(); - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/shared/genArguments.cpp#L195-L310") + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L146-L168") - protected SizeParameters computeSizeParameters(SizeParameters existing) { + private void computeSizeParameters(RawSizeParameters newParamsOnStack) { UnsignedWord minYoungSpaces = minSpaceSize(); // eden if (HeapParameters.getMaxSurvivorSpaces() > 0) { minYoungSpaces = minYoungSpaces.add(minSpaceSize().multiply(2)); // survivor from and to @@ -359,7 +311,7 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { maxHeap = UnsignedUtils.clamp(alignDown(maxHeap), minAllSpaces, heapSizeLimit); UnsignedWord maxYoung; - long optionMaxYoung = SubstrateGCOptions.MaxNewSize.getValue(); + long optionMaxYoung = getMaxNewSizeOptionValue(); if (optionMaxYoung > 0L) { maxYoung = Word.unsigned(optionMaxYoung); } else if (SerialAndEpsilonGCOptions.MaximumYoungGenerationSizePercent.hasBeenSet()) { @@ -375,7 +327,7 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { (maxHeap.belowOrEqual(unadjustedMaxHeap) || unadjustedMaxHeap.belowThan(minAllSpaces))); UnsignedWord minHeap = Word.zero(); - long optionMin = SubstrateGCOptions.MinHeapSize.getValue(); + long optionMin = getMinHeapSizeOptionValue(); if (optionMin > 0L) { minHeap = Word.unsigned(optionMin); } @@ -391,6 +343,7 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { initialYoung = initialHeap.unsignedDivide(initialNewRatio + 1); initialYoung = UnsignedUtils.clamp(alignUp(initialYoung), minYoungSpaces, maxYoung); } + UnsignedWord initialSurvivor = Word.zero(); if (HeapParameters.getMaxSurvivorSpaces() > 0) { /* @@ -404,10 +357,49 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { initialSurvivor = initialYoung.unsignedDivide(AbstractCollectionPolicy.INITIAL_SURVIVOR_RATIO); initialSurvivor = minSpaceSize(alignDown(initialSurvivor)); } + UnsignedWord initialEden = initialYoung.subtract(initialSurvivor.multiply(2)); initialEden = minSpaceSize(alignDown(initialEden)); - return SizeParameters.get(existing, maxHeap, maxYoung, initialHeap, initialEden, initialSurvivor, minHeap); + UnsignedWord maxSurvivorSize = Word.zero(); + UnsignedWord maxEdenSize = maxYoung; + if (HeapParameters.getMaxSurvivorSpaces() > 0) { + maxSurvivorSize = maxYoung.unsignedDivide(MIN_SURVIVOR_RATIO); + maxSurvivorSize = minSpaceSize(alignDown(maxSurvivorSize)); + maxEdenSize = maxYoung.subtract(minSpaceSize().multiply(2)); + } + + UnsignedWord maxOldSize = maxHeap.subtract(maxYoung); + + UnsignedWord survivorSize; + UnsignedWord edenSize; + UnsignedWord oldSize; + UnsignedWord promoSize; + if (sizes.isInitialized()) { + /* Copy and limit existing values. */ + survivorSize = UnsignedUtils.min(sizes.getSurvivorSize(), maxSurvivorSize); + edenSize = UnsignedUtils.min(sizes.getEdenSize(), maxEdenSize); + oldSize = UnsignedUtils.min(sizes.getOldSize(), maxOldSize); + promoSize = UnsignedUtils.min(sizes.getPromoSize(), maxOldSize); + } else { + /* Set initial values. */ + survivorSize = initialSurvivor; + edenSize = initialEden; + oldSize = initialHeap.subtract(initialYoung); + promoSize = UnsignedUtils.min(edenSize, oldSize); + } + + UnsignedWord initialOldSize = initialHeap.subtract(initialYoung); + UnsignedWord youngSize = edenSize.add(survivorSize); + UnsignedWord heapSize = edenSize.add(survivorSize).add(oldSize); + + RawSizeParametersOnStackAccess.initialize(newParamsOnStack, + initialEden, edenSize, maxEdenSize, + initialSurvivor, survivorSize, maxSurvivorSize, + initialOldSize, oldSize, maxOldSize, + promoSize, + initialYoung, youngSize, maxYoung, + minHeap, initialHeap, heapSize, maxHeap); } protected UnsignedWord getHeapSizeLimit() { @@ -419,83 +411,21 @@ protected UnsignedWord getYoungSizeLimit(UnsignedWord maxHeap) { } protected long getMaximumHeapSizeOptionValue() { - return SubstrateGCOptions.MaxHeapSize.getValue(); + int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MaxHeapSize); + return IsolateArgumentParser.singleton().getLongOptionValue(optionIndex); } - protected UnsignedWord getInitialHeapSize() { - return AbstractCollectionPolicy.INITIAL_HEAP_SIZE; + protected static long getMaxNewSizeOptionValue() { + int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MaxNewSize); + return IsolateArgumentParser.singleton().getLongOptionValue(optionIndex); } - protected static final class SizeParameters { - final UnsignedWord maxHeapSize; - final UnsignedWord maxYoungSize; - final UnsignedWord initialHeapSize; - final UnsignedWord initialEdenSize; - final UnsignedWord initialSurvivorSize; - final UnsignedWord minHeapSize; - - static SizeParameters get(SizeParameters existing, UnsignedWord maxHeap, UnsignedWord maxYoung, UnsignedWord initialHeap, - UnsignedWord initialEden, UnsignedWord initialSurvivor, UnsignedWord minHeap) { - if (existing != null && existing.matches(maxHeap, maxYoung, initialHeap, initialEden, initialSurvivor, minHeap)) { - return existing; - } - return new SizeParameters(maxHeap, maxYoung, initialHeap, initialEden, initialSurvivor, minHeap); - } - - private SizeParameters(UnsignedWord maxHeapSize, UnsignedWord maxYoungSize, UnsignedWord initialHeapSize, - UnsignedWord initialEdenSize, UnsignedWord initialSurvivorSize, UnsignedWord minHeapSize) { - this.maxHeapSize = maxHeapSize; - this.maxYoungSize = maxYoungSize; - this.initialHeapSize = initialHeapSize; - this.initialEdenSize = initialEdenSize; - this.initialSurvivorSize = initialSurvivorSize; - this.minHeapSize = minHeapSize; - - assert isAligned(maxHeapSize) && isAligned(maxYoungSize) && isAligned(initialHeapSize) && isAligned(initialEdenSize) && isAligned(initialSurvivorSize); - assert isAligned(maxSurvivorSize()) && isAligned(initialYoungSize()) && isAligned(initialOldSize()) && isAligned(maxOldSize()); - - assert initialHeapSize.belowOrEqual(maxHeapSize); - assert maxSurvivorSize().belowThan(maxYoungSize); - assert maxYoungSize.add(maxOldSize()).equal(maxHeapSize); - assert maxHeapSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()); - assert initialEdenSize.add(initialSurvivorSize.multiply(2)).equal(initialYoungSize()); - assert initialYoungSize().add(initialOldSize()).equal(initialHeapSize); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") - UnsignedWord maxSurvivorSize() { - if (HeapParameters.getMaxSurvivorSpaces() == 0) { - return Word.zero(); - } - UnsignedWord size = maxYoungSize.unsignedDivide(MIN_SURVIVOR_RATIO); - return minSpaceSize(alignDown(size)); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - UnsignedWord initialYoungSize() { - return initialEdenSize.add(initialSurvivorSize.multiply(2)); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - UnsignedWord initialOldSize() { - return initialHeapSize.subtract(initialYoungSize()); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - UnsignedWord maxOldSize() { - return maxHeapSize.subtract(maxYoungSize); - } - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - boolean equal(SizeParameters other) { - return other == this || other.matches(maxHeapSize, maxYoungSize, initialHeapSize, initialEdenSize, initialSurvivorSize, minHeapSize); - } + protected static long getMinHeapSizeOptionValue() { + int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MinHeapSize); + return IsolateArgumentParser.singleton().getLongOptionValue(optionIndex); + } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - boolean matches(UnsignedWord maxHeap, UnsignedWord maxYoung, UnsignedWord initialHeap, UnsignedWord initialEden, UnsignedWord initialSurvivor, UnsignedWord minHeap) { - return maxHeapSize.equal(maxHeap) && maxYoungSize.equal(maxYoung) && initialHeapSize.equal(initialHeap) && - initialEdenSize.equal(initialEden) && initialSurvivorSize.equal(initialSurvivor) && minHeapSize.equal(minHeap); - } + protected UnsignedWord getInitialHeapSize() { + return AbstractCollectionPolicy.INITIAL_HEAP_SIZE; } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java index 56cb85a010a4..f61092f206f4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java @@ -29,8 +29,8 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Isolates; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; +import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.Timer; @@ -108,7 +108,8 @@ class AdaptiveCollectionPolicy extends AbstractCollectionPolicy { private static final int ADAPTIVE_SIZE_POLICY_INITIALIZING_STEPS = ADAPTIVE_SIZE_POLICY_READY_THRESHOLD; /** * The minimum estimated decrease in {@link #gcCost()} in percent to decide in favor of - * expanding a space by 1% of the combined size of {@link #edenSize} and {@link #promoSize}. + * expanding a space by 1% of the combined size of {@link SizeParameters#getEdenSize} and + * {@link SizeParameters#getPromoSize}. */ private static final double ADAPTIVE_SIZE_ESTIMATOR_MIN_TOTAL_SIZE_COST_TRADEOFF = 0.5; /** The effective number of most recent data points used by estimator (exponential decay). */ @@ -212,7 +213,7 @@ public boolean shouldCollectCompletely(boolean followingIncrementalCollection, b */ UnsignedWord averagePromoted = UnsignedUtils.fromDouble(avgPromoted.getPaddedAverage()); UnsignedWord promotionEstimate = UnsignedUtils.min(averagePromoted, youngUsed); - UnsignedWord oldFree = oldSize.subtract(oldUsed); + UnsignedWord oldFree = sizes.getOldSize().subtract(oldUsed); return promotionEstimate.aboveThan(oldFree); } @@ -254,7 +255,7 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un targetSize = survivorLimit; decrTenuringThreshold = true; } - survivorSize = targetSize; + sizes.setSurvivorSize(targetSize); if (decrTenuringThreshold) { tenuringThreshold = Math.max(tenuringThreshold - 1, 1); @@ -264,44 +265,47 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un } protected void computeEdenSpaceSize(@SuppressWarnings("unused") boolean completeCollection, @SuppressWarnings("unused") GCCause cause) { + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord curPromo = sizes.getPromoSize(); + boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(youngGenChangeForMinorThroughput, minorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, edenSize, majorGcCost(), promoSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, curEden, majorGcCost(), curPromo); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. */ } - UnsignedWord desiredEdenSize = edenSize; + UnsignedWord desiredEdenSize = curEden; if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) { // from adjust_eden_for_throughput(): - UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(edenSize); + UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(curEden); double scaleByRatio = minorGcCost() / gcCost(); assert scaleByRatio >= 0 && scaleByRatio <= 1; UnsignedWord scaledEdenHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(edenHeapDelta)); desiredEdenSize = alignUp(desiredEdenSize.add(scaledEdenHeapDelta)); - desiredEdenSize = UnsignedUtils.max(desiredEdenSize, edenSize); + desiredEdenSize = UnsignedUtils.max(desiredEdenSize, curEden); youngGenChangeForMinorThroughput++; } if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) { - UnsignedWord desiredSum = edenSize.add(promoSize); - desiredEdenSize = adjustEdenForFootprint(edenSize, desiredSum); + UnsignedWord desiredSum = curEden.add(curPromo); + desiredEdenSize = adjustEdenForFootprint(curEden, desiredSum); } assert isAligned(desiredEdenSize); desiredEdenSize = minSpaceSize(desiredEdenSize); - UnsignedWord edenLimit = getMaximumEdenSize(); + UnsignedWord edenLimit = computeEdenLimit(); if (desiredEdenSize.aboveThan(edenLimit)) { /* * If the policy says to get a larger eden but is hitting the limit, don't decrease * eden. This can lead to a general drifting down of the eden size. Let the tenuring * calculation push more into the old gen. */ - desiredEdenSize = UnsignedUtils.max(edenLimit, edenSize); + desiredEdenSize = UnsignedUtils.max(edenLimit, curEden); } - edenSize = desiredEdenSize; + sizes.setEdenSize(desiredEdenSize); } private static boolean shouldUseEstimator(long genChangeForThroughput, double cost) { @@ -435,13 +439,13 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj if (completeCollection) { updateCollectionEndAverages(avgMajorGcCost, avgMajorPause, majorCostEstimator, avgMajorIntervalSeconds, - cause, latestMajorMutatorIntervalNanos, timer.lastIntervalNanos(), promoSize); + cause, latestMajorMutatorIntervalNanos, timer.lastIntervalNanos(), sizes.getPromoSize()); majorCount++; minorCountSinceMajorCollection = 0; } else { updateCollectionEndAverages(avgMinorGcCost, avgMinorPause, minorCostEstimator, null, - cause, latestMinorMutatorIntervalNanos, timer.lastIntervalNanos(), edenSize); + cause, latestMinorMutatorIntervalNanos, timer.lastIntervalNanos(), sizes.getEdenSize()); minorCount++; minorCountSinceMajorCollection++; @@ -455,7 +459,7 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj GCAccounting accounting = GCImpl.getAccounting(); UnsignedWord oldLive = accounting.getOldGenerationAfterChunkBytes(); - oldSizeExceededInPreviousCollection = oldLive.aboveThan(oldSize); + oldSizeExceededInPreviousCollection = oldLive.aboveThan(sizes.getOldSize()); if (!completeCollection) { /* @@ -471,7 +475,7 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj UnsignedWord tenuredChunkBytes = accounting.getLastIncrementalCollectionPromotedChunkBytes(); updateAverages(survivorOverflow, survivedChunkBytes, tenuredChunkBytes); - computeSurvivorSpaceSizeAndThreshold(survivorOverflow, sizes.maxSurvivorSize()); + computeSurvivorSpaceSizeAndThreshold(survivorOverflow, sizes.getMaxSurvivorSize()); } if (shouldUpdateStats(cause)) { computeEdenSpaceSize(completeCollection, cause); @@ -483,51 +487,55 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj } private void computeOldGenSpaceSize(UnsignedWord oldLive) { // compute_old_gen_free_space + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord curPromo = sizes.getPromoSize(); + UnsignedWord curMaxOld = sizes.getMaxOldSize(); + avgOldLive.sample(oldLive); // NOTE: if maxOldSize shrunk and difference is negative, unsigned conversion results in 0 - UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.maxOldSize()) - avgOldLive.getAverage()); - promoLimit = alignDown(UnsignedUtils.max(promoSize, promoLimit)); + UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(curMaxOld) - avgOldLive.getAverage()); + promoLimit = alignDown(UnsignedUtils.max(curPromo, promoLimit)); boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(oldGenChangeForMajorThroughput, majorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, promoSize, minorGcCost(), edenSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, curPromo, minorGcCost(), curEden); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. */ } - UnsignedWord desiredPromoSize = promoSize; + UnsignedWord desiredPromoSize = curPromo; if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) { // from adjust_promo_for_throughput(): - UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(promoSize); + UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(curPromo); double scaleByRatio = majorGcCost() / gcCost(); assert scaleByRatio >= 0 && scaleByRatio <= 1; UnsignedWord scaledPromoHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(promoHeapDelta)); - desiredPromoSize = alignUp(promoSize.add(scaledPromoHeapDelta)); - desiredPromoSize = UnsignedUtils.max(desiredPromoSize, promoSize); + desiredPromoSize = alignUp(curPromo.add(scaledPromoHeapDelta)); + desiredPromoSize = UnsignedUtils.max(desiredPromoSize, curPromo); oldGenChangeForMajorThroughput++; } if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) { - UnsignedWord desiredSum = edenSize.add(promoSize); - desiredPromoSize = adjustPromoForFootprint(promoSize, desiredSum); + UnsignedWord desiredSum = curEden.add(curPromo); + desiredPromoSize = adjustPromoForFootprint(curPromo, desiredSum); } assert isAligned(desiredPromoSize); desiredPromoSize = minSpaceSize(desiredPromoSize); desiredPromoSize = UnsignedUtils.min(desiredPromoSize, promoLimit); - promoSize = desiredPromoSize; + sizes.setPromoSize(desiredPromoSize); // from PSOldGen::resize UnsignedWord desiredFreeSpace = calculatedOldFreeSizeInBytes(); UnsignedWord desiredOldSize = alignUp(oldLive.add(desiredFreeSpace)); - oldSize = UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), sizes.maxOldSize()); + sizes.setOldSize(UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), curMaxOld)); } UnsignedWord calculatedOldFreeSizeInBytes() { - return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(promoSize) + avgPromoted.getPaddedAverage()); + return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.getPromoSize()) + avgPromoted.getPaddedAverage()); } private static UnsignedWord adjustPromoForFootprint(UnsignedWord curPromo, UnsignedWord desiredSum) { @@ -573,6 +581,11 @@ protected boolean shouldUpdateStats(GCCause cause) { // should_update_{eden,prom return cause == GenScavengeGCCause.OnAllocation || USE_ADAPTIVE_SIZE_POLICY_WITH_SYSTEM_GC; } + protected UnsignedWord computeEdenLimit() { + assert VMOperation.isGCInProgress() : "result wouldn't be consistent otherwise"; + return alignDown(sizes.getMaxYoungSize().subtract(sizes.getSurvivorSize().multiply(2))); + } + private void updateCollectionEndAverages(AdaptiveWeightedAverage costAverage, AdaptivePaddedAverage pauseAverage, ReciprocalLeastSquareFit costEstimator, AdaptiveWeightedAverage intervalSeconds, GCCause cause, long mutatorNanos, long pauseNanos, UnsignedWord sizeBytes) { if (shouldUpdateStats(cause)) { @@ -591,10 +604,4 @@ private void updateCollectionEndAverages(AdaptiveWeightedAverage costAverage, Ad costEstimator.sample(UnsignedUtils.toDouble(sizeBytes), cost); } } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected long gcCount() { - return minorCount + majorCount; - } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index b6b3174173df..d5bceb425c50 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -228,4 +228,8 @@ default boolean isOutOfMemory(UnsignedWord usedBytes) { default void onMaximumHeapSizeExceeded() { throw OutOfMemoryUtil.heapSizeExceeded(); } + + @Uninterruptible(reason = "Tear-down in progress.") + default void tearDown() { + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java index b61c9418016f..8cc138809f14 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java @@ -27,7 +27,6 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.heap.DynamicHeapSizeManager; /** @@ -46,26 +45,26 @@ protected long getMaximumHeapSizeOptionValue() { if (ImageSingletons.contains(DynamicHeapSizeManager.class)) { return ImageSingletons.lookup(DynamicHeapSizeManager.class).maxHeapSize().rawValue(); } - - return SubstrateGCOptions.MaxHeapSize.getValue(); + return super.getMaximumHeapSizeOptionValue(); } @Override public boolean isOutOfMemory(UnsignedWord usedBytes) { - if (ImageSingletons.contains(DynamicHeapSizeManager.class)) { - return ImageSingletons.lookup(DynamicHeapSizeManager.class).outOfMemory(usedBytes); + boolean outOfMemory = super.isOutOfMemory(usedBytes); + if (ImageSingletons.contains(DynamicHeapSizeManager.class) && outOfMemory) { + outOfMemory = ImageSingletons.lookup(DynamicHeapSizeManager.class).outOfMemory(usedBytes); + if (!outOfMemory) { + /* No longer out-of-memory - update the heap size parameters to reflect that. */ + GCImpl.getPolicy().updateSizeParameters(); + } } - - return super.isOutOfMemory(usedBytes); + return outOfMemory; } @Override public void onMaximumHeapSizeExceeded() { if (isOutOfMemory(HeapImpl.getAccounting().getUsedBytes())) { super.onMaximumHeapSizeExceeded(); - } else { - /* No longer out-of-memory - update the heap size parameters to reflect that. */ - GCImpl.getPolicy().updateSizeParameters(); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 17fa4f11ef34..62d9362d27af 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -151,6 +151,11 @@ public final class GCImpl implements GC { } } + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + policy.tearDown(); + } + @Override public String getName() { if (SubstrateOptions.useEpsilonGC()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index e1e7efb6b161..867beebdd70f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -76,6 +76,7 @@ import com.oracle.svm.core.metaspace.Metaspace; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.PlatformThreads; @@ -211,7 +212,8 @@ public void walkObjects(ObjectVisitor visitor) { public boolean tearDown() { youngGeneration.tearDown(); oldGeneration.tearDown(); - getChunkProvider().tearDown(); + chunkProvider.tearDown(); + gcImpl.tearDown(); if (Metaspace.isSupported()) { MetaspaceImpl.singleton().tearDown(); @@ -701,10 +703,19 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH } @Override - public void optionValueChanged(RuntimeOptionKey key) { - if (!SubstrateUtil.HOSTED) { - GCImpl.getPolicy().updateSizeParameters(); + public void optionValueChanged(NotifyGCRuntimeOptionKey key) { + if (SubstrateUtil.HOSTED || isIrrelevantForGCPolicy(key)) { + return; } + + GCImpl.getPolicy().updateSizeParameters(); + } + + /** For the GC policy, mainly heap-size-related GC options are relevant. */ + private static boolean isIrrelevantForGCPolicy(RuntimeOptionKey key) { + return key == SubstrateGCOptions.DisableExplicitGC || + key == SubstrateGCOptions.PrintGC || + key == SubstrateGCOptions.VerboseGC; } @Override @@ -1008,7 +1019,7 @@ private long totalMemory() { @Substitute private long maxMemory() { - GCImpl.getPolicy().updateSizeParameters(); + GCImpl.getPolicy().ensureSizeParametersInitialized(); return GCImpl.getPolicy().getMaximumHeapSize().rawValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 23c68c8011b2..3793a82f7a82 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -124,6 +124,7 @@ public static UnsignedWord getMaximumHeapFree() { return Word.unsigned(SerialGCOptions.MaxHeapFree.getValue()); } + @Fold public static int getHeapChunkHeaderPadding() { return SerialAndEpsilonGCOptions.HeapChunkHeaderPadding.getValue(); } @@ -160,15 +161,12 @@ public static UnsignedWord getLargeArrayThreshold() { return Word.unsigned(SerialAndEpsilonGCOptions.LargeArrayThreshold.getValue()); } - /* - * Zapping - */ - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Fold public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } + @Fold public static boolean getZapConsumedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapConsumedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java index 40e3944b7c7e..718d6ae11129 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java @@ -91,7 +91,7 @@ public boolean shouldCollectOnHint(boolean fullGC) { edenUsedBytes = edenUsedBytes.add(FULL_GC_BONUS); } return edenUsedBytes.aboveOrEqual(Word.unsigned(Options.ExpectedEdenSize.getValue())) || - (UnsignedUtils.toDouble(edenUsedBytes) / UnsignedUtils.toDouble(edenSize) >= Options.UsedEdenProportionThreshold.getValue()); + (UnsignedUtils.toDouble(edenUsedBytes) / UnsignedUtils.toDouble(sizes.getEdenSize()) >= Options.UsedEdenProportionThreshold.getValue()); } @Override @@ -142,17 +142,19 @@ protected boolean shouldUpdateStats(GCCause cause) { protected void computeEdenSpaceSize(boolean completeCollection, GCCause cause) { if (cause == GCCause.HintedGC) { if (completeCollection && lastGCCause == GCCause.HintedGC) { - UnsignedWord newEdenSize = UnsignedUtils.max(sizes.initialEdenSize, alignUp(edenSize.unsignedDivide(2))); - if (edenSize.aboveThan(newEdenSize)) { - edenSize = newEdenSize; + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord newEden = UnsignedUtils.max(sizes.getInitialEdenSize(), alignUp(curEden.unsignedDivide(2))); + if (curEden.aboveThan(newEden)) { + sizes.setEdenSize(newEden); } } } else { UnsignedWord sizeAfter = GCImpl.getChunkBytes(); if (sizeBefore.notEqual(0) && sizeBefore.belowThan(sizeAfter.multiply(2))) { - UnsignedWord newEdenSize = UnsignedUtils.min(getMaximumEdenSize(), alignUp(edenSize.multiply(2))); - if (edenSize.belowThan(newEdenSize)) { - edenSize = newEdenSize; + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord newEden = UnsignedUtils.min(computeEdenLimit(), alignUp(curEden.multiply(2))); + if (curEden.belowThan(newEden)) { + sizes.setEdenSize(newEden); } } else { super.computeEdenSpaceSize(completeCollection, cause); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java index da5a29f66d46..c01690c56c3c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java @@ -26,13 +26,13 @@ import static com.oracle.svm.core.genscavenge.CollectionPolicy.shouldCollectYoungGenSeparately; -import jdk.graal.compiler.word.Word; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.word.Word; + /** A port of HotSpot's SerialGC size policy. */ final class ProportionateSpacesPolicy extends AbstractCollectionPolicy { @@ -50,7 +50,6 @@ final class ProportionateSpacesPolicy extends AbstractCollectionPolicy { static final int MAX_TENURING_THRESHOLD = 15; static final int TARGET_SURVIVOR_RATIO = 50; - private int totalCollections; private boolean oldSizeExceededInPreviousCollection; private int shrinkFactor; @@ -95,19 +94,17 @@ public void onCollectionBegin(boolean completeCollection, long beginNanoTime) { @Override public void onCollectionEnd(boolean completeCollection, GCCause cause) { UnsignedWord oldLive = GCImpl.getAccounting().getOldGenerationAfterChunkBytes(); - oldSizeExceededInPreviousCollection = oldLive.aboveThan(oldSize); + oldSizeExceededInPreviousCollection = oldLive.aboveThan(sizes.getOldSize()); boolean resizeOldOnlyForPromotions = !completeCollection; computeNewOldGenSize(resizeOldOnlyForPromotions); computeNewYoungGenSize(); adjustDesiredTenuringThreshold(); - - totalCollections++; } private void adjustDesiredTenuringThreshold() { // DefNewGeneration::adjust_desired_tenuring_threshold // Set the desired survivor size to half the real survivor space - UnsignedWord desiredSurvivorSize = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(survivorSize) * TARGET_SURVIVOR_RATIO / 100); + UnsignedWord desiredSurvivorSize = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.getSurvivorSize()) * TARGET_SURVIVOR_RATIO / 100); // AgeTable::compute_tenuring_threshold YoungGeneration youngGen = HeapImpl.getHeapImpl().getYoungGeneration(); @@ -126,10 +123,10 @@ private void adjustDesiredTenuringThreshold() { // DefNewGeneration::adjust_desi } private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredGeneration::compute_new_size_inner - UnsignedWord capacityAtPrologue = oldSize; + UnsignedWord capacityAtPrologue = sizes.getOldSize(); UnsignedWord usedAfterGc = GCImpl.getAccounting().getOldGenerationAfterChunkBytes(); - if (oldSize.belowThan(usedAfterGc)) { - oldSize = usedAfterGc; + if (sizes.getOldSize().belowThan(usedAfterGc)) { + sizes.setOldSize(usedAfterGc); } if (resizeOnlyForPromotions) { return; @@ -142,24 +139,24 @@ private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredG double maximumUsedPercentage = 1 - minimumFreePercentage; UnsignedWord minimumDesiredCapacity = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(usedAfterGc) / maximumUsedPercentage); - minimumDesiredCapacity = UnsignedUtils.max(minimumDesiredCapacity, sizes.initialOldSize()); + minimumDesiredCapacity = UnsignedUtils.max(minimumDesiredCapacity, sizes.getInitialOldSize()); - if (oldSize.belowThan(minimumDesiredCapacity)) { - oldSize = alignUp(minimumDesiredCapacity); + if (sizes.getOldSize().belowThan(minimumDesiredCapacity)) { + sizes.setOldSize(alignUp(minimumDesiredCapacity)); return; } - UnsignedWord maxShrinkBytes = oldSize.subtract(minimumDesiredCapacity); + UnsignedWord maxShrinkBytes = sizes.getOldSize().subtract(minimumDesiredCapacity); UnsignedWord shrinkBytes = Word.zero(); if (MAX_HEAP_FREE_RATIO < 100) { double maximumFreePercentage = MAX_HEAP_FREE_RATIO / 100.0; double minimumUsedPercentage = 1 - maximumFreePercentage; UnsignedWord maximumDesiredCapacity = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(usedAfterGc) / minimumUsedPercentage); - maximumDesiredCapacity = UnsignedUtils.max(maximumDesiredCapacity, sizes.initialOldSize()); + maximumDesiredCapacity = UnsignedUtils.max(maximumDesiredCapacity, sizes.getInitialOldSize()); assert minimumDesiredCapacity.belowOrEqual(maximumDesiredCapacity); - if (oldSize.aboveThan(maximumDesiredCapacity)) { - shrinkBytes = oldSize.subtract(maximumDesiredCapacity); + if (sizes.getOldSize().aboveThan(maximumDesiredCapacity)) { + shrinkBytes = sizes.getOldSize().subtract(maximumDesiredCapacity); if (SHRINK_HEAP_IN_STEPS) { /* * We don't want to shrink all the way back to initSize if people call @@ -180,39 +177,36 @@ private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredG } } - if (oldSize.aboveThan(capacityAtPrologue)) { + if (sizes.getOldSize().aboveThan(capacityAtPrologue)) { /* * We might have expanded for promotions, in which case we might want to take back that * expansion if there's room after GC. That keeps us from stretching the heap with * promotions when there's plenty of room. */ - UnsignedWord expansionForPromotion = oldSize.subtract(capacityAtPrologue); + UnsignedWord expansionForPromotion = sizes.getOldSize().subtract(capacityAtPrologue); expansionForPromotion = UnsignedUtils.min(expansionForPromotion, maxShrinkBytes); shrinkBytes = UnsignedUtils.max(shrinkBytes, expansionForPromotion); } if (shrinkBytes.aboveThan(MIN_HEAP_FREE_RATIO)) { - oldSize = oldSize.subtract(shrinkBytes); + sizes.setOldSize(sizes.getOldSize().subtract(shrinkBytes)); } } private void computeNewYoungGenSize() { // DefNewGeneration::compute_new_size - UnsignedWord desiredNewSize = oldSize.unsignedDivide(NEW_RATIO); - desiredNewSize = UnsignedUtils.clamp(desiredNewSize, sizes.initialYoungSize(), sizes.maxYoungSize); + UnsignedWord desiredNewSize = sizes.getOldSize().unsignedDivide(NEW_RATIO); + desiredNewSize = UnsignedUtils.clamp(desiredNewSize, sizes.getInitialYoungSize(), sizes.getMaxYoungSize()); // DefNewGeneration::compute_space_boundaries, DefNewGeneration::compute_survivor_size - survivorSize = minSpaceSize(alignDown(desiredNewSize.unsignedDivide(SURVIVOR_RATIO))); + UnsignedWord newSurvivorSize = minSpaceSize(alignDown(desiredNewSize.unsignedDivide(SURVIVOR_RATIO))); + sizes.setSurvivorSize(newSurvivorSize); + UnsignedWord desiredEdenSize = Word.zero(); - if (desiredNewSize.aboveThan(survivorSize.multiply(2))) { - desiredEdenSize = desiredNewSize.subtract(survivorSize.multiply(2)); + if (desiredNewSize.aboveThan(newSurvivorSize.multiply(2))) { + desiredEdenSize = desiredNewSize.subtract(newSurvivorSize.multiply(2)); } - edenSize = minSpaceSize(alignDown(desiredEdenSize)); - assert edenSize.aboveThan(0) && survivorSize.belowOrEqual(edenSize); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected long gcCount() { - return totalCollections; + UnsignedWord newEdenSize = minSpaceSize(alignDown(desiredEdenSize)); + sizes.setEdenSize(newEdenSize); + assert newEdenSize.aboveThan(0) && newSurvivorSize.belowOrEqual(newEdenSize); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java new file mode 100644 index 000000000000..fb24ab3383bb --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.genscavenge; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +/** + * Struct that stores all GC-related sizes. See {@link SizeParameters} for more details on its + * lifecycle. When adding fields to this struct, please also change + * {@link RawSizeParametersOnStackAccess#initialize} and {@link SizeParameters#matches} accordingly. + */ +@RawStructure +interface RawSizeParameters extends PointerBase { + @RawField + UnsignedWord getInitialEdenSize(); + + @RawField + void setInitialEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getEdenSize(); + + @RawField + void setEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxEdenSize(); + + @RawField + void setMaxEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialSurvivorSize(); + + @RawField + void setInitialSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getSurvivorSize(); + + @RawField + void setSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxSurvivorSize(); + + @RawField + void setMaxSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialYoungSize(); + + @RawField + void setInitialYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getYoungSize(); + + @RawField + void setYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxYoungSize(); + + @RawField + void setMaxYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialOldSize(); + + @RawField + void setInitialOldSize(UnsignedWord value); + + @RawField + UnsignedWord getOldSize(); + + @RawField + void setOldSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxOldSize(); + + @RawField + void setMaxOldSize(UnsignedWord value); + + @RawField + UnsignedWord getPromoSize(); + + @RawField + void setPromoSize(UnsignedWord value); + + @RawField + UnsignedWord getMinHeapSize(); + + @RawField + void setMinHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialHeapSize(); + + @RawField + void setInitialHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getHeapSize(); + + @RawField + void setHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxHeapSize(); + + @RawField + void setMaxHeapSize(UnsignedWord value); + + /** + * Either points to null or to an obsolete {@link RawSizeParameters} struct that wasn't freed + * yet. + */ + @RawField + RawSizeParameters getNext(); + + @RawField + void setNext(RawSizeParameters value); +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java new file mode 100644 index 000000000000..043ff2c5999b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.genscavenge; + +import static com.oracle.svm.core.genscavenge.AbstractCollectionPolicy.isAligned; + +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.heap.ReferenceAccess; + +import jdk.graal.compiler.word.Word; + +/** + * The methods in this class may only be used on {@link RawSizeParameters} that are allocated on the + * stack and therefore not yet visible to other threads. Please keep the methods in this class to a + * minimum. + */ +final class RawSizeParametersOnStackAccess { + static void initialize(RawSizeParameters valuesOnStack, + UnsignedWord initialEdenSize, UnsignedWord edenSize, UnsignedWord maxEdenSize, + UnsignedWord initialSurvivorSize, UnsignedWord survivorSize, UnsignedWord maxSurvivorSize, + UnsignedWord initialOldSize, UnsignedWord oldSize, UnsignedWord maxOldSize, + UnsignedWord promoSize, + UnsignedWord initialYoungSize, UnsignedWord youngSize, UnsignedWord maxYoungSize, + UnsignedWord minHeapSize, UnsignedWord initialHeapSize, UnsignedWord heapSize, UnsignedWord maxHeapSize) { + assert isAligned(maxHeapSize) && isAligned(maxYoungSize) && isAligned(initialHeapSize) && isAligned(initialEdenSize) && isAligned(initialSurvivorSize); + + assert initialEdenSize.belowOrEqual(initialYoungSize); + assert edenSize.belowOrEqual(youngSize); + assert maxEdenSize.belowOrEqual(maxYoungSize); + + assert initialSurvivorSize.belowOrEqual(initialYoungSize); + assert survivorSize.belowOrEqual(youngSize); + assert maxSurvivorSize.belowOrEqual(maxYoungSize); + + assert initialOldSize.belowOrEqual(initialHeapSize); + assert oldSize.belowOrEqual(heapSize); + assert maxOldSize.belowOrEqual(maxHeapSize); + + assert initialYoungSize.belowOrEqual(initialHeapSize); + assert youngSize.belowOrEqual(heapSize); + assert maxYoungSize.belowOrEqual(maxHeapSize); + + assert minHeapSize.belowOrEqual(initialHeapSize); + assert initialHeapSize.belowOrEqual(maxHeapSize); + assert heapSize.belowOrEqual(maxHeapSize); + assert maxHeapSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()); + + valuesOnStack.setInitialEdenSize(initialEdenSize); + valuesOnStack.setEdenSize(edenSize); + valuesOnStack.setMaxEdenSize(maxEdenSize); + + valuesOnStack.setInitialSurvivorSize(initialSurvivorSize); + valuesOnStack.setSurvivorSize(survivorSize); + valuesOnStack.setMaxSurvivorSize(maxSurvivorSize); + + valuesOnStack.setInitialYoungSize(initialYoungSize); + valuesOnStack.setYoungSize(youngSize); + valuesOnStack.setMaxYoungSize(maxYoungSize); + + valuesOnStack.setInitialOldSize(initialOldSize); + valuesOnStack.setOldSize(oldSize); + valuesOnStack.setMaxOldSize(maxOldSize); + + valuesOnStack.setPromoSize(promoSize); + + valuesOnStack.setMinHeapSize(minHeapSize); + valuesOnStack.setInitialHeapSize(initialHeapSize); + valuesOnStack.setHeapSize(heapSize); + valuesOnStack.setMaxHeapSize(maxHeapSize); + + valuesOnStack.setNext(Word.nullPointer()); + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java new file mode 100644 index 000000000000..9c61e2803f7a --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.genscavenge; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.genscavenge.AbstractCollectionPolicy.isAligned; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.word.Word; + +/** + * Once a {@link RawSizeParameters} struct is visible to other threads (see {@link #update}), it may + * only be accessed by {@link Uninterruptible} code or when the VM is at a safepoint. This is + * necessary to prevent use-after-free errors because no longer needed {@link RawSizeParameters} + * structs may be freed during a GC. + *

+ * When the VM is at a safepoint, the GC may directly update some of the values in the latest + * {@link RawSizeParameters} (see setters below). + */ +final class SizeParameters { + private static final String ACCESS_RAW_SIZE_PARAMETERS = "Prevent that RawSizeParameters are freed."; + + private volatile RawSizeParameters sizes; + + @Platforms(Platform.HOSTED_ONLY.class) + SizeParameters() { + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isInitialized() { + /* + * This only checks for non-null and doesn't access any values in the struct, so inlining + * into interruptible code is allowed. + */ + return sizes.isNonNull(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getInitialEdenSize() { + assert isInitialized(); + return sizes.getInitialEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getEdenSize() { + assert isInitialized(); + return sizes.getEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setEdenSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setEdenSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxEdenSize() { + assert isInitialized(); + return sizes.getMaxEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getInitialSurvivorSize() { + assert isInitialized(); + return sizes.getInitialSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getSurvivorSize() { + assert isInitialized(); + return sizes.getSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setSurvivorSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setSurvivorSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getMaxSurvivorSize() { + assert isInitialized(); + return sizes.getMaxSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getInitialYoungSize() { + assert isInitialized(); + return sizes.getInitialYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getYoungSize() { + assert isInitialized(); + return sizes.getYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxYoungSize() { + assert isInitialized(); + return sizes.getMaxYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getInitialOldSize() { + assert isInitialized(); + return sizes.getInitialOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getOldSize() { + assert isInitialized(); + return sizes.getOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setOldSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setOldSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getMaxOldSize() { + assert isInitialized(); + return sizes.getMaxOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getPromoSize() { + assert isInitialized(); + return sizes.getPromoSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setPromoSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setPromoSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMinHeapSize() { + assert isInitialized(); + return sizes.getMinHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxHeapSize() { + assert isInitialized(); + return sizes.getMaxHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getHeapSize() { + assert isInitialized(); + assert VMOperation.isGCInProgress() : "use only during GC"; + + return sizes.getHeapSize(); + } + + /** The caller needs to ensure that . */ + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + void update(RawSizeParameters newValuesOnStack) { + RawSizeParameters prevValues = sizes; + if (prevValues.isNonNull() && matches(prevValues, newValuesOnStack)) { + /* Nothing to do - cached params are still accurate. */ + return; + } + + /* Try allocating a struct on the C heap. */ + UnsignedWord structSize = SizeOf.unsigned(RawSizeParameters.class); + RawSizeParameters newValuesOnHeap = NullableNativeMemory.malloc(structSize, NmtCategory.GC); + VMError.guarantee(newValuesOnHeap.isNonNull(), "Out-of-memory while updating GC policy sizes."); + + /* Copy the values from the stack to the C heap. */ + UnmanagedMemoryUtil.copyForward((Pointer) newValuesOnStack, (Pointer) newValuesOnHeap, structSize); + newValuesOnHeap.setNext(prevValues); + + /* + * Publish the new struct via a volatile store. Once the data is published, other threads + * need to see a fully initialized struct right away. This is guaranteed by the implicit + * STORE_STORE barrier before the volatile write. + */ + sizes = newValuesOnHeap; + + assert isAligned(getMaxSurvivorSize()) && isAligned(getInitialYoungSize()) && isAligned(getInitialOldSize()) && isAligned(getMaxOldSize()); + assert getMaxSurvivorSize().belowThan(getMaxYoungSize()); + assert getMaxYoungSize().add(getMaxOldSize()).equal(getMaxHeapSize()); + assert getInitialEdenSize().add(getInitialSurvivorSize().multiply(2)).equal(getInitialYoungSize()); + assert getInitialYoungSize().add(getInitialOldSize()).equal(sizes.getInitialHeapSize()); + } + + /** + * Frees no longer needed {@link RawSizeParameters}, so that only the most recent one remains. + */ + public void freeUnusedSizeParameters() { + assert isInitialized(); + assert VMOperation.isGCInProgress() : "would need to be uninterruptible otherwise"; + + RawSizeParameters cur = sizes; + freeSizeParameters(cur.getNext()); + cur.setNext(Word.nullPointer()); + } + + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + assert VMThreads.isTearingDown(); + + freeSizeParameters(sizes); + sizes = Word.nullPointer(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void freeSizeParameters(RawSizeParameters first) { + assert VMOperation.isGCInProgress() || VMThreads.isTearingDown(); + + RawSizeParameters cur = first; + while (cur.isNonNull()) { + RawSizeParameters next = cur.getNext(); + NullableNativeMemory.free(cur); + cur = next; + } + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + private static boolean matches(RawSizeParameters a, RawSizeParameters b) { + return a.getInitialEdenSize() == b.getInitialEdenSize() && + a.getEdenSize() == b.getEdenSize() && + a.getMaxEdenSize() == b.getMaxEdenSize() && + + a.getInitialSurvivorSize() == b.getInitialSurvivorSize() && + a.getSurvivorSize() == b.getSurvivorSize() && + a.getMaxSurvivorSize() == b.getMaxSurvivorSize() && + + a.getInitialYoungSize() == b.getInitialYoungSize() && + a.getYoungSize() == b.getYoungSize() && + a.getMaxYoungSize() == b.getMaxYoungSize() && + + a.getInitialOldSize() == b.getInitialOldSize() && + a.getOldSize() == b.getOldSize() && + a.getMaxOldSize() == b.getMaxOldSize() && + + a.getPromoSize() == b.getPromoSize() && + + a.getMinHeapSize() == b.getMinHeapSize() && + a.getInitialHeapSize() == b.getInitialHeapSize() && + a.getHeapSize() == b.getHeapSize() && + a.getMaxHeapSize() == b.getMaxHeapSize(); + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index 10d2115c534c..be552833e714 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -78,11 +78,39 @@ import jdk.graal.compiler.word.Word; /** - * Bump-pointer allocation from thread-local top and end Pointers. Many of these methods are called - * from allocation snippets, so they can not do anything fancy. It happens that prefetch - * instructions access memory outside the TLAB. At the moment, this is not an issue as we only - * support architectures where the prefetch instructions never cause a segfault, even if they try to - * access memory that is not accessible. + * Implements the thread-local allocation logic for serial and epsilon GC. + *

+ * Multiple threads may execute the methods in this class concurrently. All code transitively + * reachable from these methods can be executed as a side effect of any Java heap allocation. To + * prevent hard to debug transient issues, we execute as little code as possible in these methods. + *

+ * Executing complex logic in the allocation slow path can modify shared global state, causing + * issues that look similar to race conditions but that can even happen in single-threaded + * environments. For example: + * + * 

+ * {@code
+ * private static Object singleton;
+ *
+ * private static synchronized Object createSingleton() {
+ *     if (singleton == null) {
+ *         Object o = new Object();
+ *         // If the allocation above enters the slow path, and if that slow
+ *         // path executes code that calls createSingleton() as well, then
+ *         // the assertion below will fail because the singleton already
+ *         // got initialized by the same thread in the meanwhile.
+ *         assert singleton == null;
+ *         singleton = o;
+ *     }
+ *     return singleton;
+ * }
+ * }
+ *
+ * + * The allocation fast-path emits prefetch instructions. Those instructions may try to access memory + * that is outside the TLAB and therefore not necessarily accessible. At the moment, this is not an + * issue as we only support architectures where prefetch instructions never cause segfaults, even if + * they access memory that is not accessible. */ public final class ThreadLocalAllocation { @RawStructure @@ -183,46 +211,14 @@ static UnsignedWord getAlignedAllocatedBytes(IsolateThread thread) { return allocatedAlignedBytes.getVolatile(thread); } - /** - * NOTE: Multiple threads may execute this method concurrently. All code that is transitively - * reachable from this method may get executed as a side effect of an allocation slow path. To - * prevent hard to debug transient issues, we execute as little code as possible in this method. - * - * If the executed code is too complex, then it can happen that we unexpectedly change some - * shared global state as a side effect of an allocation. This may result in issues that look - * similar to races but that can even happen in single-threaded environments, e.g.: - * - * 
-     * {@code
-     * private static Object singleton;
-     *
-     * private static synchronized Object createSingleton() {
-     *     if (singleton == null) {
-     *         Object o = new Object();
-     *         // If the allocation above enters the allocation slow path code, and executes a
-     *         // complex slow path hook, then it is possible that createSingleton() gets
-     *         // recursively execute by the current thread. So, the assertion below may fail
-     *         // because the singleton got already initialized by the same thread in the meanwhile.
-     *         assert singleton == null;
-     *         singleton = o;
-     *     }
-     *     return result;
-     * }
-     * }
-     *
- */ - private static void runSlowPathHooks() { - GCImpl.getPolicy().updateSizeParameters(); - } - public static Object slowPathNewInstance(Word objectHeader) { DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(objectHeader); UnsignedWord size = LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()); Object result = allocateInstanceInCurrentTlab(hub, size); if (result == null) { + GCImpl.getPolicy().ensureSizeParametersInitialized(); result = slowPathNewInstanceWithoutAllocating(hub, size); - runSlowPathHooks(); sampleSlowPathAllocation(result, size, Integer.MIN_VALUE); } return result; @@ -269,10 +265,7 @@ public static Object slowPathNewArrayLikeObject(Word objectHeader, int length, b } Object result = slowPathNewArrayLikeObjectWithoutAllocating(hub, length, size, podReferenceMap); - - runSlowPathHooks(); sampleSlowPathAllocation(result, size, length); - return result; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 5fbb27a1b13b..86873366e3c8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -372,6 +372,11 @@ public long getLongOptionValue(int index) { return parsedOptionValues[index]; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void setLongOptionValue(int optionIndex, long newValue) { + parsedOptionValues[optionIndex] = newValue; + } + protected CCharPointer getCCharPointerOptionValue(int index) { return Word.pointer(parsedOptionValues[index]); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java index f4d6fc574847..0d2a3d2fba4f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java @@ -53,7 +53,12 @@ public class SubstrateGCOptions { protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { if (!SubstrateUtil.HOSTED) { HeapSizeVerifier.verifyMinHeapSizeAgainstMaxAddressSpaceSize(Word.unsigned(newValue)); + + /* Update the isolate argument parser value. */ + int optionIndex = IsolateArgumentParser.getOptionIndex(MinHeapSize); + IsolateArgumentParser.singleton().setLongOptionValue(optionIndex, newValue); } + super.onValueUpdate(values, oldValue, newValue); } }; @@ -64,7 +69,12 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { if (!SubstrateUtil.HOSTED) { HeapSizeVerifier.verifyMaxHeapSizeAgainstMaxAddressSpaceSize(Word.unsigned(newValue)); + + /* Update the isolate argument parser value. */ + int optionIndex = IsolateArgumentParser.getOptionIndex(MaxHeapSize); + IsolateArgumentParser.singleton().setLongOptionValue(optionIndex, newValue); } + super.onValueUpdate(values, oldValue, newValue); } }; @@ -75,7 +85,12 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { if (!SubstrateUtil.HOSTED) { HeapSizeVerifier.verifyMaxNewSizeAgainstMaxAddressSpaceSize(Word.unsigned(newValue)); + + /* Update the isolate argument parser value. */ + int optionIndex = IsolateArgumentParser.getOptionIndex(MaxNewSize); + IsolateArgumentParser.singleton().setLongOptionValue(optionIndex, newValue); } + super.onValueUpdate(values, oldValue, newValue); } }; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index b9ee9f1ca631..02cbe0b68562 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -42,7 +42,7 @@ import com.oracle.svm.core.hub.PredefinedClassesSupport; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.snippets.KnownIntrinsics; import jdk.graal.compiler.api.replacements.Fold; @@ -232,7 +232,7 @@ public Pointer getImageHeapStart() { /** * Notify the GC that the value of a GC-relevant option changed. */ - public abstract void optionValueChanged(RuntimeOptionKey key); + public abstract void optionValueChanged(NotifyGCRuntimeOptionKey key); /** * Returns the number of bytes that were allocated by the given thread. The caller of this diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java index 5ec777e33cca..680dcc4d5afa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java @@ -24,11 +24,11 @@ */ package com.oracle.svm.core.option; +import java.util.function.Consumer; + import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.heap.Heap; -import java.util.function.Consumer; - /** * Notifies the {@link Heap} implementation after the value of the option has changed. */ diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java index bbefa6aa82f7..4e1c679b8aa2 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java @@ -49,7 +49,7 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads.SafepointBehavior; import com.oracle.svm.core.util.VMError; @@ -329,7 +329,7 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH } @Override - public void optionValueChanged(RuntimeOptionKey key) { + public void optionValueChanged(NotifyGCRuntimeOptionKey key) { // Nothing to do } diff --git a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java index 239b63507d0d..fe3862dc942d 100644 --- a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java +++ b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java @@ -41,7 +41,7 @@ import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.word.Word; @@ -209,7 +209,7 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH } @Override - public void optionValueChanged(RuntimeOptionKey key) { + public void optionValueChanged(NotifyGCRuntimeOptionKey key) { } @Override