From 811cf88f4ef4af54176e41fd14bac26e6189279c Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 17 Dec 2025 10:55:09 +0100 Subject: [PATCH 1/5] Fix afterValueUpdate(). --- .../jdk/graal/compiler/options/ModifiableOptionValues.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java index 2b8c35a41b9b..31e5150650f1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/options/ModifiableOptionValues.java @@ -106,6 +106,12 @@ public void update(UnmodifiableEconomicMap, Object> values) { } } } while (!v.compareAndSet(expect, newMap)); + + UnmodifiableMapCursor, Object> cursor = values.getEntries(); + while (cursor.advance()) { + OptionKey key = cursor.getKey(); + key.afterValueUpdate(); + } } @Override From d1285aa354dc9fccbf7846e7c2e6096280b14bdf Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 17 Dec 2025 11:07:30 +0100 Subject: [PATCH 2/5] Use IsolateArgumentParser values. --- .../genscavenge/AbstractCollectionPolicy.java | 18 +++++++++++++++--- .../genscavenge/DynamicCollectionPolicy.java | 4 +--- .../oracle/svm/core/IsolateArgumentParser.java | 5 +++++ .../oracle/svm/core/SubstrateGCOptions.java | 15 +++++++++++++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index 9ff4478e676e..37a8f9c6daee 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -28,6 +28,7 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.IsolateArgumentParser; import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.PhysicalMemory; @@ -359,7 +360,7 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { maxHeap = UnsignedUtils.clamp(alignDown(maxHeap), minAllSpaces, heapSizeLimit); UnsignedWord maxYoung; - long optionMaxYoung = SubstrateGCOptions.MaxNewSize.getValue(); + long optionMaxYoung = getMaxNewSizeOptionValue(); if (optionMaxYoung > 0L) { maxYoung = Word.unsigned(optionMaxYoung); } else if (SerialAndEpsilonGCOptions.MaximumYoungGenerationSizePercent.hasBeenSet()) { @@ -375,7 +376,7 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { (maxHeap.belowOrEqual(unadjustedMaxHeap) || unadjustedMaxHeap.belowThan(minAllSpaces))); UnsignedWord minHeap = Word.zero(); - long optionMin = SubstrateGCOptions.MinHeapSize.getValue(); + long optionMin = getMinHeapSizeOptionValue(); if (optionMin > 0L) { minHeap = Word.unsigned(optionMin); } @@ -419,7 +420,18 @@ protected UnsignedWord getYoungSizeLimit(UnsignedWord maxHeap) { } protected long getMaximumHeapSizeOptionValue() { - return SubstrateGCOptions.MaxHeapSize.getValue(); + int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MaxHeapSize); + return IsolateArgumentParser.singleton().getLongOptionValue(optionIndex); + } + + protected static long getMaxNewSizeOptionValue() { + int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MaxNewSize); + return IsolateArgumentParser.singleton().getLongOptionValue(optionIndex); + } + + protected static long getMinHeapSizeOptionValue() { + int optionIndex = IsolateArgumentParser.getOptionIndex(SubstrateGCOptions.MinHeapSize); + return IsolateArgumentParser.singleton().getLongOptionValue(optionIndex); } protected UnsignedWord getInitialHeapSize() { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java index b61c9418016f..8b88dedb227f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java @@ -27,7 +27,6 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.heap.DynamicHeapSizeManager; /** @@ -46,8 +45,7 @@ protected long getMaximumHeapSizeOptionValue() { if (ImageSingletons.contains(DynamicHeapSizeManager.class)) { return ImageSingletons.lookup(DynamicHeapSizeManager.class).maxHeapSize().rawValue(); } - - return SubstrateGCOptions.MaxHeapSize.getValue(); + return super.getMaximumHeapSizeOptionValue(); } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java index 5fbb27a1b13b..86873366e3c8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/IsolateArgumentParser.java @@ -372,6 +372,11 @@ public long getLongOptionValue(int index) { return parsedOptionValues[index]; } + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public void setLongOptionValue(int optionIndex, long newValue) { + parsedOptionValues[optionIndex] = newValue; + } + protected CCharPointer getCCharPointerOptionValue(int index) { return Word.pointer(parsedOptionValues[index]); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java index f4d6fc574847..0d2a3d2fba4f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateGCOptions.java @@ -53,7 +53,12 @@ public class SubstrateGCOptions { protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { if (!SubstrateUtil.HOSTED) { HeapSizeVerifier.verifyMinHeapSizeAgainstMaxAddressSpaceSize(Word.unsigned(newValue)); + + /* Update the isolate argument parser value. */ + int optionIndex = IsolateArgumentParser.getOptionIndex(MinHeapSize); + IsolateArgumentParser.singleton().setLongOptionValue(optionIndex, newValue); } + super.onValueUpdate(values, oldValue, newValue); } }; @@ -64,7 +69,12 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { if (!SubstrateUtil.HOSTED) { HeapSizeVerifier.verifyMaxHeapSizeAgainstMaxAddressSpaceSize(Word.unsigned(newValue)); + + /* Update the isolate argument parser value. */ + int optionIndex = IsolateArgumentParser.getOptionIndex(MaxHeapSize); + IsolateArgumentParser.singleton().setLongOptionValue(optionIndex, newValue); } + super.onValueUpdate(values, oldValue, newValue); } }; @@ -75,7 +85,12 @@ protected void onValueUpdate(EconomicMap, Object> values, Long oldV protected void onValueUpdate(EconomicMap, Object> values, Long oldValue, Long newValue) { if (!SubstrateUtil.HOSTED) { HeapSizeVerifier.verifyMaxNewSizeAgainstMaxAddressSpaceSize(Word.unsigned(newValue)); + + /* Update the isolate argument parser value. */ + int optionIndex = IsolateArgumentParser.getOptionIndex(MaxNewSize); + IsolateArgumentParser.singleton().setLongOptionValue(optionIndex, newValue); } + super.onValueUpdate(values, oldValue, newValue); } }; From 70f997d9054e61f2790438eea79c284aa59e669e Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Wed, 17 Dec 2025 11:17:29 +0100 Subject: [PATCH 3/5] Don't recompute GC policy sizes when an option changes that is not relevant for the GC policy. --- .../com/oracle/svm/core/genscavenge/HeapImpl.java | 15 ++++++++++++--- .../src/com/oracle/svm/core/heap/Heap.java | 4 ++-- .../svm/hosted/webimage/wasm/gc/WasmHeap.java | 4 ++-- .../oracle/svm/webimage/heap/WebImageJSHeap.java | 4 ++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index e1e7efb6b161..9a03ae38a238 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -76,6 +76,7 @@ import com.oracle.svm.core.metaspace.Metaspace; import com.oracle.svm.core.nodes.CFunctionEpilogueNode; import com.oracle.svm.core.nodes.CFunctionPrologueNode; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.option.RuntimeOptionKey; import com.oracle.svm.core.snippets.KnownIntrinsics; import com.oracle.svm.core.thread.PlatformThreads; @@ -701,10 +702,18 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH } @Override - public void optionValueChanged(RuntimeOptionKey key) { - if (!SubstrateUtil.HOSTED) { - GCImpl.getPolicy().updateSizeParameters(); + public void optionValueChanged(NotifyGCRuntimeOptionKey key) { + if (SubstrateUtil.HOSTED || isIrrelevantForGCPolicy(key)) { + return; } + + GCImpl.getPolicy().updateSizeParameters(); + } + + private static boolean isIrrelevantForGCPolicy(RuntimeOptionKey key) { + return key == SubstrateGCOptions.DisableExplicitGC || + key == SubstrateGCOptions.PrintGC || + key == SubstrateGCOptions.VerboseGC; } @Override diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java index b9ee9f1ca631..02cbe0b68562 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/Heap.java @@ -42,7 +42,7 @@ import com.oracle.svm.core.hub.PredefinedClassesSupport; import com.oracle.svm.core.identityhashcode.IdentityHashCodeSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.snippets.KnownIntrinsics; import jdk.graal.compiler.api.replacements.Fold; @@ -232,7 +232,7 @@ public Pointer getImageHeapStart() { /** * Notify the GC that the value of a GC-relevant option changed. */ - public abstract void optionValueChanged(RuntimeOptionKey key); + public abstract void optionValueChanged(NotifyGCRuntimeOptionKey key); /** * Returns the number of bytes that were allocated by the given thread. The caller of this diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java index bbefa6aa82f7..4e1c679b8aa2 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmHeap.java @@ -49,7 +49,7 @@ import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads.SafepointBehavior; import com.oracle.svm.core.util.VMError; @@ -329,7 +329,7 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH } @Override - public void optionValueChanged(RuntimeOptionKey key) { + public void optionValueChanged(NotifyGCRuntimeOptionKey key) { // Nothing to do } diff --git a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java index 239b63507d0d..fe3862dc942d 100644 --- a/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java +++ b/web-image/src/com.oracle.svm.webimage/src/com/oracle/svm/webimage/heap/WebImageJSHeap.java @@ -41,7 +41,7 @@ import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.heap.RuntimeCodeInfoGCSupport; import com.oracle.svm.core.log.Log; -import com.oracle.svm.core.option.RuntimeOptionKey; +import com.oracle.svm.core.option.NotifyGCRuntimeOptionKey; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.word.Word; @@ -209,7 +209,7 @@ public boolean printLocationInfo(Log log, UnsignedWord value, boolean allowJavaH } @Override - public void optionValueChanged(RuntimeOptionKey key) { + public void optionValueChanged(NotifyGCRuntimeOptionKey key) { } @Override From 40b566ff15b8834fb907064fffabab3695045e5e Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 27 Nov 2025 10:07:44 +0100 Subject: [PATCH 4/5] Remove the Java heap allocation in AbstractCollectionPolicy.updateSizeParameters(). --- .../genscavenge/AbstractCollectionPolicy.java | 668 +++++++++++++----- .../genscavenge/AdaptiveCollectionPolicy.java | 69 +- .../core/genscavenge/CollectionPolicy.java | 4 + .../genscavenge/DynamicCollectionPolicy.java | 15 +- .../oracle/svm/core/genscavenge/GCImpl.java | 5 + .../oracle/svm/core/genscavenge/HeapImpl.java | 5 +- .../svm/core/genscavenge/HeapParameters.java | 8 +- .../genscavenge/LibGraalCollectionPolicy.java | 16 +- .../ProportionateSpacesPolicy.java | 60 +- .../genscavenge/ThreadLocalAllocation.java | 74 +- .../core/option/NotifyGCRuntimeOptionKey.java | 4 +- 11 files changed, 635 insertions(+), 293 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index 37a8f9c6daee..dc1378717734 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -24,32 +24,46 @@ */ package com.oracle.svm.core.genscavenge; +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; + import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.StackValue; +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.IsolateArgumentParser; import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.jdk.UninterruptibleUtils; +import com.oracle.svm.core.locks.VMMutex; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.CommittedMemoryProvider; -import com.oracle.svm.core.thread.JavaSpinLockUtils; +import com.oracle.svm.core.stack.StackOverflowCheck; +import com.oracle.svm.core.thread.RecurringCallbackSupport; import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; import jdk.graal.compiler.api.replacements.Fold; import jdk.graal.compiler.word.Word; -import jdk.internal.misc.Unsafe; /** * Note that a lot of methods in this class are final. Subclasses may only override certain methods * to avoid inconsistencies between the different heap size values. */ abstract class AbstractCollectionPolicy implements CollectionPolicy { + private static final VMMutex MUTEX = new VMMutex("AbstractCollectionPolicy.sizeParams"); protected static final int MIN_SPACE_SIZE_AS_NUMBER_OF_ALIGNED_CHUNKS = 8; @@ -62,9 +76,6 @@ static int getMaxSurvivorSpaces(Integer userValue) { return (userValue != null) ? userValue : AbstractCollectionPolicy.MAX_TENURING_THRESHOLD; } - private static final Unsafe U = Unsafe.getUnsafe(); - private static final long SIZES_UPDATE_LOCK_OFFSET = U.objectFieldOffset(AbstractCollectionPolicy.class, "sizesUpdateLock"); - /* * Constants that can be made options if desirable. These are -XX options in HotSpot, refer to * their descriptions for details. The values are HotSpot defaults unless labeled otherwise. @@ -85,29 +96,27 @@ static int getMaxSurvivorSpaces(Integer userValue) { protected static final int NEW_RATIO = 2; protected final AdaptiveWeightedAverage avgYoungGenAlignedChunkFraction = new AdaptiveWeightedAverage(ADAPTIVE_TIME_WEIGHT); + protected final SizeParameters sizes = new SizeParameters(); private final int initialNewRatio; - protected UnsignedWord survivorSize; - protected UnsignedWord edenSize; - protected UnsignedWord promoSize; - protected UnsignedWord oldSize; protected int tenuringThreshold; - protected volatile SizeParameters sizes = null; - @SuppressWarnings("unused") private volatile int sizesUpdateLock; - protected AbstractCollectionPolicy(int initialNewRatio, int initialTenuringThreshold) { this.initialNewRatio = initialNewRatio; this.tenuringThreshold = UninterruptibleUtils.Math.clamp(initialTenuringThreshold, 1, HeapParameters.getMaxSurvivorSpaces() + 1); } + @Override + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + sizes.tearDown(); + } + @Override public boolean shouldCollectOnAllocation() { - if (sizes == null) { - return false; // updateSizeParameters() has never been called - } + guaranteeSizeParametersInitialized(); UnsignedWord edenUsed = HeapImpl.getAccounting().getEdenUsedBytes(); - return edenUsed.aboveOrEqual(edenSize); + return edenUsed.aboveOrEqual(sizes.getEdenSize()); } @Override @@ -148,155 +157,96 @@ static UnsignedWord minSpaceSize(UnsignedWord size) { @Override public void ensureSizeParametersInitialized() { - if (sizes == null) { + if (!sizes.isInitialized()) { updateSizeParameters(); } } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected void guaranteeSizeParametersInitialized() { - VMError.guarantee(sizes != null); + VMError.guarantee(sizes.isInitialized()); } @Override public void updateSizeParameters() { - /* - * Read the old object before computing the new values. Otherwise, we risk reusing an - * outdated SizeParameters object. - */ - SizeParameters prevParams = sizes; - SizeParameters newParams = computeSizeParameters(prevParams); - if (prevParams != null && newParams.equal(prevParams)) { - return; // nothing to do - } - updateSizeParameters0(newParams, prevParams); - guaranteeSizeParametersInitialized(); // sanity - } - - @Uninterruptible(reason = "Holding the spin lock at a safepoint can result in deadlocks.") - private void updateSizeParameters0(SizeParameters newParams, SizeParameters prevParams) { - /* - * We use a primitive spin lock because at this point, the current thread might be unable to - * use a Java lock (e.g. no Thread object yet), and the critical section is short, so we do - * not want to suspend and wake up threads for it. - */ - JavaSpinLockUtils.lockNoTransition(this, SIZES_UPDATE_LOCK_OFFSET); + StackOverflowCheck.singleton().makeYellowZoneAvailable(); try { - if (sizes != prevParams) { - /* - * Some other thread beat us and we cannot tell if our values or their values are - * newer, so back off - any newer values will be applied eventually. - */ - return; - } - updateSizeParametersLocked(newParams, prevParams); - } finally { - JavaSpinLockUtils.unlock(this, SIZES_UPDATE_LOCK_OFFSET); - } - } + MUTEX.lock(); + try { + assert RecurringCallbackSupport.isCallbackUnsupportedOrTimerSuspended() : "recurring callbacks could trigger recursive locking, which isn't supported"; - @Uninterruptible(reason = "Holding the spin lock at a safepoint can result in deadlocks. Updating the size parameters must be atomic with regard to garbage collection.") - private void updateSizeParametersLocked(SizeParameters newParams, SizeParameters prevParams) { - sizes = newParams; + RawSizeParameters newValuesOnStack = StackValue.get(RawSizeParameters.class); + computeSizeParameters(newValuesOnStack); - if (prevParams == null || gcCount() == 0) { - survivorSize = newParams.initialSurvivorSize; - edenSize = newParams.initialEdenSize; - oldSize = newParams.initialOldSize(); - promoSize = UnsignedUtils.min(edenSize, oldSize); + sizes.update(newValuesOnStack); + } finally { + MUTEX.unlock(); + } + } finally { + StackOverflowCheck.singleton().protectYellowZone(); } - - /* - * NOTE: heap limits can change when options are updated at runtime or once the physical - * memory size becomes known. This means that we start off with sizes which can cause higher - * GC costs initially, and when shrinking the heap, that previously computed values such as - * GC costs and intervals and survived/promoted objects are likely no longer representative. - * - * We assume that such changes happen very early on and values then adapt reasonably quick, - * but we must still ensure that computations can handle it (for example, no overflows). - */ - survivorSize = UnsignedUtils.min(survivorSize, newParams.maxSurvivorSize()); - edenSize = UnsignedUtils.min(edenSize, getMaximumEdenSize()); - oldSize = UnsignedUtils.min(oldSize, newParams.maxOldSize()); - promoSize = UnsignedUtils.min(promoSize, newParams.maxOldSize()); } @Override public final UnsignedWord getInitialEdenSize() { - guaranteeSizeParametersInitialized(); - return sizes.initialEdenSize; + return sizes.getInitialEdenSize(); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public final UnsignedWord getMaximumEdenSize() { - guaranteeSizeParametersInitialized(); - return alignDown(sizes.maxYoungSize.subtract(survivorSize.multiply(2))); + return sizes.getMaxEdenSize(); } @Override public final UnsignedWord getMaximumHeapSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxHeapSize; + return sizes.getMaxHeapSize(); } @Override public final UnsignedWord getMaximumYoungGenerationSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxYoungSize; + return sizes.getMaxYoungSize(); } @Override public final UnsignedWord getInitialSurvivorSize() { - guaranteeSizeParametersInitialized(); - return sizes.initialSurvivorSize; + return sizes.getInitialSurvivorSize(); } @Override public final UnsignedWord getMaximumSurvivorSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxSurvivorSize(); + return sizes.getMaxSurvivorSize(); } @Override public final UnsignedWord getCurrentHeapCapacity() { - assert VMOperation.isGCInProgress() : "use only during GC"; - guaranteeSizeParametersInitialized(); - return edenSize.add(survivorSize).add(oldSize); + return sizes.getCurrentHeapCapacity(); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public final UnsignedWord getSurvivorSpacesCapacity() { - assert VMOperation.isGCInProgress() : "use only during GC"; - guaranteeSizeParametersInitialized(); - return survivorSize; + return sizes.getSurvivorSize(); } @Override - @Uninterruptible(reason = "Ensure reading a consistent value.") public final UnsignedWord getYoungGenerationCapacity() { - guaranteeSizeParametersInitialized(); - return edenSize.add(survivorSize); + return sizes.getYoungSize(); } @Override public final UnsignedWord getInitialOldSize() { - guaranteeSizeParametersInitialized(); - return sizes.initialOldSize(); + return sizes.getInitialOldSize(); } @Override public final UnsignedWord getMaximumOldSize() { - guaranteeSizeParametersInitialized(); - return sizes.maxOldSize(); + return sizes.getMaxOldSize(); } @Override public final UnsignedWord getOldGenerationCapacity() { - guaranteeSizeParametersInitialized(); - return oldSize; + return sizes.getOldSize(); } @Override @@ -308,13 +258,13 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { * survivor spaces in a future collection. We could alternatively return * getCurrentHeapCapacity() to have chunks ready during full GCs as well. */ - UnsignedWord total = edenSize.add(HeapImpl.getAccounting().getSurvivorUsedBytes()); + UnsignedWord total = sizes.getEdenSize().add(HeapImpl.getAccounting().getSurvivorUsedBytes()); double alignedFraction = Math.min(1, Math.max(0, avgYoungGenAlignedChunkFraction.getAverage())); return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(total) * alignedFraction); } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; @@ -322,6 +272,8 @@ public int getTenuringAge() { @Override public void onCollectionBegin(boolean completeCollection, long beginNanoTime) { + sizes.freeUnusedSizeParameters(); + // Capture the fraction of bytes in aligned chunks at the start to include all allocated // (also dead) objects, because we use it to reserve aligned chunks for future allocations UnsignedWord youngChunkBytes = GCImpl.getAccounting().getYoungChunkBytesBefore(); @@ -333,15 +285,13 @@ public void onCollectionBegin(boolean completeCollection, long beginNanoTime) { @Override public final UnsignedWord getMinimumHeapSize() { - return sizes.minHeapSize; + return sizes.getMinHeapSize(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract long gcCount(); - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/shared/genArguments.cpp#L195-L310") + @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L146-L168") - protected SizeParameters computeSizeParameters(SizeParameters existing) { + private void computeSizeParameters(RawSizeParameters newParams) { UnsignedWord minYoungSpaces = minSpaceSize(); // eden if (HeapParameters.getMaxSurvivorSpaces() > 0) { minYoungSpaces = minYoungSpaces.add(minSpaceSize().multiply(2)); // survivor from and to @@ -392,6 +342,7 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { initialYoung = initialHeap.unsignedDivide(initialNewRatio + 1); initialYoung = UnsignedUtils.clamp(alignUp(initialYoung), minYoungSpaces, maxYoung); } + UnsignedWord initialSurvivor = Word.zero(); if (HeapParameters.getMaxSurvivorSpaces() > 0) { /* @@ -405,10 +356,49 @@ protected SizeParameters computeSizeParameters(SizeParameters existing) { initialSurvivor = initialYoung.unsignedDivide(AbstractCollectionPolicy.INITIAL_SURVIVOR_RATIO); initialSurvivor = minSpaceSize(alignDown(initialSurvivor)); } + UnsignedWord initialEden = initialYoung.subtract(initialSurvivor.multiply(2)); initialEden = minSpaceSize(alignDown(initialEden)); - return SizeParameters.get(existing, maxHeap, maxYoung, initialHeap, initialEden, initialSurvivor, minHeap); + UnsignedWord maxSurvivorSize = Word.zero(); + UnsignedWord maxEdenSize = maxYoung; + if (HeapParameters.getMaxSurvivorSpaces() > 0) { + maxSurvivorSize = maxYoung.unsignedDivide(MIN_SURVIVOR_RATIO); + maxSurvivorSize = minSpaceSize(alignDown(maxSurvivorSize)); + maxEdenSize = maxYoung.subtract(minSpaceSize().multiply(2)); + } + + UnsignedWord maxOldSize = maxHeap.subtract(maxYoung); + + UnsignedWord survivorSize; + UnsignedWord edenSize; + UnsignedWord oldSize; + UnsignedWord promoSize; + if (sizes.isInitialized()) { + /* Copy and limit existing values. */ + survivorSize = UnsignedUtils.min(sizes.getSurvivorSize(), maxSurvivorSize); + edenSize = UnsignedUtils.min(sizes.getEdenSize(), maxEdenSize); + oldSize = UnsignedUtils.min(sizes.getOldSize(), maxOldSize); + promoSize = UnsignedUtils.min(sizes.getPromoSize(), maxOldSize); + } else { + /* Set initial values. */ + survivorSize = initialSurvivor; + edenSize = initialEden; + oldSize = initialHeap.subtract(initialYoung); + promoSize = UnsignedUtils.min(edenSize, oldSize); + } + + UnsignedWord initialOldSize = initialHeap.subtract(initialYoung); + UnsignedWord youngSize = edenSize.add(survivorSize); + UnsignedWord heapSize = edenSize.add(survivorSize).add(oldSize); + + RawSizeParametersAccess.initialize(newParams, + initialEden, edenSize, maxEdenSize, + initialSurvivor, survivorSize, maxSurvivorSize, + initialOldSize, oldSize, maxOldSize, + promoSize, + initialYoung, youngSize, maxYoung, + minHeap, initialHeap, heapSize, maxHeap); } protected UnsignedWord getHeapSizeLimit() { @@ -438,76 +428,430 @@ protected UnsignedWord getInitialHeapSize() { return AbstractCollectionPolicy.INITIAL_HEAP_SIZE; } + /** + * Once a {@link RawSizeParameters} struct is visible to other threads (see {@link #update}), it + * may only be accessed by {@link Uninterruptible} code or when the VM is at a safepoint. This + * is necessary to prevent use-after-free errors because no longer needed + * {@link RawSizeParameters} structs may be freed during a GC. + *

+ * When the VM is at a safepoint, the GC may directly update some of the values in the latest + * {@link RawSizeParameters} (see setters below). + */ protected static final class SizeParameters { - final UnsignedWord maxHeapSize; - final UnsignedWord maxYoungSize; - final UnsignedWord initialHeapSize; - final UnsignedWord initialEdenSize; - final UnsignedWord initialSurvivorSize; - final UnsignedWord minHeapSize; - - static SizeParameters get(SizeParameters existing, UnsignedWord maxHeap, UnsignedWord maxYoung, UnsignedWord initialHeap, - UnsignedWord initialEden, UnsignedWord initialSurvivor, UnsignedWord minHeap) { - if (existing != null && existing.matches(maxHeap, maxYoung, initialHeap, initialEden, initialSurvivor, minHeap)) { - return existing; - } - return new SizeParameters(maxHeap, maxYoung, initialHeap, initialEden, initialSurvivor, minHeap); + private static final String ACCESS_RAW_SIZE_PARAMETERS = "Prevent that RawSizeParameters are freed."; + + private volatile RawSizeParameters sizes; + + @Platforms(Platform.HOSTED_ONLY.class) + private SizeParameters() { } - private SizeParameters(UnsignedWord maxHeapSize, UnsignedWord maxYoungSize, UnsignedWord initialHeapSize, - UnsignedWord initialEdenSize, UnsignedWord initialSurvivorSize, UnsignedWord minHeapSize) { - this.maxHeapSize = maxHeapSize; - this.maxYoungSize = maxYoungSize; - this.initialHeapSize = initialHeapSize; - this.initialEdenSize = initialEdenSize; - this.initialSurvivorSize = initialSurvivorSize; - this.minHeapSize = minHeapSize; + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isInitialized() { + /* + * This only checks for non-null and doesn't access any values in the struct, so + * inlining into interruptible code is allowed. + */ + return sizes.isNonNull(); + } - assert isAligned(maxHeapSize) && isAligned(maxYoungSize) && isAligned(initialHeapSize) && isAligned(initialEdenSize) && isAligned(initialSurvivorSize); - assert isAligned(maxSurvivorSize()) && isAligned(initialYoungSize()) && isAligned(initialOldSize()) && isAligned(maxOldSize()); + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getInitialEdenSize() { + assert isInitialized(); + return sizes.getInitialEdenSize(); + } - assert initialHeapSize.belowOrEqual(maxHeapSize); - assert maxSurvivorSize().belowThan(maxYoungSize); - assert maxYoungSize.add(maxOldSize()).equal(maxHeapSize); - assert maxHeapSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()); - assert initialEdenSize.add(initialSurvivorSize.multiply(2)).equal(initialYoungSize()); - assert initialYoungSize().add(initialOldSize()).equal(initialHeapSize); + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getEdenSize() { + assert isInitialized(); + return sizes.getEdenSize(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setEdenSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setEdenSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxEdenSize() { + assert isInitialized(); + return sizes.getMaxEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getInitialSurvivorSize() { + assert isInitialized(); + return sizes.getInitialSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getSurvivorSize() { + assert isInitialized(); + return sizes.getSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setSurvivorSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setSurvivorSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") - UnsignedWord maxSurvivorSize() { - if (HeapParameters.getMaxSurvivorSpaces() == 0) { - return Word.zero(); + UnsignedWord getMaxSurvivorSize() { + assert isInitialized(); + return sizes.getMaxSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getInitialYoungSize() { + assert isInitialized(); + return sizes.getInitialYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getYoungSize() { + assert isInitialized(); + return sizes.getYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxYoungSize() { + assert isInitialized(); + return sizes.getMaxYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getInitialOldSize() { + assert isInitialized(); + return sizes.getInitialOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getOldSize() { + assert isInitialized(); + return sizes.getOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setOldSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setOldSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getMaxOldSize() { + assert isInitialized(); + return sizes.getMaxOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getPromoSize() { + assert isInitialized(); + return sizes.getPromoSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setPromoSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setPromoSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMinHeapSize() { + assert isInitialized(); + return sizes.getMinHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxHeapSize() { + assert isInitialized(); + return sizes.getMaxHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getCurrentHeapCapacity() { + assert isInitialized(); + assert VMOperation.isGCInProgress() : "use only during GC"; + + return sizes.getHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + private void update(RawSizeParameters newValuesOnStack) { + RawSizeParameters prevValues = sizes; + if (prevValues.isNonNull() && matches(prevValues, newValuesOnStack)) { + /* Nothing to do - cached params are still accurate. */ + return; } - UnsignedWord size = maxYoungSize.unsignedDivide(MIN_SURVIVOR_RATIO); - return minSpaceSize(alignDown(size)); + + /* Try allocating a struct on the C heap. */ + UnsignedWord structSize = SizeOf.unsigned(RawSizeParameters.class); + RawSizeParameters newValuesOnHeap = NullableNativeMemory.malloc(structSize, NmtCategory.GC); + VMError.guarantee(newValuesOnHeap.isNonNull(), "Out-of-memory while updating GC policy sizes."); + + /* Copy the values from the stack to the C heap. */ + UnmanagedMemoryUtil.copyForward((Pointer) newValuesOnStack, (Pointer) newValuesOnHeap, structSize); + newValuesOnHeap.setNext(prevValues); + + /* + * Publish the new struct via a volatile store. Once the data is published, other + * threads need to see a fully initialized struct right away. This is guaranteed by the + * implicit STORE_STORE barrier before the volatile write. + */ + sizes = newValuesOnHeap; + + assert isAligned(getMaxSurvivorSize()) && isAligned(getInitialYoungSize()) && isAligned(getInitialOldSize()) && isAligned(getMaxOldSize()); + assert getMaxSurvivorSize().belowThan(getMaxYoungSize()); + assert getMaxYoungSize().add(getMaxOldSize()).equal(getMaxHeapSize()); + assert getInitialEdenSize().add(getInitialSurvivorSize().multiply(2)).equal(getInitialYoungSize()); + assert getInitialYoungSize().add(getInitialOldSize()).equal(sizes.getInitialHeapSize()); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - UnsignedWord initialYoungSize() { - return initialEdenSize.add(initialSurvivorSize.multiply(2)); + /** + * Frees no longer needed {@link RawSizeParameters}, so that only the most recent one + * remains. + */ + public void freeUnusedSizeParameters() { + assert isInitialized(); + assert VMOperation.isGCInProgress() : "would need to be uninterruptible otherwise"; + + RawSizeParameters cur = sizes; + freeSizeParameters(cur.getNext()); + cur.setNext(Word.nullPointer()); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - UnsignedWord initialOldSize() { - return initialHeapSize.subtract(initialYoungSize()); + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + assert VMThreads.isTearingDown(); + + freeSizeParameters(sizes); + sizes = Word.nullPointer(); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - UnsignedWord maxOldSize() { - return maxHeapSize.subtract(maxYoungSize); + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void freeSizeParameters(RawSizeParameters first) { + assert VMOperation.isGCInProgress() || VMThreads.isTearingDown(); + + RawSizeParameters cur = first; + while (cur.isNonNull()) { + RawSizeParameters next = cur.getNext(); + NullableNativeMemory.free(cur); + cur = next; + } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - boolean equal(SizeParameters other) { - return other == this || other.matches(maxHeapSize, maxYoungSize, initialHeapSize, initialEdenSize, initialSurvivorSize, minHeapSize); + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + private static boolean matches(RawSizeParameters a, RawSizeParameters b) { + return a.getInitialEdenSize() == b.getInitialEdenSize() && + a.getEdenSize() == b.getEdenSize() && + a.getMaxEdenSize() == b.getMaxEdenSize() && + + a.getInitialSurvivorSize() == b.getInitialSurvivorSize() && + a.getSurvivorSize() == b.getSurvivorSize() && + a.getMaxSurvivorSize() == b.getMaxSurvivorSize() && + + a.getInitialYoungSize() == b.getInitialYoungSize() && + a.getYoungSize() == b.getYoungSize() && + a.getMaxYoungSize() == b.getMaxYoungSize() && + + a.getInitialOldSize() == b.getInitialOldSize() && + a.getOldSize() == b.getOldSize() && + a.getMaxOldSize() == b.getMaxOldSize() && + + a.getPromoSize() == b.getPromoSize() && + + a.getMinHeapSize() == b.getMinHeapSize() && + a.getInitialHeapSize() == b.getInitialHeapSize() && + a.getHeapSize() == b.getHeapSize() && + a.getMaxHeapSize() == b.getMaxHeapSize(); } + } + + /** + * Struct that stores all GC-related sizes. See {@link SizeParameters} for more details on its + * lifecycle. + */ + @RawStructure + private interface RawSizeParameters extends PointerBase { + @RawField + UnsignedWord getInitialEdenSize(); + + @RawField + void setInitialEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getEdenSize(); + + @RawField + void setEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxEdenSize(); + + @RawField + void setMaxEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialSurvivorSize(); + + @RawField + void setInitialSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getSurvivorSize(); + + @RawField + void setSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxSurvivorSize(); + + @RawField + void setMaxSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialYoungSize(); + + @RawField + void setInitialYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getYoungSize(); + + @RawField + void setYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxYoungSize(); + + @RawField + void setMaxYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialOldSize(); + + @RawField + void setInitialOldSize(UnsignedWord value); + + @RawField + UnsignedWord getOldSize(); + + @RawField + void setOldSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxOldSize(); + + @RawField + void setMaxOldSize(UnsignedWord value); + + @RawField + UnsignedWord getPromoSize(); + + @RawField + void setPromoSize(UnsignedWord value); + + @RawField + UnsignedWord getMinHeapSize(); + + @RawField + void setMinHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialHeapSize(); + + @RawField + void setInitialHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getHeapSize(); + + @RawField + void setHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxHeapSize(); + + @RawField + void setMaxHeapSize(UnsignedWord value); + + @RawField + RawSizeParameters getNext(); + + @RawField + void setNext(RawSizeParameters value); + } + + /** + * The methods in this class may only be used on {@link RawSizeParameters} that are allocated on + * the stack and therefore not yet visible to other threads. Please keep the methods in this + * class to a minimum. + */ + private static final class RawSizeParametersAccess { + static void initialize(RawSizeParameters valuesOnStack, + UnsignedWord initialEdenSize, UnsignedWord edenSize, UnsignedWord maxEdenSize, + UnsignedWord initialSurvivorSize, UnsignedWord survivorSize, UnsignedWord maxSurvivorSize, + UnsignedWord initialOldSize, UnsignedWord oldSize, UnsignedWord maxOldSize, + UnsignedWord promoSize, + UnsignedWord initialYoungSize, UnsignedWord youngSize, UnsignedWord maxYoungSize, + UnsignedWord minHeapSize, UnsignedWord initialHeapSize, UnsignedWord heapSize, UnsignedWord maxHeapSize) { + assert isAligned(maxHeapSize) && isAligned(maxYoungSize) && isAligned(initialHeapSize) && isAligned(initialEdenSize) && isAligned(initialSurvivorSize); + + assert initialEdenSize.belowOrEqual(initialYoungSize); + assert edenSize.belowOrEqual(youngSize); + assert maxEdenSize.belowOrEqual(maxYoungSize); + + assert initialSurvivorSize.belowOrEqual(initialYoungSize); + assert survivorSize.belowOrEqual(youngSize); + assert maxSurvivorSize.belowOrEqual(maxYoungSize); + + assert initialOldSize.belowOrEqual(initialHeapSize); + assert oldSize.belowOrEqual(heapSize); + assert maxOldSize.belowOrEqual(maxHeapSize); + + assert initialYoungSize.belowOrEqual(initialHeapSize); + assert youngSize.belowOrEqual(heapSize); + assert maxYoungSize.belowOrEqual(maxHeapSize); + + assert minHeapSize.belowOrEqual(initialHeapSize); + assert initialHeapSize.belowOrEqual(maxHeapSize); + assert heapSize.belowOrEqual(maxHeapSize); + assert maxHeapSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()); + + valuesOnStack.setInitialEdenSize(initialEdenSize); + valuesOnStack.setEdenSize(edenSize); + valuesOnStack.setMaxEdenSize(maxEdenSize); + + valuesOnStack.setInitialSurvivorSize(initialSurvivorSize); + valuesOnStack.setSurvivorSize(survivorSize); + valuesOnStack.setMaxSurvivorSize(maxSurvivorSize); + + valuesOnStack.setInitialYoungSize(initialYoungSize); + valuesOnStack.setYoungSize(youngSize); + valuesOnStack.setMaxYoungSize(maxYoungSize); + + valuesOnStack.setInitialOldSize(initialOldSize); + valuesOnStack.setOldSize(oldSize); + valuesOnStack.setMaxOldSize(maxOldSize); + + valuesOnStack.setPromoSize(promoSize); + + valuesOnStack.setMinHeapSize(minHeapSize); + valuesOnStack.setInitialHeapSize(initialHeapSize); + valuesOnStack.setHeapSize(heapSize); + valuesOnStack.setMaxHeapSize(maxHeapSize); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - boolean matches(UnsignedWord maxHeap, UnsignedWord maxYoung, UnsignedWord initialHeap, UnsignedWord initialEden, UnsignedWord initialSurvivor, UnsignedWord minHeap) { - return maxHeapSize.equal(maxHeap) && maxYoungSize.equal(maxYoung) && initialHeapSize.equal(initialHeap) && - initialEdenSize.equal(initialEden) && initialSurvivorSize.equal(initialSurvivor) && minHeapSize.equal(minHeap); + valuesOnStack.setNext(Word.nullPointer()); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java index 56cb85a010a4..d49684b22230 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java @@ -29,7 +29,6 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.Isolates; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.TimeUtils; @@ -108,7 +107,8 @@ class AdaptiveCollectionPolicy extends AbstractCollectionPolicy { private static final int ADAPTIVE_SIZE_POLICY_INITIALIZING_STEPS = ADAPTIVE_SIZE_POLICY_READY_THRESHOLD; /** * The minimum estimated decrease in {@link #gcCost()} in percent to decide in favor of - * expanding a space by 1% of the combined size of {@link #edenSize} and {@link #promoSize}. + * expanding a space by 1% of the combined size of {@link SizeParameters#getEdenSize} and + * {@link SizeParameters#getPromoSize}. */ private static final double ADAPTIVE_SIZE_ESTIMATOR_MIN_TOTAL_SIZE_COST_TRADEOFF = 0.5; /** The effective number of most recent data points used by estimator (exponential decay). */ @@ -212,7 +212,7 @@ public boolean shouldCollectCompletely(boolean followingIncrementalCollection, b */ UnsignedWord averagePromoted = UnsignedUtils.fromDouble(avgPromoted.getPaddedAverage()); UnsignedWord promotionEstimate = UnsignedUtils.min(averagePromoted, youngUsed); - UnsignedWord oldFree = oldSize.subtract(oldUsed); + UnsignedWord oldFree = sizes.getOldSize().subtract(oldUsed); return promotionEstimate.aboveThan(oldFree); } @@ -254,7 +254,7 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un targetSize = survivorLimit; decrTenuringThreshold = true; } - survivorSize = targetSize; + sizes.setSurvivorSize(targetSize); if (decrTenuringThreshold) { tenuringThreshold = Math.max(tenuringThreshold - 1, 1); @@ -264,30 +264,33 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un } protected void computeEdenSpaceSize(@SuppressWarnings("unused") boolean completeCollection, @SuppressWarnings("unused") GCCause cause) { + UnsignedWord curEdenSize = sizes.getEdenSize(); + UnsignedWord curPromoSize = sizes.getPromoSize(); + boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(youngGenChangeForMinorThroughput, minorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, edenSize, majorGcCost(), promoSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, curEdenSize, majorGcCost(), curPromoSize); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. */ } - UnsignedWord desiredEdenSize = edenSize; + UnsignedWord desiredEdenSize = curEdenSize; if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) { // from adjust_eden_for_throughput(): - UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(edenSize); + UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(curEdenSize); double scaleByRatio = minorGcCost() / gcCost(); assert scaleByRatio >= 0 && scaleByRatio <= 1; UnsignedWord scaledEdenHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(edenHeapDelta)); desiredEdenSize = alignUp(desiredEdenSize.add(scaledEdenHeapDelta)); - desiredEdenSize = UnsignedUtils.max(desiredEdenSize, edenSize); + desiredEdenSize = UnsignedUtils.max(desiredEdenSize, curEdenSize); youngGenChangeForMinorThroughput++; } if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) { - UnsignedWord desiredSum = edenSize.add(promoSize); - desiredEdenSize = adjustEdenForFootprint(edenSize, desiredSum); + UnsignedWord desiredSum = curEdenSize.add(curPromoSize); + desiredEdenSize = adjustEdenForFootprint(curEdenSize, desiredSum); } assert isAligned(desiredEdenSize); desiredEdenSize = minSpaceSize(desiredEdenSize); @@ -299,9 +302,9 @@ protected void computeEdenSpaceSize(@SuppressWarnings("unused") boolean complete * eden. This can lead to a general drifting down of the eden size. Let the tenuring * calculation push more into the old gen. */ - desiredEdenSize = UnsignedUtils.max(edenLimit, edenSize); + desiredEdenSize = UnsignedUtils.max(edenLimit, curEdenSize); } - edenSize = desiredEdenSize; + sizes.setEdenSize(desiredEdenSize); } private static boolean shouldUseEstimator(long genChangeForThroughput, double cost) { @@ -435,13 +438,13 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj if (completeCollection) { updateCollectionEndAverages(avgMajorGcCost, avgMajorPause, majorCostEstimator, avgMajorIntervalSeconds, - cause, latestMajorMutatorIntervalNanos, timer.lastIntervalNanos(), promoSize); + cause, latestMajorMutatorIntervalNanos, timer.lastIntervalNanos(), sizes.getPromoSize()); majorCount++; minorCountSinceMajorCollection = 0; } else { updateCollectionEndAverages(avgMinorGcCost, avgMinorPause, minorCostEstimator, null, - cause, latestMinorMutatorIntervalNanos, timer.lastIntervalNanos(), edenSize); + cause, latestMinorMutatorIntervalNanos, timer.lastIntervalNanos(), sizes.getEdenSize()); minorCount++; minorCountSinceMajorCollection++; @@ -455,7 +458,7 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj GCAccounting accounting = GCImpl.getAccounting(); UnsignedWord oldLive = accounting.getOldGenerationAfterChunkBytes(); - oldSizeExceededInPreviousCollection = oldLive.aboveThan(oldSize); + oldSizeExceededInPreviousCollection = oldLive.aboveThan(sizes.getOldSize()); if (!completeCollection) { /* @@ -471,7 +474,7 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj UnsignedWord tenuredChunkBytes = accounting.getLastIncrementalCollectionPromotedChunkBytes(); updateAverages(survivorOverflow, survivedChunkBytes, tenuredChunkBytes); - computeSurvivorSpaceSizeAndThreshold(survivorOverflow, sizes.maxSurvivorSize()); + computeSurvivorSpaceSizeAndThreshold(survivorOverflow, sizes.getMaxSurvivorSize()); } if (shouldUpdateStats(cause)) { computeEdenSpaceSize(completeCollection, cause); @@ -483,51 +486,55 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj } private void computeOldGenSpaceSize(UnsignedWord oldLive) { // compute_old_gen_free_space + UnsignedWord curEdenSize = sizes.getEdenSize(); + UnsignedWord curMaxOldSize = sizes.getMaxOldSize(); + UnsignedWord curPromoSize = sizes.getPromoSize(); + avgOldLive.sample(oldLive); // NOTE: if maxOldSize shrunk and difference is negative, unsigned conversion results in 0 - UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.maxOldSize()) - avgOldLive.getAverage()); - promoLimit = alignDown(UnsignedUtils.max(promoSize, promoLimit)); + UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(curMaxOldSize) - avgOldLive.getAverage()); + promoLimit = alignDown(UnsignedUtils.max(curPromoSize, promoLimit)); boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(oldGenChangeForMajorThroughput, majorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, promoSize, minorGcCost(), edenSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, curPromoSize, minorGcCost(), curEdenSize); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. */ } - UnsignedWord desiredPromoSize = promoSize; + UnsignedWord desiredPromoSize = curPromoSize; if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) { // from adjust_promo_for_throughput(): - UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(promoSize); + UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(curPromoSize); double scaleByRatio = majorGcCost() / gcCost(); assert scaleByRatio >= 0 && scaleByRatio <= 1; UnsignedWord scaledPromoHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(promoHeapDelta)); - desiredPromoSize = alignUp(promoSize.add(scaledPromoHeapDelta)); - desiredPromoSize = UnsignedUtils.max(desiredPromoSize, promoSize); + desiredPromoSize = alignUp(curPromoSize.add(scaledPromoHeapDelta)); + desiredPromoSize = UnsignedUtils.max(desiredPromoSize, curPromoSize); oldGenChangeForMajorThroughput++; } if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) { - UnsignedWord desiredSum = edenSize.add(promoSize); - desiredPromoSize = adjustPromoForFootprint(promoSize, desiredSum); + UnsignedWord desiredSum = curEdenSize.add(curPromoSize); + desiredPromoSize = adjustPromoForFootprint(curPromoSize, desiredSum); } assert isAligned(desiredPromoSize); desiredPromoSize = minSpaceSize(desiredPromoSize); desiredPromoSize = UnsignedUtils.min(desiredPromoSize, promoLimit); - promoSize = desiredPromoSize; + sizes.setPromoSize(desiredPromoSize); // from PSOldGen::resize UnsignedWord desiredFreeSpace = calculatedOldFreeSizeInBytes(); UnsignedWord desiredOldSize = alignUp(oldLive.add(desiredFreeSpace)); - oldSize = UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), sizes.maxOldSize()); + sizes.setOldSize(UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), curMaxOldSize)); } UnsignedWord calculatedOldFreeSizeInBytes() { - return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(promoSize) + avgPromoted.getPaddedAverage()); + return UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.getPromoSize()) + avgPromoted.getPaddedAverage()); } private static UnsignedWord adjustPromoForFootprint(UnsignedWord curPromo, UnsignedWord desiredSum) { @@ -591,10 +598,4 @@ private void updateCollectionEndAverages(AdaptiveWeightedAverage costAverage, Ad costEstimator.sample(UnsignedUtils.toDouble(sizeBytes), cost); } } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected long gcCount() { - return minorCount + majorCount; - } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index b6b3174173df..d5bceb425c50 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -228,4 +228,8 @@ default boolean isOutOfMemory(UnsignedWord usedBytes) { default void onMaximumHeapSizeExceeded() { throw OutOfMemoryUtil.heapSizeExceeded(); } + + @Uninterruptible(reason = "Tear-down in progress.") + default void tearDown() { + } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java index 8b88dedb227f..8cc138809f14 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/DynamicCollectionPolicy.java @@ -50,20 +50,21 @@ protected long getMaximumHeapSizeOptionValue() { @Override public boolean isOutOfMemory(UnsignedWord usedBytes) { - if (ImageSingletons.contains(DynamicHeapSizeManager.class)) { - return ImageSingletons.lookup(DynamicHeapSizeManager.class).outOfMemory(usedBytes); + boolean outOfMemory = super.isOutOfMemory(usedBytes); + if (ImageSingletons.contains(DynamicHeapSizeManager.class) && outOfMemory) { + outOfMemory = ImageSingletons.lookup(DynamicHeapSizeManager.class).outOfMemory(usedBytes); + if (!outOfMemory) { + /* No longer out-of-memory - update the heap size parameters to reflect that. */ + GCImpl.getPolicy().updateSizeParameters(); + } } - - return super.isOutOfMemory(usedBytes); + return outOfMemory; } @Override public void onMaximumHeapSizeExceeded() { if (isOutOfMemory(HeapImpl.getAccounting().getUsedBytes())) { super.onMaximumHeapSizeExceeded(); - } else { - /* No longer out-of-memory - update the heap size parameters to reflect that. */ - GCImpl.getPolicy().updateSizeParameters(); } } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 17fa4f11ef34..62d9362d27af 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -151,6 +151,11 @@ public final class GCImpl implements GC { } } + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + policy.tearDown(); + } + @Override public String getName() { if (SubstrateOptions.useEpsilonGC()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 9a03ae38a238..10f276220d92 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -212,7 +212,8 @@ public void walkObjects(ObjectVisitor visitor) { public boolean tearDown() { youngGeneration.tearDown(); oldGeneration.tearDown(); - getChunkProvider().tearDown(); + chunkProvider.tearDown(); + gcImpl.tearDown(); if (Metaspace.isSupported()) { MetaspaceImpl.singleton().tearDown(); @@ -1017,7 +1018,7 @@ private long totalMemory() { @Substitute private long maxMemory() { - GCImpl.getPolicy().updateSizeParameters(); + GCImpl.getPolicy().ensureSizeParametersInitialized(); return GCImpl.getPolicy().getMaximumHeapSize().rawValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 23c68c8011b2..3793a82f7a82 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -124,6 +124,7 @@ public static UnsignedWord getMaximumHeapFree() { return Word.unsigned(SerialGCOptions.MaxHeapFree.getValue()); } + @Fold public static int getHeapChunkHeaderPadding() { return SerialAndEpsilonGCOptions.HeapChunkHeaderPadding.getValue(); } @@ -160,15 +161,12 @@ public static UnsignedWord getLargeArrayThreshold() { return Word.unsigned(SerialAndEpsilonGCOptions.LargeArrayThreshold.getValue()); } - /* - * Zapping - */ - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @Fold public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } + @Fold public static boolean getZapConsumedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapConsumedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java index 40e3944b7c7e..d34b3f3424bc 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java @@ -91,7 +91,7 @@ public boolean shouldCollectOnHint(boolean fullGC) { edenUsedBytes = edenUsedBytes.add(FULL_GC_BONUS); } return edenUsedBytes.aboveOrEqual(Word.unsigned(Options.ExpectedEdenSize.getValue())) || - (UnsignedUtils.toDouble(edenUsedBytes) / UnsignedUtils.toDouble(edenSize) >= Options.UsedEdenProportionThreshold.getValue()); + (UnsignedUtils.toDouble(edenUsedBytes) / UnsignedUtils.toDouble(sizes.getEdenSize()) >= Options.UsedEdenProportionThreshold.getValue()); } @Override @@ -142,17 +142,19 @@ protected boolean shouldUpdateStats(GCCause cause) { protected void computeEdenSpaceSize(boolean completeCollection, GCCause cause) { if (cause == GCCause.HintedGC) { if (completeCollection && lastGCCause == GCCause.HintedGC) { - UnsignedWord newEdenSize = UnsignedUtils.max(sizes.initialEdenSize, alignUp(edenSize.unsignedDivide(2))); - if (edenSize.aboveThan(newEdenSize)) { - edenSize = newEdenSize; + UnsignedWord curEdenSize = sizes.getEdenSize(); + UnsignedWord newEdenSize = UnsignedUtils.max(sizes.getInitialEdenSize(), alignUp(curEdenSize.unsignedDivide(2))); + if (curEdenSize.aboveThan(newEdenSize)) { + sizes.setEdenSize(newEdenSize); } } } else { UnsignedWord sizeAfter = GCImpl.getChunkBytes(); if (sizeBefore.notEqual(0) && sizeBefore.belowThan(sizeAfter.multiply(2))) { - UnsignedWord newEdenSize = UnsignedUtils.min(getMaximumEdenSize(), alignUp(edenSize.multiply(2))); - if (edenSize.belowThan(newEdenSize)) { - edenSize = newEdenSize; + UnsignedWord curEdenSize = sizes.getEdenSize(); + UnsignedWord newEdenSize = UnsignedUtils.min(getMaximumEdenSize(), alignUp(curEdenSize.multiply(2))); + if (curEdenSize.belowThan(newEdenSize)) { + sizes.setEdenSize(newEdenSize); } } else { super.computeEdenSpaceSize(completeCollection, cause); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java index da5a29f66d46..c01690c56c3c 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ProportionateSpacesPolicy.java @@ -26,13 +26,13 @@ import static com.oracle.svm.core.genscavenge.CollectionPolicy.shouldCollectYoungGenSeparately; -import jdk.graal.compiler.word.Word; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.util.UnsignedUtils; +import jdk.graal.compiler.word.Word; + /** A port of HotSpot's SerialGC size policy. */ final class ProportionateSpacesPolicy extends AbstractCollectionPolicy { @@ -50,7 +50,6 @@ final class ProportionateSpacesPolicy extends AbstractCollectionPolicy { static final int MAX_TENURING_THRESHOLD = 15; static final int TARGET_SURVIVOR_RATIO = 50; - private int totalCollections; private boolean oldSizeExceededInPreviousCollection; private int shrinkFactor; @@ -95,19 +94,17 @@ public void onCollectionBegin(boolean completeCollection, long beginNanoTime) { @Override public void onCollectionEnd(boolean completeCollection, GCCause cause) { UnsignedWord oldLive = GCImpl.getAccounting().getOldGenerationAfterChunkBytes(); - oldSizeExceededInPreviousCollection = oldLive.aboveThan(oldSize); + oldSizeExceededInPreviousCollection = oldLive.aboveThan(sizes.getOldSize()); boolean resizeOldOnlyForPromotions = !completeCollection; computeNewOldGenSize(resizeOldOnlyForPromotions); computeNewYoungGenSize(); adjustDesiredTenuringThreshold(); - - totalCollections++; } private void adjustDesiredTenuringThreshold() { // DefNewGeneration::adjust_desired_tenuring_threshold // Set the desired survivor size to half the real survivor space - UnsignedWord desiredSurvivorSize = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(survivorSize) * TARGET_SURVIVOR_RATIO / 100); + UnsignedWord desiredSurvivorSize = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(sizes.getSurvivorSize()) * TARGET_SURVIVOR_RATIO / 100); // AgeTable::compute_tenuring_threshold YoungGeneration youngGen = HeapImpl.getHeapImpl().getYoungGeneration(); @@ -126,10 +123,10 @@ private void adjustDesiredTenuringThreshold() { // DefNewGeneration::adjust_desi } private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredGeneration::compute_new_size_inner - UnsignedWord capacityAtPrologue = oldSize; + UnsignedWord capacityAtPrologue = sizes.getOldSize(); UnsignedWord usedAfterGc = GCImpl.getAccounting().getOldGenerationAfterChunkBytes(); - if (oldSize.belowThan(usedAfterGc)) { - oldSize = usedAfterGc; + if (sizes.getOldSize().belowThan(usedAfterGc)) { + sizes.setOldSize(usedAfterGc); } if (resizeOnlyForPromotions) { return; @@ -142,24 +139,24 @@ private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredG double maximumUsedPercentage = 1 - minimumFreePercentage; UnsignedWord minimumDesiredCapacity = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(usedAfterGc) / maximumUsedPercentage); - minimumDesiredCapacity = UnsignedUtils.max(minimumDesiredCapacity, sizes.initialOldSize()); + minimumDesiredCapacity = UnsignedUtils.max(minimumDesiredCapacity, sizes.getInitialOldSize()); - if (oldSize.belowThan(minimumDesiredCapacity)) { - oldSize = alignUp(minimumDesiredCapacity); + if (sizes.getOldSize().belowThan(minimumDesiredCapacity)) { + sizes.setOldSize(alignUp(minimumDesiredCapacity)); return; } - UnsignedWord maxShrinkBytes = oldSize.subtract(minimumDesiredCapacity); + UnsignedWord maxShrinkBytes = sizes.getOldSize().subtract(minimumDesiredCapacity); UnsignedWord shrinkBytes = Word.zero(); if (MAX_HEAP_FREE_RATIO < 100) { double maximumFreePercentage = MAX_HEAP_FREE_RATIO / 100.0; double minimumUsedPercentage = 1 - maximumFreePercentage; UnsignedWord maximumDesiredCapacity = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(usedAfterGc) / minimumUsedPercentage); - maximumDesiredCapacity = UnsignedUtils.max(maximumDesiredCapacity, sizes.initialOldSize()); + maximumDesiredCapacity = UnsignedUtils.max(maximumDesiredCapacity, sizes.getInitialOldSize()); assert minimumDesiredCapacity.belowOrEqual(maximumDesiredCapacity); - if (oldSize.aboveThan(maximumDesiredCapacity)) { - shrinkBytes = oldSize.subtract(maximumDesiredCapacity); + if (sizes.getOldSize().aboveThan(maximumDesiredCapacity)) { + shrinkBytes = sizes.getOldSize().subtract(maximumDesiredCapacity); if (SHRINK_HEAP_IN_STEPS) { /* * We don't want to shrink all the way back to initSize if people call @@ -180,39 +177,36 @@ private void computeNewOldGenSize(boolean resizeOnlyForPromotions) { // TenuredG } } - if (oldSize.aboveThan(capacityAtPrologue)) { + if (sizes.getOldSize().aboveThan(capacityAtPrologue)) { /* * We might have expanded for promotions, in which case we might want to take back that * expansion if there's room after GC. That keeps us from stretching the heap with * promotions when there's plenty of room. */ - UnsignedWord expansionForPromotion = oldSize.subtract(capacityAtPrologue); + UnsignedWord expansionForPromotion = sizes.getOldSize().subtract(capacityAtPrologue); expansionForPromotion = UnsignedUtils.min(expansionForPromotion, maxShrinkBytes); shrinkBytes = UnsignedUtils.max(shrinkBytes, expansionForPromotion); } if (shrinkBytes.aboveThan(MIN_HEAP_FREE_RATIO)) { - oldSize = oldSize.subtract(shrinkBytes); + sizes.setOldSize(sizes.getOldSize().subtract(shrinkBytes)); } } private void computeNewYoungGenSize() { // DefNewGeneration::compute_new_size - UnsignedWord desiredNewSize = oldSize.unsignedDivide(NEW_RATIO); - desiredNewSize = UnsignedUtils.clamp(desiredNewSize, sizes.initialYoungSize(), sizes.maxYoungSize); + UnsignedWord desiredNewSize = sizes.getOldSize().unsignedDivide(NEW_RATIO); + desiredNewSize = UnsignedUtils.clamp(desiredNewSize, sizes.getInitialYoungSize(), sizes.getMaxYoungSize()); // DefNewGeneration::compute_space_boundaries, DefNewGeneration::compute_survivor_size - survivorSize = minSpaceSize(alignDown(desiredNewSize.unsignedDivide(SURVIVOR_RATIO))); + UnsignedWord newSurvivorSize = minSpaceSize(alignDown(desiredNewSize.unsignedDivide(SURVIVOR_RATIO))); + sizes.setSurvivorSize(newSurvivorSize); + UnsignedWord desiredEdenSize = Word.zero(); - if (desiredNewSize.aboveThan(survivorSize.multiply(2))) { - desiredEdenSize = desiredNewSize.subtract(survivorSize.multiply(2)); + if (desiredNewSize.aboveThan(newSurvivorSize.multiply(2))) { + desiredEdenSize = desiredNewSize.subtract(newSurvivorSize.multiply(2)); } - edenSize = minSpaceSize(alignDown(desiredEdenSize)); - assert edenSize.aboveThan(0) && survivorSize.belowOrEqual(edenSize); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected long gcCount() { - return totalCollections; + UnsignedWord newEdenSize = minSpaceSize(alignDown(desiredEdenSize)); + sizes.setEdenSize(newEdenSize); + assert newEdenSize.aboveThan(0) && newSurvivorSize.belowOrEqual(newEdenSize); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index 10d2115c534c..c558da101095 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -78,11 +78,38 @@ import jdk.graal.compiler.word.Word; /** - * Bump-pointer allocation from thread-local top and end Pointers. Many of these methods are called - * from allocation snippets, so they can not do anything fancy. It happens that prefetch - * instructions access memory outside the TLAB. At the moment, this is not an issue as we only - * support architectures where the prefetch instructions never cause a segfault, even if they try to - * access memory that is not accessible. + * Multiple threads may execute the methods in this class concurrently. All code that is + * transitively reachable from those method can get executed as a side effect of an allocation slow + * path. To prevent hard to debug transient issues, we execute as little code as possible in these + * methods. + * + * If the executed code is too complex, then it can happen that we unexpectedly change some shared + * global state as a side effect of an allocation. This may result in issues that look similar to + * races but that can even happen in single-threaded environments, e.g.: + * + * 

+ * {@code
+ * private static Object singleton;
+ *
+ * private static synchronized Object createSingleton() {
+ *     if (singleton == null) {
+ *         Object o = new Object();
+ *         // If the allocation above enters the allocation slow path code, and executes a
+ *         // complex slow path hook, then it is possible that createSingleton() gets
+ *         // recursively execute by the current thread. So, the assertion below may fail
+ *         // because the singleton got already initialized by the same thread in the meanwhile.
+ *         assert singleton == null;
+ *         singleton = o;
+ *     }
+ *     return result;
+ * }
+ * }
+ *
+ * + * The allocation fast-path emits prefetch instructions. Those instructions may try to access memory + * that is outside the TLAB and therefore not necessarily accessible. At the moment, this is not an + * issue as we only support architectures where prefetch instructions never cause segfaults, even if + * they access memory that is not accessible. */ public final class ThreadLocalAllocation { @RawStructure @@ -183,46 +210,14 @@ static UnsignedWord getAlignedAllocatedBytes(IsolateThread thread) { return allocatedAlignedBytes.getVolatile(thread); } - /** - * NOTE: Multiple threads may execute this method concurrently. All code that is transitively - * reachable from this method may get executed as a side effect of an allocation slow path. To - * prevent hard to debug transient issues, we execute as little code as possible in this method. - * - * If the executed code is too complex, then it can happen that we unexpectedly change some - * shared global state as a side effect of an allocation. This may result in issues that look - * similar to races but that can even happen in single-threaded environments, e.g.: - * - * 
-     * {@code
-     * private static Object singleton;
-     *
-     * private static synchronized Object createSingleton() {
-     *     if (singleton == null) {
-     *         Object o = new Object();
-     *         // If the allocation above enters the allocation slow path code, and executes a
-     *         // complex slow path hook, then it is possible that createSingleton() gets
-     *         // recursively execute by the current thread. So, the assertion below may fail
-     *         // because the singleton got already initialized by the same thread in the meanwhile.
-     *         assert singleton == null;
-     *         singleton = o;
-     *     }
-     *     return result;
-     * }
-     * }
-     *
- */ - private static void runSlowPathHooks() { - GCImpl.getPolicy().updateSizeParameters(); - } - public static Object slowPathNewInstance(Word objectHeader) { DynamicHub hub = ObjectHeaderImpl.getObjectHeaderImpl().dynamicHubFromObjectHeader(objectHeader); UnsignedWord size = LayoutEncoding.getPureInstanceAllocationSize(hub.getLayoutEncoding()); Object result = allocateInstanceInCurrentTlab(hub, size); if (result == null) { + GCImpl.getPolicy().ensureSizeParametersInitialized(); result = slowPathNewInstanceWithoutAllocating(hub, size); - runSlowPathHooks(); sampleSlowPathAllocation(result, size, Integer.MIN_VALUE); } return result; @@ -269,10 +264,7 @@ public static Object slowPathNewArrayLikeObject(Word objectHeader, int length, b } Object result = slowPathNewArrayLikeObjectWithoutAllocating(hub, length, size, podReferenceMap); - - runSlowPathHooks(); sampleSlowPathAllocation(result, size, length); - return result; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java index 5ec777e33cca..680dcc4d5afa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/option/NotifyGCRuntimeOptionKey.java @@ -24,11 +24,11 @@ */ package com.oracle.svm.core.option; +import java.util.function.Consumer; + import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.heap.Heap; -import java.util.function.Consumer; - /** * Notifies the {@link Heap} implementation after the value of the option has changed. */ From 333471df0130aa5d80259539ba8896e353f1db8f Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Thu, 18 Dec 2025 16:41:16 +0100 Subject: [PATCH 5/5] Cleanups. --- .../genscavenge/AbstractCollectionPolicy.java | 460 +----------------- .../genscavenge/AdaptiveCollectionPolicy.java | 52 +- .../oracle/svm/core/genscavenge/HeapImpl.java | 1 + .../genscavenge/LibGraalCollectionPolicy.java | 16 +- .../core/genscavenge/RawSizeParameters.java | 150 ++++++ .../RawSizeParametersOnStackAccess.java | 96 ++++ .../svm/core/genscavenge/SizeParameters.java | 292 +++++++++++ .../genscavenge/ThreadLocalAllocation.java | 27 +- 8 files changed, 607 insertions(+), 487 deletions(-) create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java create mode 100644 substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index dc1378717734..482b0fda5f38 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -29,28 +29,18 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.StackValue; -import org.graalvm.nativeimage.c.struct.RawField; -import org.graalvm.nativeimage.c.struct.RawStructure; -import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.word.Pointer; -import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.IsolateArgumentParser; import com.oracle.svm.core.SubstrateGCOptions; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.heap.PhysicalMemory; -import com.oracle.svm.core.heap.ReferenceAccess; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.locks.VMMutex; -import com.oracle.svm.core.memory.NullableNativeMemory; -import com.oracle.svm.core.nmt.NmtCategory; import com.oracle.svm.core.os.CommittedMemoryProvider; import com.oracle.svm.core.stack.StackOverflowCheck; import com.oracle.svm.core.thread.RecurringCallbackSupport; import com.oracle.svm.core.thread.VMOperation; -import com.oracle.svm.core.thread.VMThreads; import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.UnsignedUtils; import com.oracle.svm.core.util.VMError; @@ -63,7 +53,7 @@ * to avoid inconsistencies between the different heap size values. */ abstract class AbstractCollectionPolicy implements CollectionPolicy { - private static final VMMutex MUTEX = new VMMutex("AbstractCollectionPolicy.sizeParams"); + private static final VMMutex SIZES_MUTEX = new VMMutex("AbstractCollectionPolicy.sizes"); protected static final int MIN_SPACE_SIZE_AS_NUMBER_OF_ALIGNED_CHUNKS = 8; @@ -167,11 +157,22 @@ protected void guaranteeSizeParametersInitialized() { VMError.guarantee(sizes.isInitialized()); } + /** + * (Re)computes and updates the {@link SizeParameters} used by this garbage collection policy to + * reflect the latest values derived from relevant heap size options and physical memory + * constraints. + *

+ * This method is thread-safe and may be invoked concurrently by multiple threads. It is called + * at least once during VM startup, and is called again if the value of a relevant runtime + * option changes. A mutex is used to guarantee consistency of the computed parameters and + * prevent lost updates (note that a CAS would not be sufficient to prevent lost updates because + * threads could overwrite more recent values with outdated values). + */ @Override public void updateSizeParameters() { StackOverflowCheck.singleton().makeYellowZoneAvailable(); try { - MUTEX.lock(); + SIZES_MUTEX.lock(); try { assert RecurringCallbackSupport.isCallbackUnsupportedOrTimerSuspended() : "recurring callbacks could trigger recursive locking, which isn't supported"; @@ -180,7 +181,7 @@ public void updateSizeParameters() { sizes.update(newValuesOnStack); } finally { - MUTEX.unlock(); + SIZES_MUTEX.unlock(); } } finally { StackOverflowCheck.singleton().protectYellowZone(); @@ -220,7 +221,7 @@ public final UnsignedWord getMaximumSurvivorSize() { @Override public final UnsignedWord getCurrentHeapCapacity() { - return sizes.getCurrentHeapCapacity(); + return sizes.getHeapSize(); } @Override @@ -291,7 +292,7 @@ public final UnsignedWord getMinimumHeapSize() { @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/shared/genArguments.cpp#L195-L310") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L146-L168") - private void computeSizeParameters(RawSizeParameters newParams) { + private void computeSizeParameters(RawSizeParameters newParamsOnStack) { UnsignedWord minYoungSpaces = minSpaceSize(); // eden if (HeapParameters.getMaxSurvivorSpaces() > 0) { minYoungSpaces = minYoungSpaces.add(minSpaceSize().multiply(2)); // survivor from and to @@ -392,7 +393,7 @@ private void computeSizeParameters(RawSizeParameters newParams) { UnsignedWord youngSize = edenSize.add(survivorSize); UnsignedWord heapSize = edenSize.add(survivorSize).add(oldSize); - RawSizeParametersAccess.initialize(newParams, + RawSizeParametersOnStackAccess.initialize(newParamsOnStack, initialEden, edenSize, maxEdenSize, initialSurvivor, survivorSize, maxSurvivorSize, initialOldSize, oldSize, maxOldSize, @@ -427,431 +428,4 @@ protected static long getMinHeapSizeOptionValue() { protected UnsignedWord getInitialHeapSize() { return AbstractCollectionPolicy.INITIAL_HEAP_SIZE; } - - /** - * Once a {@link RawSizeParameters} struct is visible to other threads (see {@link #update}), it - * may only be accessed by {@link Uninterruptible} code or when the VM is at a safepoint. This - * is necessary to prevent use-after-free errors because no longer needed - * {@link RawSizeParameters} structs may be freed during a GC. - *

- * When the VM is at a safepoint, the GC may directly update some of the values in the latest - * {@link RawSizeParameters} (see setters below). - */ - protected static final class SizeParameters { - private static final String ACCESS_RAW_SIZE_PARAMETERS = "Prevent that RawSizeParameters are freed."; - - private volatile RawSizeParameters sizes; - - @Platforms(Platform.HOSTED_ONLY.class) - private SizeParameters() { - } - - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - public boolean isInitialized() { - /* - * This only checks for non-null and doesn't access any values in the struct, so - * inlining into interruptible code is allowed. - */ - return sizes.isNonNull(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getInitialEdenSize() { - assert isInitialized(); - return sizes.getInitialEdenSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getEdenSize() { - assert isInitialized(); - return sizes.getEdenSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public void setEdenSize(UnsignedWord value) { - assert isInitialized(); - assert VMOperation.isGCInProgress(); - - sizes.setEdenSize(value); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getMaxEdenSize() { - assert isInitialized(); - return sizes.getMaxEdenSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getInitialSurvivorSize() { - assert isInitialized(); - return sizes.getInitialSurvivorSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getSurvivorSize() { - assert isInitialized(); - return sizes.getSurvivorSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public void setSurvivorSize(UnsignedWord value) { - assert isInitialized(); - assert VMOperation.isGCInProgress(); - - sizes.setSurvivorSize(value); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-25+21/src/hotspot/share/gc/parallel/psYoungGen.cpp#L104-L116") - UnsignedWord getMaxSurvivorSize() { - assert isInitialized(); - return sizes.getMaxSurvivorSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - UnsignedWord getInitialYoungSize() { - assert isInitialized(); - return sizes.getInitialYoungSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getYoungSize() { - assert isInitialized(); - return sizes.getYoungSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getMaxYoungSize() { - assert isInitialized(); - return sizes.getMaxYoungSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - UnsignedWord getInitialOldSize() { - assert isInitialized(); - return sizes.getInitialOldSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getOldSize() { - assert isInitialized(); - return sizes.getOldSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public void setOldSize(UnsignedWord value) { - assert isInitialized(); - assert VMOperation.isGCInProgress(); - - sizes.setOldSize(value); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - UnsignedWord getMaxOldSize() { - assert isInitialized(); - return sizes.getMaxOldSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getPromoSize() { - assert isInitialized(); - return sizes.getPromoSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public void setPromoSize(UnsignedWord value) { - assert isInitialized(); - assert VMOperation.isGCInProgress(); - - sizes.setPromoSize(value); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getMinHeapSize() { - assert isInitialized(); - return sizes.getMinHeapSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getMaxHeapSize() { - assert isInitialized(); - return sizes.getMaxHeapSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - public UnsignedWord getCurrentHeapCapacity() { - assert isInitialized(); - assert VMOperation.isGCInProgress() : "use only during GC"; - - return sizes.getHeapSize(); - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - private void update(RawSizeParameters newValuesOnStack) { - RawSizeParameters prevValues = sizes; - if (prevValues.isNonNull() && matches(prevValues, newValuesOnStack)) { - /* Nothing to do - cached params are still accurate. */ - return; - } - - /* Try allocating a struct on the C heap. */ - UnsignedWord structSize = SizeOf.unsigned(RawSizeParameters.class); - RawSizeParameters newValuesOnHeap = NullableNativeMemory.malloc(structSize, NmtCategory.GC); - VMError.guarantee(newValuesOnHeap.isNonNull(), "Out-of-memory while updating GC policy sizes."); - - /* Copy the values from the stack to the C heap. */ - UnmanagedMemoryUtil.copyForward((Pointer) newValuesOnStack, (Pointer) newValuesOnHeap, structSize); - newValuesOnHeap.setNext(prevValues); - - /* - * Publish the new struct via a volatile store. Once the data is published, other - * threads need to see a fully initialized struct right away. This is guaranteed by the - * implicit STORE_STORE barrier before the volatile write. - */ - sizes = newValuesOnHeap; - - assert isAligned(getMaxSurvivorSize()) && isAligned(getInitialYoungSize()) && isAligned(getInitialOldSize()) && isAligned(getMaxOldSize()); - assert getMaxSurvivorSize().belowThan(getMaxYoungSize()); - assert getMaxYoungSize().add(getMaxOldSize()).equal(getMaxHeapSize()); - assert getInitialEdenSize().add(getInitialSurvivorSize().multiply(2)).equal(getInitialYoungSize()); - assert getInitialYoungSize().add(getInitialOldSize()).equal(sizes.getInitialHeapSize()); - } - - /** - * Frees no longer needed {@link RawSizeParameters}, so that only the most recent one - * remains. - */ - public void freeUnusedSizeParameters() { - assert isInitialized(); - assert VMOperation.isGCInProgress() : "would need to be uninterruptible otherwise"; - - RawSizeParameters cur = sizes; - freeSizeParameters(cur.getNext()); - cur.setNext(Word.nullPointer()); - } - - @Uninterruptible(reason = "Tear-down in progress.") - public void tearDown() { - assert VMThreads.isTearingDown(); - - freeSizeParameters(sizes); - sizes = Word.nullPointer(); - } - - @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) - private static void freeSizeParameters(RawSizeParameters first) { - assert VMOperation.isGCInProgress() || VMThreads.isTearingDown(); - - RawSizeParameters cur = first; - while (cur.isNonNull()) { - RawSizeParameters next = cur.getNext(); - NullableNativeMemory.free(cur); - cur = next; - } - } - - @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) - private static boolean matches(RawSizeParameters a, RawSizeParameters b) { - return a.getInitialEdenSize() == b.getInitialEdenSize() && - a.getEdenSize() == b.getEdenSize() && - a.getMaxEdenSize() == b.getMaxEdenSize() && - - a.getInitialSurvivorSize() == b.getInitialSurvivorSize() && - a.getSurvivorSize() == b.getSurvivorSize() && - a.getMaxSurvivorSize() == b.getMaxSurvivorSize() && - - a.getInitialYoungSize() == b.getInitialYoungSize() && - a.getYoungSize() == b.getYoungSize() && - a.getMaxYoungSize() == b.getMaxYoungSize() && - - a.getInitialOldSize() == b.getInitialOldSize() && - a.getOldSize() == b.getOldSize() && - a.getMaxOldSize() == b.getMaxOldSize() && - - a.getPromoSize() == b.getPromoSize() && - - a.getMinHeapSize() == b.getMinHeapSize() && - a.getInitialHeapSize() == b.getInitialHeapSize() && - a.getHeapSize() == b.getHeapSize() && - a.getMaxHeapSize() == b.getMaxHeapSize(); - } - } - - /** - * Struct that stores all GC-related sizes. See {@link SizeParameters} for more details on its - * lifecycle. - */ - @RawStructure - private interface RawSizeParameters extends PointerBase { - @RawField - UnsignedWord getInitialEdenSize(); - - @RawField - void setInitialEdenSize(UnsignedWord value); - - @RawField - UnsignedWord getEdenSize(); - - @RawField - void setEdenSize(UnsignedWord value); - - @RawField - UnsignedWord getMaxEdenSize(); - - @RawField - void setMaxEdenSize(UnsignedWord value); - - @RawField - UnsignedWord getInitialSurvivorSize(); - - @RawField - void setInitialSurvivorSize(UnsignedWord value); - - @RawField - UnsignedWord getSurvivorSize(); - - @RawField - void setSurvivorSize(UnsignedWord value); - - @RawField - UnsignedWord getMaxSurvivorSize(); - - @RawField - void setMaxSurvivorSize(UnsignedWord value); - - @RawField - UnsignedWord getInitialYoungSize(); - - @RawField - void setInitialYoungSize(UnsignedWord value); - - @RawField - UnsignedWord getYoungSize(); - - @RawField - void setYoungSize(UnsignedWord value); - - @RawField - UnsignedWord getMaxYoungSize(); - - @RawField - void setMaxYoungSize(UnsignedWord value); - - @RawField - UnsignedWord getInitialOldSize(); - - @RawField - void setInitialOldSize(UnsignedWord value); - - @RawField - UnsignedWord getOldSize(); - - @RawField - void setOldSize(UnsignedWord value); - - @RawField - UnsignedWord getMaxOldSize(); - - @RawField - void setMaxOldSize(UnsignedWord value); - - @RawField - UnsignedWord getPromoSize(); - - @RawField - void setPromoSize(UnsignedWord value); - - @RawField - UnsignedWord getMinHeapSize(); - - @RawField - void setMinHeapSize(UnsignedWord value); - - @RawField - UnsignedWord getInitialHeapSize(); - - @RawField - void setInitialHeapSize(UnsignedWord value); - - @RawField - UnsignedWord getHeapSize(); - - @RawField - void setHeapSize(UnsignedWord value); - - @RawField - UnsignedWord getMaxHeapSize(); - - @RawField - void setMaxHeapSize(UnsignedWord value); - - @RawField - RawSizeParameters getNext(); - - @RawField - void setNext(RawSizeParameters value); - } - - /** - * The methods in this class may only be used on {@link RawSizeParameters} that are allocated on - * the stack and therefore not yet visible to other threads. Please keep the methods in this - * class to a minimum. - */ - private static final class RawSizeParametersAccess { - static void initialize(RawSizeParameters valuesOnStack, - UnsignedWord initialEdenSize, UnsignedWord edenSize, UnsignedWord maxEdenSize, - UnsignedWord initialSurvivorSize, UnsignedWord survivorSize, UnsignedWord maxSurvivorSize, - UnsignedWord initialOldSize, UnsignedWord oldSize, UnsignedWord maxOldSize, - UnsignedWord promoSize, - UnsignedWord initialYoungSize, UnsignedWord youngSize, UnsignedWord maxYoungSize, - UnsignedWord minHeapSize, UnsignedWord initialHeapSize, UnsignedWord heapSize, UnsignedWord maxHeapSize) { - assert isAligned(maxHeapSize) && isAligned(maxYoungSize) && isAligned(initialHeapSize) && isAligned(initialEdenSize) && isAligned(initialSurvivorSize); - - assert initialEdenSize.belowOrEqual(initialYoungSize); - assert edenSize.belowOrEqual(youngSize); - assert maxEdenSize.belowOrEqual(maxYoungSize); - - assert initialSurvivorSize.belowOrEqual(initialYoungSize); - assert survivorSize.belowOrEqual(youngSize); - assert maxSurvivorSize.belowOrEqual(maxYoungSize); - - assert initialOldSize.belowOrEqual(initialHeapSize); - assert oldSize.belowOrEqual(heapSize); - assert maxOldSize.belowOrEqual(maxHeapSize); - - assert initialYoungSize.belowOrEqual(initialHeapSize); - assert youngSize.belowOrEqual(heapSize); - assert maxYoungSize.belowOrEqual(maxHeapSize); - - assert minHeapSize.belowOrEqual(initialHeapSize); - assert initialHeapSize.belowOrEqual(maxHeapSize); - assert heapSize.belowOrEqual(maxHeapSize); - assert maxHeapSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()); - - valuesOnStack.setInitialEdenSize(initialEdenSize); - valuesOnStack.setEdenSize(edenSize); - valuesOnStack.setMaxEdenSize(maxEdenSize); - - valuesOnStack.setInitialSurvivorSize(initialSurvivorSize); - valuesOnStack.setSurvivorSize(survivorSize); - valuesOnStack.setMaxSurvivorSize(maxSurvivorSize); - - valuesOnStack.setInitialYoungSize(initialYoungSize); - valuesOnStack.setYoungSize(youngSize); - valuesOnStack.setMaxYoungSize(maxYoungSize); - - valuesOnStack.setInitialOldSize(initialOldSize); - valuesOnStack.setOldSize(oldSize); - valuesOnStack.setMaxOldSize(maxOldSize); - - valuesOnStack.setPromoSize(promoSize); - - valuesOnStack.setMinHeapSize(minHeapSize); - valuesOnStack.setInitialHeapSize(initialHeapSize); - valuesOnStack.setHeapSize(heapSize); - valuesOnStack.setMaxHeapSize(maxHeapSize); - - valuesOnStack.setNext(Word.nullPointer()); - } - } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java index d49684b22230..f61092f206f4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AdaptiveCollectionPolicy.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.Isolates; import com.oracle.svm.core.heap.GCCause; +import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.util.BasedOnJDKFile; import com.oracle.svm.core.util.TimeUtils; import com.oracle.svm.core.util.Timer; @@ -264,45 +265,45 @@ private void computeSurvivorSpaceSizeAndThreshold(boolean isSurvivorOverflow, Un } protected void computeEdenSpaceSize(@SuppressWarnings("unused") boolean completeCollection, @SuppressWarnings("unused") GCCause cause) { - UnsignedWord curEdenSize = sizes.getEdenSize(); - UnsignedWord curPromoSize = sizes.getPromoSize(); + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord curPromo = sizes.getPromoSize(); boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(youngGenChangeForMinorThroughput, minorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, curEdenSize, majorGcCost(), curPromoSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(minorCostEstimator, curEden, majorGcCost(), curPromo); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. */ } - UnsignedWord desiredEdenSize = curEdenSize; + UnsignedWord desiredEdenSize = curEden; if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) { // from adjust_eden_for_throughput(): - UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(curEdenSize); + UnsignedWord edenHeapDelta = edenIncrementWithSupplementAlignedUp(curEden); double scaleByRatio = minorGcCost() / gcCost(); assert scaleByRatio >= 0 && scaleByRatio <= 1; UnsignedWord scaledEdenHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(edenHeapDelta)); desiredEdenSize = alignUp(desiredEdenSize.add(scaledEdenHeapDelta)); - desiredEdenSize = UnsignedUtils.max(desiredEdenSize, curEdenSize); + desiredEdenSize = UnsignedUtils.max(desiredEdenSize, curEden); youngGenChangeForMinorThroughput++; } if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) { - UnsignedWord desiredSum = curEdenSize.add(curPromoSize); - desiredEdenSize = adjustEdenForFootprint(curEdenSize, desiredSum); + UnsignedWord desiredSum = curEden.add(curPromo); + desiredEdenSize = adjustEdenForFootprint(curEden, desiredSum); } assert isAligned(desiredEdenSize); desiredEdenSize = minSpaceSize(desiredEdenSize); - UnsignedWord edenLimit = getMaximumEdenSize(); + UnsignedWord edenLimit = computeEdenLimit(); if (desiredEdenSize.aboveThan(edenLimit)) { /* * If the policy says to get a larger eden but is hitting the limit, don't decrease * eden. This can lead to a general drifting down of the eden size. Let the tenuring * calculation push more into the old gen. */ - desiredEdenSize = UnsignedUtils.max(edenLimit, curEdenSize); + desiredEdenSize = UnsignedUtils.max(edenLimit, curEden); } sizes.setEdenSize(desiredEdenSize); } @@ -486,40 +487,40 @@ public void onCollectionEnd(boolean completeCollection, GCCause cause) { // {maj } private void computeOldGenSpaceSize(UnsignedWord oldLive) { // compute_old_gen_free_space - UnsignedWord curEdenSize = sizes.getEdenSize(); - UnsignedWord curMaxOldSize = sizes.getMaxOldSize(); - UnsignedWord curPromoSize = sizes.getPromoSize(); + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord curPromo = sizes.getPromoSize(); + UnsignedWord curMaxOld = sizes.getMaxOldSize(); avgOldLive.sample(oldLive); // NOTE: if maxOldSize shrunk and difference is negative, unsigned conversion results in 0 - UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(curMaxOldSize) - avgOldLive.getAverage()); - promoLimit = alignDown(UnsignedUtils.max(curPromoSize, promoLimit)); + UnsignedWord promoLimit = UnsignedUtils.fromDouble(UnsignedUtils.toDouble(curMaxOld) - avgOldLive.getAverage()); + promoLimit = alignDown(UnsignedUtils.max(curPromo, promoLimit)); boolean expansionReducesCost = true; // general assumption if (shouldUseEstimator(oldGenChangeForMajorThroughput, majorGcCost())) { - expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, curPromoSize, minorGcCost(), curEdenSize); + expansionReducesCost = expansionSignificantlyReducesTotalCost(majorCostEstimator, curPromo, minorGcCost(), curEden); /* * Note that if the estimator thinks expanding does not lead to significant improvement, * shrink so to not get stuck in a supposed optimum and to keep collecting data points. */ } - UnsignedWord desiredPromoSize = curPromoSize; + UnsignedWord desiredPromoSize = curPromo; if (expansionReducesCost && adjustedMutatorCost() < THROUGHPUT_GOAL && gcCost() > 0) { // from adjust_promo_for_throughput(): - UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(curPromoSize); + UnsignedWord promoHeapDelta = promoIncrementWithSupplementAlignedUp(curPromo); double scaleByRatio = majorGcCost() / gcCost(); assert scaleByRatio >= 0 && scaleByRatio <= 1; UnsignedWord scaledPromoHeapDelta = UnsignedUtils.fromDouble(scaleByRatio * UnsignedUtils.toDouble(promoHeapDelta)); - desiredPromoSize = alignUp(curPromoSize.add(scaledPromoHeapDelta)); - desiredPromoSize = UnsignedUtils.max(desiredPromoSize, curPromoSize); + desiredPromoSize = alignUp(curPromo.add(scaledPromoHeapDelta)); + desiredPromoSize = UnsignedUtils.max(desiredPromoSize, curPromo); oldGenChangeForMajorThroughput++; } if (!expansionReducesCost || (USE_ADAPTIVE_SIZE_POLICY_FOOTPRINT_GOAL && youngGenPolicyIsReady && adjustedMutatorCost() >= THROUGHPUT_GOAL)) { - UnsignedWord desiredSum = curEdenSize.add(curPromoSize); - desiredPromoSize = adjustPromoForFootprint(curPromoSize, desiredSum); + UnsignedWord desiredSum = curEden.add(curPromo); + desiredPromoSize = adjustPromoForFootprint(curPromo, desiredSum); } assert isAligned(desiredPromoSize); desiredPromoSize = minSpaceSize(desiredPromoSize); @@ -530,7 +531,7 @@ private void computeOldGenSpaceSize(UnsignedWord oldLive) { // compute_old_gen_f // from PSOldGen::resize UnsignedWord desiredFreeSpace = calculatedOldFreeSizeInBytes(); UnsignedWord desiredOldSize = alignUp(oldLive.add(desiredFreeSpace)); - sizes.setOldSize(UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), curMaxOldSize)); + sizes.setOldSize(UnsignedUtils.clamp(desiredOldSize, minSpaceSize(), curMaxOld)); } UnsignedWord calculatedOldFreeSizeInBytes() { @@ -580,6 +581,11 @@ protected boolean shouldUpdateStats(GCCause cause) { // should_update_{eden,prom return cause == GenScavengeGCCause.OnAllocation || USE_ADAPTIVE_SIZE_POLICY_WITH_SYSTEM_GC; } + protected UnsignedWord computeEdenLimit() { + assert VMOperation.isGCInProgress() : "result wouldn't be consistent otherwise"; + return alignDown(sizes.getMaxYoungSize().subtract(sizes.getSurvivorSize().multiply(2))); + } + private void updateCollectionEndAverages(AdaptiveWeightedAverage costAverage, AdaptivePaddedAverage pauseAverage, ReciprocalLeastSquareFit costEstimator, AdaptiveWeightedAverage intervalSeconds, GCCause cause, long mutatorNanos, long pauseNanos, UnsignedWord sizeBytes) { if (shouldUpdateStats(cause)) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index 10f276220d92..867beebdd70f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -711,6 +711,7 @@ public void optionValueChanged(NotifyGCRuntimeOptionKey key) { GCImpl.getPolicy().updateSizeParameters(); } + /** For the GC policy, mainly heap-size-related GC options are relevant. */ private static boolean isIrrelevantForGCPolicy(RuntimeOptionKey key) { return key == SubstrateGCOptions.DisableExplicitGC || key == SubstrateGCOptions.PrintGC || diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java index d34b3f3424bc..718d6ae11129 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/LibGraalCollectionPolicy.java @@ -142,19 +142,19 @@ protected boolean shouldUpdateStats(GCCause cause) { protected void computeEdenSpaceSize(boolean completeCollection, GCCause cause) { if (cause == GCCause.HintedGC) { if (completeCollection && lastGCCause == GCCause.HintedGC) { - UnsignedWord curEdenSize = sizes.getEdenSize(); - UnsignedWord newEdenSize = UnsignedUtils.max(sizes.getInitialEdenSize(), alignUp(curEdenSize.unsignedDivide(2))); - if (curEdenSize.aboveThan(newEdenSize)) { - sizes.setEdenSize(newEdenSize); + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord newEden = UnsignedUtils.max(sizes.getInitialEdenSize(), alignUp(curEden.unsignedDivide(2))); + if (curEden.aboveThan(newEden)) { + sizes.setEdenSize(newEden); } } } else { UnsignedWord sizeAfter = GCImpl.getChunkBytes(); if (sizeBefore.notEqual(0) && sizeBefore.belowThan(sizeAfter.multiply(2))) { - UnsignedWord curEdenSize = sizes.getEdenSize(); - UnsignedWord newEdenSize = UnsignedUtils.min(getMaximumEdenSize(), alignUp(curEdenSize.multiply(2))); - if (curEdenSize.belowThan(newEdenSize)) { - sizes.setEdenSize(newEdenSize); + UnsignedWord curEden = sizes.getEdenSize(); + UnsignedWord newEden = UnsignedUtils.min(computeEdenLimit(), alignUp(curEden.multiply(2))); + if (curEden.belowThan(newEden)) { + sizes.setEdenSize(newEden); } } else { super.computeEdenSpaceSize(completeCollection, cause); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java new file mode 100644 index 000000000000..fb24ab3383bb --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParameters.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.genscavenge; + +import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawStructure; +import org.graalvm.word.PointerBase; +import org.graalvm.word.UnsignedWord; + +/** + * Struct that stores all GC-related sizes. See {@link SizeParameters} for more details on its + * lifecycle. When adding fields to this struct, please also change + * {@link RawSizeParametersOnStackAccess#initialize} and {@link SizeParameters#matches} accordingly. + */ +@RawStructure +interface RawSizeParameters extends PointerBase { + @RawField + UnsignedWord getInitialEdenSize(); + + @RawField + void setInitialEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getEdenSize(); + + @RawField + void setEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxEdenSize(); + + @RawField + void setMaxEdenSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialSurvivorSize(); + + @RawField + void setInitialSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getSurvivorSize(); + + @RawField + void setSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxSurvivorSize(); + + @RawField + void setMaxSurvivorSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialYoungSize(); + + @RawField + void setInitialYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getYoungSize(); + + @RawField + void setYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxYoungSize(); + + @RawField + void setMaxYoungSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialOldSize(); + + @RawField + void setInitialOldSize(UnsignedWord value); + + @RawField + UnsignedWord getOldSize(); + + @RawField + void setOldSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxOldSize(); + + @RawField + void setMaxOldSize(UnsignedWord value); + + @RawField + UnsignedWord getPromoSize(); + + @RawField + void setPromoSize(UnsignedWord value); + + @RawField + UnsignedWord getMinHeapSize(); + + @RawField + void setMinHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getInitialHeapSize(); + + @RawField + void setInitialHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getHeapSize(); + + @RawField + void setHeapSize(UnsignedWord value); + + @RawField + UnsignedWord getMaxHeapSize(); + + @RawField + void setMaxHeapSize(UnsignedWord value); + + /** + * Either points to null or to an obsolete {@link RawSizeParameters} struct that wasn't freed + * yet. + */ + @RawField + RawSizeParameters getNext(); + + @RawField + void setNext(RawSizeParameters value); +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java new file mode 100644 index 000000000000..043ff2c5999b --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/RawSizeParametersOnStackAccess.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.genscavenge; + +import static com.oracle.svm.core.genscavenge.AbstractCollectionPolicy.isAligned; + +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.heap.ReferenceAccess; + +import jdk.graal.compiler.word.Word; + +/** + * The methods in this class may only be used on {@link RawSizeParameters} that are allocated on the + * stack and therefore not yet visible to other threads. Please keep the methods in this class to a + * minimum. + */ +final class RawSizeParametersOnStackAccess { + static void initialize(RawSizeParameters valuesOnStack, + UnsignedWord initialEdenSize, UnsignedWord edenSize, UnsignedWord maxEdenSize, + UnsignedWord initialSurvivorSize, UnsignedWord survivorSize, UnsignedWord maxSurvivorSize, + UnsignedWord initialOldSize, UnsignedWord oldSize, UnsignedWord maxOldSize, + UnsignedWord promoSize, + UnsignedWord initialYoungSize, UnsignedWord youngSize, UnsignedWord maxYoungSize, + UnsignedWord minHeapSize, UnsignedWord initialHeapSize, UnsignedWord heapSize, UnsignedWord maxHeapSize) { + assert isAligned(maxHeapSize) && isAligned(maxYoungSize) && isAligned(initialHeapSize) && isAligned(initialEdenSize) && isAligned(initialSurvivorSize); + + assert initialEdenSize.belowOrEqual(initialYoungSize); + assert edenSize.belowOrEqual(youngSize); + assert maxEdenSize.belowOrEqual(maxYoungSize); + + assert initialSurvivorSize.belowOrEqual(initialYoungSize); + assert survivorSize.belowOrEqual(youngSize); + assert maxSurvivorSize.belowOrEqual(maxYoungSize); + + assert initialOldSize.belowOrEqual(initialHeapSize); + assert oldSize.belowOrEqual(heapSize); + assert maxOldSize.belowOrEqual(maxHeapSize); + + assert initialYoungSize.belowOrEqual(initialHeapSize); + assert youngSize.belowOrEqual(heapSize); + assert maxYoungSize.belowOrEqual(maxHeapSize); + + assert minHeapSize.belowOrEqual(initialHeapSize); + assert initialHeapSize.belowOrEqual(maxHeapSize); + assert heapSize.belowOrEqual(maxHeapSize); + assert maxHeapSize.belowOrEqual(ReferenceAccess.singleton().getMaxAddressSpaceSize()); + + valuesOnStack.setInitialEdenSize(initialEdenSize); + valuesOnStack.setEdenSize(edenSize); + valuesOnStack.setMaxEdenSize(maxEdenSize); + + valuesOnStack.setInitialSurvivorSize(initialSurvivorSize); + valuesOnStack.setSurvivorSize(survivorSize); + valuesOnStack.setMaxSurvivorSize(maxSurvivorSize); + + valuesOnStack.setInitialYoungSize(initialYoungSize); + valuesOnStack.setYoungSize(youngSize); + valuesOnStack.setMaxYoungSize(maxYoungSize); + + valuesOnStack.setInitialOldSize(initialOldSize); + valuesOnStack.setOldSize(oldSize); + valuesOnStack.setMaxOldSize(maxOldSize); + + valuesOnStack.setPromoSize(promoSize); + + valuesOnStack.setMinHeapSize(minHeapSize); + valuesOnStack.setInitialHeapSize(initialHeapSize); + valuesOnStack.setHeapSize(heapSize); + valuesOnStack.setMaxHeapSize(maxHeapSize); + + valuesOnStack.setNext(Word.nullPointer()); + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java new file mode 100644 index 000000000000..9c61e2803f7a --- /dev/null +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/SizeParameters.java @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.svm.core.genscavenge; + +import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE; +import static com.oracle.svm.core.genscavenge.AbstractCollectionPolicy.isAligned; + +import org.graalvm.nativeimage.Platform; +import org.graalvm.nativeimage.Platforms; +import org.graalvm.nativeimage.c.struct.SizeOf; +import org.graalvm.word.Pointer; +import org.graalvm.word.UnsignedWord; + +import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; +import com.oracle.svm.core.memory.NullableNativeMemory; +import com.oracle.svm.core.nmt.NmtCategory; +import com.oracle.svm.core.thread.VMOperation; +import com.oracle.svm.core.thread.VMThreads; +import com.oracle.svm.core.util.VMError; + +import jdk.graal.compiler.word.Word; + +/** + * Once a {@link RawSizeParameters} struct is visible to other threads (see {@link #update}), it may + * only be accessed by {@link Uninterruptible} code or when the VM is at a safepoint. This is + * necessary to prevent use-after-free errors because no longer needed {@link RawSizeParameters} + * structs may be freed during a GC. + *

+ * When the VM is at a safepoint, the GC may directly update some of the values in the latest + * {@link RawSizeParameters} (see setters below). + */ +final class SizeParameters { + private static final String ACCESS_RAW_SIZE_PARAMETERS = "Prevent that RawSizeParameters are freed."; + + private volatile RawSizeParameters sizes; + + @Platforms(Platform.HOSTED_ONLY.class) + SizeParameters() { + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + public boolean isInitialized() { + /* + * This only checks for non-null and doesn't access any values in the struct, so inlining + * into interruptible code is allowed. + */ + return sizes.isNonNull(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getInitialEdenSize() { + assert isInitialized(); + return sizes.getInitialEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getEdenSize() { + assert isInitialized(); + return sizes.getEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setEdenSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setEdenSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxEdenSize() { + assert isInitialized(); + return sizes.getMaxEdenSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getInitialSurvivorSize() { + assert isInitialized(); + return sizes.getInitialSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getSurvivorSize() { + assert isInitialized(); + return sizes.getSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setSurvivorSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setSurvivorSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getMaxSurvivorSize() { + assert isInitialized(); + return sizes.getMaxSurvivorSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getInitialYoungSize() { + assert isInitialized(); + return sizes.getInitialYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getYoungSize() { + assert isInitialized(); + return sizes.getYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxYoungSize() { + assert isInitialized(); + return sizes.getMaxYoungSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getInitialOldSize() { + assert isInitialized(); + return sizes.getInitialOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getOldSize() { + assert isInitialized(); + return sizes.getOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setOldSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setOldSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + UnsignedWord getMaxOldSize() { + assert isInitialized(); + return sizes.getMaxOldSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getPromoSize() { + assert isInitialized(); + return sizes.getPromoSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public void setPromoSize(UnsignedWord value) { + assert isInitialized(); + assert VMOperation.isGCInProgress(); + + sizes.setPromoSize(value); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMinHeapSize() { + assert isInitialized(); + return sizes.getMinHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getMaxHeapSize() { + assert isInitialized(); + return sizes.getMaxHeapSize(); + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + public UnsignedWord getHeapSize() { + assert isInitialized(); + assert VMOperation.isGCInProgress() : "use only during GC"; + + return sizes.getHeapSize(); + } + + /** The caller needs to ensure that . */ + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + void update(RawSizeParameters newValuesOnStack) { + RawSizeParameters prevValues = sizes; + if (prevValues.isNonNull() && matches(prevValues, newValuesOnStack)) { + /* Nothing to do - cached params are still accurate. */ + return; + } + + /* Try allocating a struct on the C heap. */ + UnsignedWord structSize = SizeOf.unsigned(RawSizeParameters.class); + RawSizeParameters newValuesOnHeap = NullableNativeMemory.malloc(structSize, NmtCategory.GC); + VMError.guarantee(newValuesOnHeap.isNonNull(), "Out-of-memory while updating GC policy sizes."); + + /* Copy the values from the stack to the C heap. */ + UnmanagedMemoryUtil.copyForward((Pointer) newValuesOnStack, (Pointer) newValuesOnHeap, structSize); + newValuesOnHeap.setNext(prevValues); + + /* + * Publish the new struct via a volatile store. Once the data is published, other threads + * need to see a fully initialized struct right away. This is guaranteed by the implicit + * STORE_STORE barrier before the volatile write. + */ + sizes = newValuesOnHeap; + + assert isAligned(getMaxSurvivorSize()) && isAligned(getInitialYoungSize()) && isAligned(getInitialOldSize()) && isAligned(getMaxOldSize()); + assert getMaxSurvivorSize().belowThan(getMaxYoungSize()); + assert getMaxYoungSize().add(getMaxOldSize()).equal(getMaxHeapSize()); + assert getInitialEdenSize().add(getInitialSurvivorSize().multiply(2)).equal(getInitialYoungSize()); + assert getInitialYoungSize().add(getInitialOldSize()).equal(sizes.getInitialHeapSize()); + } + + /** + * Frees no longer needed {@link RawSizeParameters}, so that only the most recent one remains. + */ + public void freeUnusedSizeParameters() { + assert isInitialized(); + assert VMOperation.isGCInProgress() : "would need to be uninterruptible otherwise"; + + RawSizeParameters cur = sizes; + freeSizeParameters(cur.getNext()); + cur.setNext(Word.nullPointer()); + } + + @Uninterruptible(reason = "Tear-down in progress.") + public void tearDown() { + assert VMThreads.isTearingDown(); + + freeSizeParameters(sizes); + sizes = Word.nullPointer(); + } + + @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true) + private static void freeSizeParameters(RawSizeParameters first) { + assert VMOperation.isGCInProgress() || VMThreads.isTearingDown(); + + RawSizeParameters cur = first; + while (cur.isNonNull()) { + RawSizeParameters next = cur.getNext(); + NullableNativeMemory.free(cur); + cur = next; + } + } + + @Uninterruptible(reason = ACCESS_RAW_SIZE_PARAMETERS) + private static boolean matches(RawSizeParameters a, RawSizeParameters b) { + return a.getInitialEdenSize() == b.getInitialEdenSize() && + a.getEdenSize() == b.getEdenSize() && + a.getMaxEdenSize() == b.getMaxEdenSize() && + + a.getInitialSurvivorSize() == b.getInitialSurvivorSize() && + a.getSurvivorSize() == b.getSurvivorSize() && + a.getMaxSurvivorSize() == b.getMaxSurvivorSize() && + + a.getInitialYoungSize() == b.getInitialYoungSize() && + a.getYoungSize() == b.getYoungSize() && + a.getMaxYoungSize() == b.getMaxYoungSize() && + + a.getInitialOldSize() == b.getInitialOldSize() && + a.getOldSize() == b.getOldSize() && + a.getMaxOldSize() == b.getMaxOldSize() && + + a.getPromoSize() == b.getPromoSize() && + + a.getMinHeapSize() == b.getMinHeapSize() && + a.getInitialHeapSize() == b.getInitialHeapSize() && + a.getHeapSize() == b.getHeapSize() && + a.getMaxHeapSize() == b.getMaxHeapSize(); + } +} diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java index c558da101095..be552833e714 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ThreadLocalAllocation.java @@ -78,14 +78,15 @@ import jdk.graal.compiler.word.Word; /** - * Multiple threads may execute the methods in this class concurrently. All code that is - * transitively reachable from those method can get executed as a side effect of an allocation slow - * path. To prevent hard to debug transient issues, we execute as little code as possible in these - * methods. - * - * If the executed code is too complex, then it can happen that we unexpectedly change some shared - * global state as a side effect of an allocation. This may result in issues that look similar to - * races but that can even happen in single-threaded environments, e.g.: + * Implements the thread-local allocation logic for serial and epsilon GC. + *

+ * Multiple threads may execute the methods in this class concurrently. All code transitively + * reachable from these methods can be executed as a side effect of any Java heap allocation. To + * prevent hard to debug transient issues, we execute as little code as possible in these methods. + *

+ * Executing complex logic in the allocation slow path can modify shared global state, causing + * issues that look similar to race conditions but that can even happen in single-threaded + * environments. For example: * * 

  * {@code
@@ -94,14 +95,14 @@
  * private static synchronized Object createSingleton() {
  *     if (singleton == null) {
  *         Object o = new Object();
- *         // If the allocation above enters the allocation slow path code, and executes a
- *         // complex slow path hook, then it is possible that createSingleton() gets
- *         // recursively execute by the current thread. So, the assertion below may fail
- *         // because the singleton got already initialized by the same thread in the meanwhile.
+ *         // If the allocation above enters the slow path, and if that slow
+ *         // path executes code that calls createSingleton() as well, then
+ *         // the assertion below will fail because the singleton already
+ *         // got initialized by the same thread in the meanwhile.
  *         assert singleton == null;
  *         singleton = o;
  *     }
- *     return result;
+ *     return singleton;
  * }
  * }
  *