Rename nativeGCCollectMode -> nativeGCAfterIteration, auto -> false, iteration -> true

Co-authored-by: ilya-g <ilya.gorbunov@jetbrains.com>

Author: Abduqodiri Qurbonzoda

README.md

@@ -151,10 +151,7 @@ Available configuration options:
 * `nativeFork`
   - "perBenchmark" (default) – executes all iterations of a benchmark in the same process (one binary execution)
   - "perIteration" – executes each iteration of a benchmark in a separate process, measures in cold Kotlin/Native runtime environment
-* `nativeGCCollectMode`
-  - "auto" (default) – collects garbage at the moments choosen by K/N compiler, time for each iteration can be very
-    different because of unpredictable GC calls.
-  - "iteration" – collects garbage after each measuring iteration
+* `nativeGCAfterIteration` – when set to `true`, additionally collects garbage after each measuring iteration (default is `false`).
 * `include("…")` – regular expression to include benchmarks with fully qualified names matching it, as a substring
 * `exclude("…")` – regular expression to exclude benchmarks with fully qualified names matching it, as a substring
 * `param("name", "value1", "value2")` – specify a parameter for a public mutable property `name` annotated with `@Param`

examples/kotlin-multiplatform/build.gradle

@@ -80,7 +80,7 @@ benchmark {
             iterationTimeUnit = "ms" // time in ms per iteration
             advanced("forks", 1)
             advanced("nativeFork", "perIteration")
-            advanced("nativeGCCollectMode", "iteration")
+            advanced("nativeGCAfterIteration", "true")
         }
 
         csv {

runtime/commonMain/src/kotlinx/benchmark/BenchmarkReportFormatter.kt

@@ -175,7 +175,7 @@ private object JsonBenchmarkReportFormatter : BenchmarkReportFormatter() {
           ${result.params.entries.joinToString(separator = ",\n          ") { "\"${it.key}\" : \"${it.value}\"" }}
     },
     "nativeFork" : "${result.config.nativeFork.toText()}",
-    "nativeGCCollectMode" : "${result.config.nativeGCCollectMode.toText()}",
+    "nativeGCAfterIteration" : "${result.config.nativeGCAfterIteration}",
     "primaryMetric" : {
        "score": ${result.score},
        "scoreError": ${result.error},

runtime/commonMain/src/kotlinx/benchmark/CommonBenchmarkAnnotations.kt

@@ -28,10 +28,6 @@ enum class NativeFork {
     PerBenchmark, PerIteration
 }
 
-enum class NativeGCCollectMode {
-    Auto, Iteration
-}
-
 @Target(AnnotationTarget.CLASS)
 expect annotation class OutputTimeUnit(val value: BenchmarkTimeUnit)
 
@@ -71,13 +67,6 @@ fun NativeFork.toText() = when (this) {
     else -> throw UnsupportedOperationException("$this is not supported")
 }
 
-@Suppress("REDUNDANT_ELSE_IN_WHEN")
-fun NativeGCCollectMode.toText() = when (this) {
-    NativeGCCollectMode.Auto -> "auto"
-    NativeGCCollectMode.Iteration -> "iteration"
-    else -> throw UnsupportedOperationException("$this is not supported")
-}
-
 @Suppress("REDUNDANT_ELSE_IN_WHEN")
 fun BenchmarkTimeUnit.toMultiplier() = when (this) {
     BenchmarkTimeUnit.NANOSECONDS -> 1

runtime/commonMain/src/kotlinx/benchmark/ExecutorConfiguration.kt

@@ -8,7 +8,7 @@ class BenchmarkConfiguration private constructor(
     val outputTimeUnit: BenchmarkTimeUnit,
     val mode: Mode,
     val nativeFork: NativeFork,
-    val nativeGCCollectMode: NativeGCCollectMode) {
+    val nativeGCAfterIteration: Boolean) {
 
     constructor(runner: RunnerConfiguration, suite: SuiteDescriptor<*>) : this(
         runner.iterations ?: suite.iterations,
@@ -18,14 +18,14 @@ class BenchmarkConfiguration private constructor(
         runner.outputTimeUnit ?: suite.outputTimeUnit,
         runner.mode ?: suite.mode,
         runner.nativeFork ?: NativeFork.PerBenchmark,
-        runner.nativeGCCollectMode ?: NativeGCCollectMode.Auto
+        runner.nativeGCAfterIteration ?: false
     )
 
     override fun toString() =
         "iterations=$iterations, warmups=$warmups, iterationTime=$iterationTime, " +
                 "iterationTimeUnit=${iterationTimeUnit.toText()}, outputTimeUnit=${outputTimeUnit.toText()}, " +
                 "mode=${mode.toText()}, nativeFork=${nativeFork.toText()}, " +
-                "nativeGCCollectMode=${nativeGCCollectMode.toText()}"
+                "nativeGCAfterIteration=$nativeGCAfterIteration"
 
     companion object {
         fun parse(description: String): BenchmarkConfiguration {
@@ -42,7 +42,7 @@ class BenchmarkConfiguration private constructor(
                 parseTimeUnit(getParameterValue("outputTimeUnit")),
                 getParameterValue("mode").toMode(),
                 NativeFork.valueOf(getParameterValue("nativeFork").replaceFirstChar { it.uppercaseChar() }),
-                NativeGCCollectMode.valueOf(getParameterValue("nativeGCCollectMode").replaceFirstChar { it.uppercaseChar() })
+                getParameterValue("nativeGCAfterIteration").toBooleanStrict()
             )
         }
     }

runtime/commonMain/src/kotlinx/benchmark/RunnerConfiguration.kt

@@ -72,9 +72,7 @@ class RunnerConfiguration(config: String) {
         "nativeFork"
     ) { NativeFork.valueOf(it.replaceFirstChar { firstChar -> firstChar.uppercaseChar() }) }
 
-    val nativeGCCollectMode = singleValueOrNull(
-        "nativeGCCollectMode"
-    ) { NativeGCCollectMode.valueOf(it.replaceFirstChar { firstChar -> firstChar.uppercaseChar() }) }
+    val nativeGCAfterIteration = singleValueOrNull("nativeGCAfterIteration") { it.toBooleanStrict() }
 
     override fun toString(): String {
         return """$name -> $reportFile ($traceFormat, $reportFormat)
@@ -88,7 +86,7 @@ iterationTimeUnit: $iterationTimeUnit
 outputTimeUnit: $outputTimeUnit            
 mode: $mode
 nativeFork: $nativeFork
-nativeGCCollectMode: $nativeGCCollectMode
+nativeGCAfterIteration: $nativeGCAfterIteration
 """
     }
 }

runtime/nativeMain/src/kotlinx/benchmark/native/NativeExecutor.kt

@@ -123,7 +123,7 @@ class NativeExecutor(
             // Execute warmup
             val cycles = cyclesPerIteration ?: warmup(suite.name, benchmarkRun.config, instance, benchmark)
             DoubleArray(iterations) { iteration ->
-                val nanosecondsPerOperation = measure(instance, benchmark, cycles, benchmarkRun.config.nativeGCCollectMode)
+                val nanosecondsPerOperation = measure(instance, benchmark, cycles, benchmarkRun.config.nativeGCAfterIteration)
                 val text = nanosecondsPerOperation.nanosToText(benchmarkRun.config.mode, benchmarkRun.config.outputTimeUnit)
                 val iterationNumber = currentIteration ?: iteration
                 reporter.output(
@@ -210,18 +210,18 @@ class NativeExecutor(
         instance: T,
         benchmark: BenchmarkDescriptor<T>,
         cycles: Int,
-        nativeGCCollectMode: NativeGCCollectMode
+        nativeGCAfterIteration: Boolean
     ): Double {
         val executeFunction = benchmark.function
         var counter = cycles
-        if (nativeGCCollectMode == NativeGCCollectMode.Iteration)
+        if (nativeGCAfterIteration)
             GC.collect()
         val startTime = getTimeNanos()
         while (counter-- > 0) {
             @Suppress("UNUSED_VARIABLE")
             val result = instance.executeFunction() // ignore result for now, but might need to consume it somehow
         }
-        if (nativeGCCollectMode == NativeGCCollectMode.Iteration)
+        if (nativeGCAfterIteration)
             GC.collect()
         val endTime = getTimeNanos()
         val time = endTime - startTime
@@ -246,7 +246,7 @@ class NativeExecutor(
             val benchmarkNanos = config.iterationTime * config.iterationTimeUnit.toMultiplier()
             val executeFunction = benchmark.function
 
-            if (config.nativeGCCollectMode == NativeGCCollectMode.Iteration)
+            if (config.nativeGCAfterIteration)
                 GC.collect()
             val startTime = getTimeNanos()
             var endTime = startTime
@@ -256,7 +256,7 @@ class NativeExecutor(
                 endTime = getTimeNanos()
                 iterations++
             }
-            if (config.nativeGCCollectMode == NativeGCCollectMode.Iteration)
+            if (config.nativeGCAfterIteration)
                 GC.collect()
 
             val time = endTime - startTime