Add MCP conformance test coverage

Author: skarpovdev

.github/workflows/build.yml

@@ -28,6 +28,11 @@ jobs:
           java-version: '21'
           distribution: 'temurin'
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v5
         with:
@@ -57,6 +62,16 @@ jobs:
         working-directory: ./samples/weather-stdio-server
         run: ./gradlew --no-daemon clean build -Pmcp.kotlin.overrideVersion=0.0.1-SNAPSHOT
 
+      - name: Run Conformance Tests
+        run: ./gradlew conformance
+
+      - name: Upload Conformance Results
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: conformance-results
+          path: kotlin-sdk-test/results/
+
       - name: Upload Reports
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v5

.gitignore

@@ -56,3 +56,6 @@ dist
 ### SWE agents ###
 .claude/
 .junie/
+
+### Conformance test results ###
+kotlin-sdk-test/results/

kotlin-sdk-test/build.gradle.kts

@@ -1,3 +1,6 @@
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJvmCompilation
+
 plugins {
     id("mcp.multiplatform")
 }
@@ -31,3 +34,37 @@ kotlin {
         }
     }
 }
+
+tasks.register<Test>("conformance") {
+    group = "conformance"
+    description = "Run MCP conformance tests with detailed output"
+
+    val jvmCompilation = kotlin.targets["jvm"].compilations["test"] as KotlinJvmCompilation
+    testClassesDirs = jvmCompilation.output.classesDirs
+    classpath = jvmCompilation.runtimeDependencyFiles
+
+    useJUnitPlatform()
+
+    filter {
+        includeTestsMatching("*ConformanceTest*")
+    }
+
+    testLogging {
+        events("passed", "skipped", "failed")
+        showStandardStreams = true
+        showExceptions = true
+        showCauses = true
+        showStackTraces = true
+        exceptionFormat = TestExceptionFormat.FULL
+    }
+
+    doFirst {
+        systemProperty("test.classpath", classpath.asPath)
+
+        println("\n" + "=".repeat(60))
+        println("MCP CONFORMANCE TESTS")
+        println("=".repeat(60))
+        println("These tests validate compliance with the MCP specification.")
+        println("=".repeat(60) + "\n")
+    }
+}

kotlin-sdk-test/src/jvmTest/kotlin/io/modelcontextprotocol/kotlin/sdk/conformance/ConformanceClient.kt

@@ -0,0 +1,100 @@
+package io.modelcontextprotocol.kotlin.sdk.conformance
+
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.ktor.client.HttpClient
+import io.ktor.client.engine.cio.CIO
+import io.ktor.client.plugins.sse.SSE
+import io.modelcontextprotocol.kotlin.sdk.client.Client
+import io.modelcontextprotocol.kotlin.sdk.client.StreamableHttpClientTransport
+import io.modelcontextprotocol.kotlin.sdk.shared.Transport
+import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequest
+import io.modelcontextprotocol.kotlin.sdk.types.CallToolRequestParams
+import io.modelcontextprotocol.kotlin.sdk.types.Implementation
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.buildJsonObject
+
+private val logger = KotlinLogging.logger {}
+
+fun main(args: Array<String>) {
+    require(args.isNotEmpty()) {
+        "Usage: ConformanceClient <server-url>"
+    }
+
+    val serverUrl = args.last()
+    logger.info { "Connecting to test server at: $serverUrl" }
+
+    val httpClient = HttpClient(CIO) {
+        install(SSE)
+    }
+    val transport: Transport = StreamableHttpClientTransport(httpClient, serverUrl)
+
+    val client = Client(
+        clientInfo = Implementation(
+            name = "kotlin-conformance-client",
+            version = "1.0.0"
+        )
+    )
+
+    var exitCode = 0
+
+    runBlocking {
+        try {
+            client.connect(transport)
+            logger.info { "✅ Connected to server successfully" }
+
+            try {
+                val tools = client.listTools()
+                logger.info { "Available tools: ${tools.tools.map { it.name }}" }
+
+                val addNumbersTool = tools.tools.find { it.name == "add_numbers" }
+                if (addNumbersTool != null) {
+                    logger.info { "Calling tool: add_numbers" }
+                    val result = client.callTool(
+                        CallToolRequest(
+                            params = CallToolRequestParams(
+                                name = "add_numbers",
+                                arguments = buildJsonObject {
+                                    put("a", JsonPrimitive(5))
+                                    put("b", JsonPrimitive(3))
+                                }
+                            )
+                        )
+                    )
+                    logger.info { "Tool result: ${result.content}" }
+                } else if (tools.tools.isNotEmpty()) {
+                    val toolName = tools.tools.first().name
+                    logger.info { "Calling tool: $toolName" }
+
+                    val result = client.callTool(
+                        CallToolRequest(
+                            params = CallToolRequestParams(
+                                name = toolName,
+                                arguments = buildJsonObject {
+                                    put("input", JsonPrimitive("test"))
+                                }
+                            )
+                        )
+                    )
+                    logger.info { "Tool result: ${result.content}" }
+                }
+            } catch (e: Exception) {
+                logger.debug(e) { "Error during tool operations (may be expected for some scenarios)" }
+            }
+
+            logger.info { "✅ Client operations completed successfully" }
+        } catch (e: Exception) {
+            logger.error(e) { "❌ Client failed" }
+            exitCode = 1
+        } finally {
+            try {
+                transport.close()
+            } catch (e: Exception) {
+                logger.warn(e) { "Error closing transport" }
+            }
+            httpClient.close()
+        }
+    }
+
+    kotlin.system.exitProcess(exitCode)
+}