MCP Prerelease Compatibility Testing #20
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MCP Prerelease Compatibility Testing | |
| on: | |
| schedule: | |
| # Run daily at 2 AM UTC to catch new prereleases | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| test_specific_version: | |
| description: 'Specific version to test (optional)' | |
| required: false | |
| type: string | |
| jobs: | |
| discover-mcp-prereleases: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| mcp-prereleases: ${{ steps.get-mcp-prereleases.outputs.versions }} | |
| has-prereleases: ${{ steps.get-mcp-prereleases.outputs.has-prereleases }} | |
| steps: | |
| - name: Get available MCP prerelease versions | |
| id: get-mcp-prereleases | |
| run: | | |
| # Get all available versions from PyPI including prereleases | |
| all_versions=$(pip index versions mcp --pre 2>/dev/null | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+[ab][0-9]\+\|[0-9]\+\.[0-9]\+\.[0-9]\+rc[0-9]\+' | sort -V || echo "") | |
| if [ -z "$all_versions" ]; then | |
| echo "No MCP prereleases found" | |
| echo "versions=[]" >> $GITHUB_OUTPUT | |
| echo "has-prereleases=false" >> $GITHUB_OUTPUT | |
| else | |
| # Get only the latest prerelease | |
| latest=$(echo "$all_versions" | tail -n 1) | |
| json_array="[\"$latest\"]" | |
| echo "Found MCP prerelease: $json_array" | |
| echo "versions=$json_array" >> $GITHUB_OUTPUT | |
| echo "has-prereleases=true" >> $GITHUB_OUTPUT | |
| fi | |
| discover-fastmcp-prereleases: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| fastmcp-prereleases: ${{ steps.get-fastmcp-prereleases.outputs.versions }} | |
| has-prereleases: ${{ steps.get-fastmcp-prereleases.outputs.has-prereleases }} | |
| steps: | |
| - name: Get available FastMCP prerelease versions | |
| id: get-fastmcp-prereleases | |
| run: | | |
| # Get all available versions from PyPI including prereleases | |
| all_versions=$(pip index versions fastmcp --pre 2>/dev/null | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+[ab][0-9]\+\|[0-9]\+\.[0-9]\+\.[0-9]\+rc[0-9]\+' | sort -V || echo "") | |
| if [ -z "$all_versions" ]; then | |
| echo "No FastMCP prereleases found" | |
| echo "versions=[]" >> $GITHUB_OUTPUT | |
| echo "has-prereleases=false" >> $GITHUB_OUTPUT | |
| else | |
| # Get only the latest prerelease | |
| latest=$(echo "$all_versions" | tail -n 1) | |
| json_array="[\"$latest\"]" | |
| echo "Found FastMCP prerelease: $json_array" | |
| echo "versions=$json_array" >> $GITHUB_OUTPUT | |
| echo "has-prereleases=true" >> $GITHUB_OUTPUT | |
| fi | |
| test-fastmcp-prerelease: | |
| runs-on: ubuntu-latest | |
| needs: discover-fastmcp-prereleases | |
| if: needs.discover-fastmcp-prereleases.outputs.has-prereleases == 'true' | |
| strategy: | |
| matrix: | |
| fastmcp-version: ${{ fromJson(needs.discover-fastmcp-prereleases.outputs.fastmcp-prereleases) }} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| version: "latest" | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Update pyproject.toml with prerelease FastMCP ${{ matrix.fastmcp-version }} | |
| run: | | |
| # Update the community extras section with the specific FastMCP prerelease version | |
| sed -i -E 's/"fastmcp[>=]+[^"]+"/\"fastmcp==${{ matrix.fastmcp-version }}\"/' pyproject.toml | |
| # Keep MCP flexible so FastMCP can choose its version | |
| sed -i -E 's/"mcp[>=]+[^"]+"/\"mcp>=1.2.0\"/' pyproject.toml | |
| # Show the updated dependencies for verification | |
| echo "Updated dependencies:" | |
| grep -A 5 "^dependencies = \[" pyproject.toml | |
| echo "" | |
| echo "Updated community extras:" | |
| grep -A 2 "^\[project.optional-dependencies\]" pyproject.toml | grep -A 2 "community" | |
| - name: Install dependencies with prerelease FastMCP ${{ matrix.fastmcp-version }} | |
| run: | | |
| # Install with both dev and community extras, allow prereleases | |
| # FastMCP will determine which MCP version it needs | |
| uv sync --extra dev --extra community --prerelease allow | |
| - name: Show installed versions | |
| run: | | |
| echo "Installed package versions:" | |
| uv pip show mcp | grep "Version:" || echo "MCP not found" | |
| uv pip show fastmcp | grep "Version:" || echo "FastMCP not found" | |
| echo "" | |
| echo "Full MCP package info:" | |
| uv pip show mcp || echo "MCP package details not available" | |
| - name: Run community FastMCP tests with prerelease version ${{ matrix.fastmcp-version }} | |
| run: | | |
| echo "Running community FastMCP tests with prerelease version ${{ matrix.fastmcp-version }}" | |
| set -o pipefail | |
| uv run pytest -v 2>&1 | tee pytest.log | |
| - name: Collect failure details | |
| if: failure() | |
| env: | |
| PACKAGE_UNDER_TEST: fastmcp | |
| PACKAGE_VERSION: ${{ matrix.fastmcp-version }} | |
| run: | | |
| python .github/scripts/collect_failure.py pytest.log | |
| - name: Upload failure artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: prerelease-failure-fastmcp-${{ matrix.fastmcp-version }} | |
| path: failure.json | |
| retention-days: 7 | |
| test-mcp-prerelease: | |
| runs-on: ubuntu-latest | |
| needs: discover-mcp-prereleases | |
| if: needs.discover-mcp-prereleases.outputs.has-prereleases == 'true' | |
| strategy: | |
| matrix: | |
| mcp-version: ${{ fromJson(needs.discover-mcp-prereleases.outputs.mcp-prereleases) }} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| version: "latest" | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies with prerelease MCP ${{ matrix.mcp-version }} (no FastMCP) | |
| run: | | |
| # Create virtual environment | |
| uv venv | |
| # Install base dependencies first | |
| uv pip install pydantic>=2.0.0 requests>=2.31.0 mcpcat-api==0.1.4 | |
| # Install the specific MCP prerelease version | |
| uv pip install "mcp==${{ matrix.mcp-version }}" --prerelease=allow | |
| # Install dev dependencies for testing | |
| uv pip install pytest>=7.0.0 pytest-asyncio>=0.21.0 mypy>=1.0.0 ruff>=0.1.0 freezegun>=1.2.0 | |
| # Explicitly uninstall fastmcp if it somehow got installed | |
| uv pip uninstall fastmcp -y 2>/dev/null || true | |
| # Install the package itself in editable mode | |
| uv pip install -e . | |
| - name: Verify FastMCP is not installed | |
| run: | | |
| if uv pip show fastmcp 2>/dev/null; then | |
| echo "ERROR: fastmcp should not be installed for MCP-only tests" | |
| exit 1 | |
| else | |
| echo "✓ Confirmed: fastmcp is not installed" | |
| fi | |
| - name: Show installed MCP version | |
| run: | | |
| echo "Installed MCP version:" | |
| uv pip show mcp | grep "Version:" || echo "MCP not found" | |
| echo "" | |
| echo "Full MCP package info:" | |
| uv pip show mcp || echo "MCP package details not available" | |
| - name: Run tests with prerelease MCP ${{ matrix.mcp-version }} | |
| run: | | |
| echo "Running tests with prerelease MCP version ${{ matrix.mcp-version }} (no FastMCP)" | |
| # Run all tests except community tests | |
| set -o pipefail | |
| uv run pytest -v 2>&1 | tee pytest.log | |
| - name: Collect failure details | |
| if: failure() | |
| env: | |
| PACKAGE_UNDER_TEST: mcp | |
| PACKAGE_VERSION: ${{ matrix.mcp-version }} | |
| run: | | |
| python .github/scripts/collect_failure.py pytest.log | |
| - name: Upload failure artifact | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: prerelease-failure-mcp-${{ matrix.mcp-version }} | |
| path: failure.json | |
| retention-days: 7 | |
| send-failure-notification: | |
| runs-on: ubuntu-latest | |
| needs: [discover-mcp-prereleases, discover-fastmcp-prereleases, test-fastmcp-prerelease, test-mcp-prerelease] | |
| if: failure() | |
| steps: | |
| - name: Download failure details | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: prerelease-failure-* | |
| path: failure_artifacts | |
| merge-multiple: true | |
| if-no-files-found: ignore | |
| - name: Send failure notification email | |
| run: | | |
| # Install resend | |
| pip install resend | |
| # Send email notification | |
| python -c " | |
| import resend | |
| import os | |
| import sys | |
| import json | |
| from pathlib import Path | |
| # Get sensitive data from environment | |
| api_key = os.environ.get('RESEND_API_KEY') | |
| alert_email = os.environ.get('ALERT_EMAIL') | |
| if not api_key: | |
| print('Error: RESEND_API_KEY secret not configured') | |
| sys.exit(1) | |
| if not alert_email: | |
| print('Error: ALERT_EMAIL secret not configured') | |
| sys.exit(1) | |
| # Set API key without printing it | |
| resend.api_key = api_key | |
| # Determine which tests failed | |
| workflow_url = 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| # Get the prerelease versions that were being tested | |
| mcp_prereleases = '${{ needs.discover-mcp-prereleases.outputs.mcp-prereleases }}' | |
| fastmcp_prereleases = '${{ needs.discover-fastmcp-prereleases.outputs.fastmcp-prereleases }}' | |
| has_mcp_prereleases = '${{ needs.discover-mcp-prereleases.outputs.has-prereleases }}' | |
| has_fastmcp_prereleases = '${{ needs.discover-fastmcp-prereleases.outputs.has-prereleases }}' | |
| def parse_versions(raw: str, label: str, enabled: bool) -> list[str]: | |
| raw = (raw or '').strip() | |
| if not raw or not enabled: | |
| return [] | |
| try: | |
| parsed = json.loads(raw) | |
| except json.JSONDecodeError as exc: | |
| print(f'Warning: Could not parse {label} JSON: {exc}') | |
| cleaned = raw.strip('[]') | |
| if not cleaned: | |
| return [] | |
| items = [] | |
| for piece in cleaned.split(','): | |
| candidate = piece.strip() | |
| candidate = candidate.strip(chr(39)) | |
| candidate = candidate.strip(chr(34)) | |
| if candidate: | |
| items.append(candidate) | |
| return items | |
| if isinstance(parsed, list): | |
| return [str(item).strip() for item in parsed if str(item).strip()] | |
| print( | |
| f'Warning: Unexpected {label} payload type: ' | |
| f'{type(parsed).__name__}' | |
| ) | |
| return [] | |
| mcp_list = parse_versions( | |
| mcp_prereleases, | |
| 'MCP prereleases', | |
| has_mcp_prereleases == 'true' | |
| ) | |
| fastmcp_list = parse_versions( | |
| fastmcp_prereleases, | |
| 'FastMCP prereleases', | |
| has_fastmcp_prereleases == 'true' | |
| ) | |
| print( | |
| f'Using {len(mcp_list)} MCP prereleases and ' | |
| f'{len(fastmcp_list)} FastMCP prereleases' | |
| ) | |
| # Format version lists | |
| mcp_versions_html = '<br>'.join([f' • MCP {v}' for v in mcp_list]) if mcp_list else 'No MCP prereleases tested' | |
| fastmcp_versions_html = '<br>'.join([f' • FastMCP {v}' for v in fastmcp_list]) if fastmcp_list else 'No FastMCP prereleases tested' | |
| failure_dir = Path('failure_artifacts') | |
| failure_details = [] | |
| if failure_dir.exists(): | |
| for failure_path in failure_dir.rglob('failure.json'): | |
| try: | |
| failure_payload = json.loads(failure_path.read_text()) | |
| except Exception as exc: | |
| print(f'Warning: Could not parse failure file {failure_path}: {exc}') | |
| continue | |
| failure_payload['_source'] = str(failure_path) | |
| failure_details.append(failure_payload) | |
| print(f'Collected {len(failure_details)} failure detail files') | |
| def failing_versions(package_name: str) -> list[str]: | |
| versions = [] | |
| for entry in failure_details: | |
| package_value = str(entry.get('package', '')).lower() | |
| if package_value != package_name.lower(): | |
| continue | |
| version_value = str(entry.get('version', '')).strip() | |
| if version_value and version_value not in versions: | |
| versions.append(version_value) | |
| return versions | |
| failing_mcp = failing_versions('mcp') | |
| failing_fastmcp = failing_versions('fastmcp') | |
| failing_sections = [] | |
| if failing_mcp: | |
| failing_sections.append( | |
| '<p><strong>MCP Prerelease Versions Failing:</strong><br>' | |
| + '<br>'.join([f' • MCP {version}' for version in failing_mcp]) | |
| + '</p>' | |
| ) | |
| if failing_fastmcp: | |
| failing_sections.append( | |
| '<p><strong>FastMCP Prerelease Versions Failing:</strong><br>' | |
| + '<br>'.join([f' • FastMCP {version}' for version in failing_fastmcp]) | |
| + '</p>' | |
| ) | |
| if not failing_sections: | |
| if failure_details: | |
| failing_sections.append( | |
| '<p>Failures detected but version details were unavailable.</p>' | |
| ) | |
| else: | |
| failing_sections.append( | |
| '<p>No failure artifacts were collected. Review the workflow logs for more information.</p>' | |
| ) | |
| failing_versions_html = '\n'.join(failing_sections) | |
| subject_parts: list[str] = [] | |
| if failing_mcp: | |
| subject_parts.append('MCP ' + ', '.join(failing_mcp)) | |
| if failing_fastmcp: | |
| subject_parts.append('FastMCP ' + ', '.join(failing_fastmcp)) | |
| subject_suffix = ' | '.join(subject_parts) if subject_parts else 'See workflow logs' | |
| subject_line = f'⚠️ MCPCat Prerelease Tests Failed - {subject_suffix}' | |
| print(f'Email subject suffix: {subject_suffix}') | |
| email_html = f''' | |
| <h2>⚠️ MCPCat Prerelease Compatibility Tests Failed</h2> | |
| <p>The prerelease compatibility tests have failed. This may indicate upcoming breaking changes in MCP or FastMCP.</p> | |
| <h3>Details:</h3> | |
| <ul> | |
| <li>Workflow: ${{ github.workflow }}</li> | |
| <li>Repository: ${{ github.repository }}</li> | |
| <li>Branch: ${{ github.ref }}</li> | |
| <li>Run ID: ${{ github.run_id }}</li> | |
| </ul> | |
| <p><a href=\"{workflow_url}\">View full workflow run details</a></p> | |
| <h3>Failing Versions:</h3> | |
| {failing_versions_html} | |
| <h3>Prerelease Versions Tested:</h3> | |
| <p><strong>MCP Prerelease Versions:</strong><br> | |
| {mcp_versions_html}</p> | |
| <p><strong>FastMCP Prerelease Versions:</strong><br> | |
| {fastmcp_versions_html}</p> | |
| <h3>What to check:</h3> | |
| <ul> | |
| <li>Review the specific failing versions in the workflow</li> | |
| <li>Check for deprecation warnings or breaking changes</li> | |
| <li>Update compatibility if needed before the stable release</li> | |
| <li>Consider if changes are needed to support upcoming versions</li> | |
| </ul> | |
| ''' | |
| # Replace slashes with underscores for Resend tags | |
| repo_tag = '${{ github.repository }}'.replace('/', '_') | |
| params = { | |
| 'from': 'no-reply@mcpcat.io', # Use Resend's default verified domain | |
| 'to': [alert_email], # Use variable, not inline secret | |
| 'subject': subject_line, | |
| 'html': email_html, | |
| 'tags': [ | |
| {'name': 'alert_type', 'value': 'prerelease_failure'}, | |
| {'name': 'repository', 'value': repo_tag} | |
| ] | |
| } | |
| try: | |
| # Validate email format | |
| if not alert_email or '@' not in alert_email: | |
| print('Error: Invalid email address format') | |
| sys.exit(1) | |
| # Send email without logging sensitive params | |
| email = resend.Emails.send(params) | |
| # Only print success message, not the email object which might contain sensitive data | |
| print('Email notification sent successfully') | |
| except resend.exceptions.ValidationError as e: | |
| # ValidationError might mean the API key is invalid or email format issue | |
| print(f'Resend ValidationError - Check that:') | |
| print('1. RESEND_API_KEY is valid and starts with re_') | |
| print('2. ALERT_EMAIL is a valid email address') | |
| print('3. The from address domain is verified in Resend') | |
| sys.exit(1) | |
| except Exception as e: | |
| # Print error type without sensitive details | |
| print(f'Failed to send email notification: {type(e).__name__}') | |
| # Exit with error code to indicate failure (but workflow continues) | |
| sys.exit(1) | |
| " | |
| env: | |
| RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} | |
| ALERT_EMAIL: ${{ secrets.ALERT_EMAIL }} | |
| report-prerelease-compatibility: | |
| runs-on: ubuntu-latest | |
| needs: [discover-mcp-prereleases, discover-fastmcp-prereleases, test-fastmcp-prerelease, test-mcp-prerelease, send-failure-notification] | |
| if: always() | |
| steps: | |
| - name: Generate Prerelease Compatibility Report | |
| run: | | |
| echo "==========================================" | |
| echo " 🔬 PRERELEASE COMPATIBILITY REPORT " | |
| echo "==========================================" | |
| echo "" | |
| echo "⚠️ PRERELEASE VERSION TESTING ⚠️" | |
| echo "This report shows compatibility with PRERELEASE versions" | |
| echo "which may be unstable or have breaking changes." | |
| echo "" | |
| echo "Generated on: $(date)" | |
| echo "" | |
| echo "📦 MCP PRERELEASE STATUS" | |
| echo "────────────────────────────────────────" | |
| if [ "${{ needs.discover-mcp-prereleases.outputs.has-prereleases }}" == "true" ]; then | |
| echo "Prerelease versions tested: ${{ needs.discover-mcp-prereleases.outputs.mcp-prereleases }}" | |
| echo "Test Status: Check individual job results above" | |
| else | |
| echo "No MCP prerelease versions currently available" | |
| fi | |
| echo "" | |
| echo "📦 FASTMCP PRERELEASE STATUS" | |
| echo "────────────────────────────────────────" | |
| if [ "${{ needs.discover-fastmcp-prereleases.outputs.has-prereleases }}" == "true" ]; then | |
| echo "Prerelease versions tested: ${{ needs.discover-fastmcp-prereleases.outputs.fastmcp-prereleases }}" | |
| echo "Test Status: Check individual job results above" | |
| echo "Note: FastMCP determines its own MCP dependency version" | |
| else | |
| echo "No FastMCP prerelease versions currently available" | |
| fi | |
| echo "" | |
| echo "🔍 TEST CONFIGURATION" | |
| echo "────────────────────────────────────────" | |
| echo "• Python version: 3.12" | |
| echo "• Email alerts: Sent on failure via Resend" | |
| echo "• MCP dependency: >=1.2.0 (flexible, FastMCP determines actual version)" | |
| echo "" | |
| echo "📊 HOW TO INTERPRET RESULTS" | |
| echo "────────────────────────────────────────" | |
| echo "✅ PASSED - Prerelease is compatible" | |
| echo "❌ FAILED - Prerelease has compatibility issues" | |
| echo "" | |
| echo "Failed tests will trigger email notifications for early warning" | |
| echo "about upcoming compatibility changes in MCP/FastMCP prereleases." | |
| echo "" | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| echo "📝 MANUAL RUN DETAILS" | |
| echo "────────────────────────────────────────" | |
| echo "Triggered manually via workflow_dispatch" | |
| if [ -n "${{ github.event.inputs.test_specific_version }}" ]; then | |
| echo "Specific version requested: ${{ github.event.inputs.test_specific_version }}" | |
| fi | |
| echo "" | |
| fi | |
| echo "==========================================" |