Skip to content

Commit 09f0394

Browse files
jprakash-dbjprakash-db
committed
merged main
2 parents 6eba353 + 701f7f6 commit 09f0394

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+9036
-535
lines changed

‎.github/workflows/coverage-check.yml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: Code Coverage
2+
3+
permissions:
4+
contents: read
5+
6+
on: [pull_request, workflow_dispatch]
7+
8+
jobs:
9+
coverage:
10+
runs-on: ubuntu-latest
11+
environment: azure-prod
12+
env:
13+
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
14+
DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
15+
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
16+
DATABRICKS_CATALOG: peco
17+
DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
18+
steps:
19+
#----------------------------------------------
20+
# check-out repo and set-up python
21+
#----------------------------------------------
22+
- name: Check out repository
23+
uses: actions/checkout@v4
24+
with:
25+
fetch-depth: 0 # Needed for coverage comparison
26+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
27+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
28+
- name: Set up python
29+
id: setup-python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.10"
33+
#----------------------------------------------
34+
# ----- install & configure poetry -----
35+
#----------------------------------------------
36+
- name: Install Poetry
37+
uses: snok/install-poetry@v1
38+
with:
39+
virtualenvs-create: true
40+
virtualenvs-in-project: true
41+
installer-parallel: true
42+
43+
#----------------------------------------------
44+
# load cached venv if cache exists
45+
#----------------------------------------------
46+
- name: Load cached venv
47+
id: cached-poetry-dependencies
48+
uses: actions/cache@v4
49+
with:
50+
path: .venv
51+
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
52+
#----------------------------------------------
53+
# install dependencies if cache does not exist
54+
#----------------------------------------------
55+
- name: Install dependencies
56+
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
57+
run: poetry install --no-interaction --no-root
58+
#----------------------------------------------
59+
# install your root project, if required
60+
#----------------------------------------------
61+
- name: Install library
62+
run: poetry install --no-interaction --all-extras
63+
#----------------------------------------------
64+
# run all tests
65+
#----------------------------------------------
66+
- name: Run tests with coverage
67+
continue-on-error: true
68+
run: |
69+
poetry run python -m pytest \
70+
tests/unit tests/e2e \
71+
--cov=src --cov-report=xml --cov-report=term -v
72+
#----------------------------------------------
73+
# check for coverage override
74+
#----------------------------------------------
75+
- name: Check for coverage override
76+
id: override
77+
run: |
78+
OVERRIDE_COMMENT=$(echo "${{ github.event.pull_request.body }}" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "")
79+
if [ -n "$OVERRIDE_COMMENT" ]; then
80+
echo "override=true" >> $GITHUB_OUTPUT
81+
REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/')
82+
echo "reason=$REASON" >> $GITHUB_OUTPUT
83+
echo "Coverage override found in PR description: $REASON"
84+
else
85+
echo "override=false" >> $GITHUB_OUTPUT
86+
echo "No coverage override found"
87+
fi
88+
#----------------------------------------------
89+
# check coverage percentage
90+
#----------------------------------------------
91+
- name: Check coverage percentage
92+
if: steps.override.outputs.override == 'false'
93+
run: |
94+
COVERAGE_FILE="coverage.xml"
95+
if [ ! -f "$COVERAGE_FILE" ]; then
96+
echo "ERROR: Coverage file not found at $COVERAGE_FILE"
97+
exit 1
98+
fi
99+
100+
# Install xmllint if not available
101+
if ! command -v xmllint &> /dev/null; then
102+
sudo apt-get update && sudo apt-get install -y libxml2-utils
103+
fi
104+
105+
COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
106+
TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")
107+
PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")
108+
109+
echo "Branch Coverage: $PERCENTAGE%"
110+
echo "Required Coverage: 85%"
111+
112+
# Use Python to compare the coverage with 85
113+
python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)"
114+
if [ $? -eq 1 ]; then
115+
echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%"
116+
exit 1
117+
else
118+
echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%"
119+
fi
120+
121+
#----------------------------------------------
122+
# coverage enforcement summary
123+
#----------------------------------------------
124+
- name: Coverage enforcement summary
125+
run: |
126+
if [ "${{ steps.override.outputs.override }}" == "true" ]; then
127+
echo "⚠️ Coverage checks bypassed: ${{ steps.override.outputs.reason }}"
128+
echo "Please ensure this override is justified and temporary"
129+
else
130+
echo "✅ Coverage checks enforced - minimum 85% required"
131+
fi

‎examples/experimental/sea_connector_test.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
"""
2+
Main script to run all SEA connector tests.
3+
4+
This script runs all the individual test modules and displays
5+
a summary of test results with visual indicators.
6+
7+
In order to run the script, the following environment variables need to be set:
8+
- DATABRICKS_SERVER_HOSTNAME: The hostname of the Databricks server
9+
- DATABRICKS_HTTP_PATH: The HTTP path of the Databricks server
10+
- DATABRICKS_TOKEN: The token to use for authentication
11+
"""
12+
13+
import os
14+
import sys
15+
import logging
16+
import subprocess
17+
from typing import List, Tuple
18+
19+
logging.basicConfig(level=logging.DEBUG)
20+
logger = logging.getLogger(__name__)
21+
22+
TEST_MODULES = [
23+
"test_sea_session",
24+
"test_sea_sync_query",
25+
"test_sea_async_query",
26+
"test_sea_metadata",
27+
]
28+
29+
30+
def run_test_module(module_name: str) -> bool:
31+
"""Run a test module and return success status."""
32+
module_path = os.path.join(
33+
os.path.dirname(os.path.abspath(__file__)), "tests", f"{module_name}.py"
34+
)
35+
36+
# Simply run the module as a script - each module handles its own test execution
37+
result = subprocess.run(
38+
[sys.executable, module_path], capture_output=True, text=True
39+
)
40+
41+
# Log the output from the test module
42+
if result.stdout:
43+
for line in result.stdout.strip().split("\n"):
44+
logger.info(line)
45+
46+
if result.stderr:
47+
for line in result.stderr.strip().split("\n"):
48+
logger.error(line)
49+
50+
return result.returncode == 0
51+
52+
53+
def run_tests() -> List[Tuple[str, bool]]:
54+
"""Run all tests and return results."""
55+
results = []
56+
57+
for module_name in TEST_MODULES:
58+
try:
59+
logger.info(f"\n{'=' * 50}")
60+
logger.info(f"Running test: {module_name}")
61+
logger.info(f"{'-' * 50}")
62+
63+
success = run_test_module(module_name)
64+
results.append((module_name, success))
65+
66+
status = "✅ PASSED" if success else "❌ FAILED"
67+
logger.info(f"Test {module_name}: {status}")
68+
69+
except Exception as e:
70+
logger.error(f"Error loading or running test {module_name}: {str(e)}")
71+
import traceback
72+
73+
logger.error(traceback.format_exc())
74+
results.append((module_name, False))
75+
76+
return results
77+
78+
79+
def print_summary(results: List[Tuple[str, bool]]) -> None:
80+
"""Print a summary of test results."""
81+
logger.info(f"\n{'=' * 50}")
82+
logger.info("TEST SUMMARY")
83+
logger.info(f"{'-' * 50}")
84+
85+
passed = sum(1 for _, success in results if success)
86+
total = len(results)
87+
88+
for module_name, success in results:
89+
status = "✅ PASSED" if success else "❌ FAILED"
90+
logger.info(f"{status} - {module_name}")
91+
92+
logger.info(f"{'-' * 50}")
93+
logger.info(f"Total: {total} | Passed: {passed} | Failed: {total - passed}")
94+
logger.info(f"{'=' * 50}")
95+
96+
97+
if __name__ == "__main__":
98+
# Check if required environment variables are set
99+
required_vars = [
100+
"DATABRICKS_SERVER_HOSTNAME",
101+
"DATABRICKS_HTTP_PATH",
102+
"DATABRICKS_TOKEN",
103+
]
104+
missing_vars = [var for var in required_vars if not os.environ.get(var)]
105+
106+
if missing_vars:
107+
logger.error(
108+
f"Missing required environment variables: {', '.join(missing_vars)}"
109+
)
110+
logger.error("Please set these variables before running the tests.")
111+
sys.exit(1)
112+
113+
# Run all tests
114+
results = run_tests()
115+
116+
# Print summary
117+
print_summary(results)
118+
119+
# Exit with appropriate status code
120+
all_passed = all(success for _, success in results)
121+
sys.exit(0 if all_passed else 1)

‎examples/experimental/tests/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)