Skip to content

Commit 177c197

Browse files
shivam2680shivam2680
committed
resolve merge conflicts
2 parents 5354788 + 6e0f5ff commit 177c197

File tree

86 files changed

+15131
-1981
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

86 files changed

+15131
-1981
lines changed

‎.github/CODEOWNERS‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎.github/workflows/code-quality-checks.yml‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,16 @@ jobs:
88
strategy:
99
matrix:
1010
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
11+
dependency-version: ["default", "min"]
12+
# Optimize matrix - test min/max on subset of Python versions
13+
exclude:
14+
- python-version: "3.12"
15+
dependency-version: "min"
16+
- python-version: "3.13"
17+
dependency-version: "min"
18+
19+
name: "Unit Tests (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)"
20+
1121
steps:
1222
#----------------------------------------------
1323
# check-out repo and set-up python
@@ -37,7 +47,7 @@ jobs:
3747
uses: actions/cache@v4
3848
with:
3949
path: .venv
40-
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
50+
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
4151
#----------------------------------------------
4252
# install dependencies if cache does not exist
4353
#----------------------------------------------
@@ -50,15 +60,47 @@ jobs:
5060
- name: Install library
5161
run: poetry install --no-interaction
5262
#----------------------------------------------
63+
# override with custom dependency versions
64+
#----------------------------------------------
65+
- name: Install Python tools for custom versions
66+
if: matrix.dependency-version != 'default'
67+
run: poetry run pip install toml packaging
68+
69+
- name: Generate requirements file
70+
if: matrix.dependency-version != 'default'
71+
run: |
72+
poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}.txt
73+
echo "Generated requirements for ${{ matrix.dependency-version }} versions:"
74+
cat requirements-${{ matrix.dependency-version }}.txt
75+
76+
- name: Override with custom dependency versions
77+
if: matrix.dependency-version != 'default'
78+
run: poetry run pip install -r requirements-${{ matrix.dependency-version }}.txt
79+
80+
#----------------------------------------------
5381
# run test suite
5482
#----------------------------------------------
83+
- name: Show installed versions
84+
run: |
85+
echo "=== Dependency Version: ${{ matrix.dependency-version }} ==="
86+
poetry run pip list
87+
5588
- name: Run tests
5689
run: poetry run python -m pytest tests/unit
5790
run-unit-tests-with-arrow:
5891
runs-on: ubuntu-latest
5992
strategy:
6093
matrix:
6194
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
95+
dependency-version: ["default", "min"]
96+
exclude:
97+
- python-version: "3.12"
98+
dependency-version: "min"
99+
- python-version: "3.13"
100+
dependency-version: "min"
101+
102+
name: "Unit Tests + PyArrow (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)"
103+
62104
steps:
63105
#----------------------------------------------
64106
# check-out repo and set-up python
@@ -88,7 +130,7 @@ jobs:
88130
uses: actions/cache@v4
89131
with:
90132
path: .venv-pyarrow
91-
key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
133+
key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ matrix.dependency-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
92134
#----------------------------------------------
93135
# install dependencies if cache does not exist
94136
#----------------------------------------------
@@ -101,8 +143,30 @@ jobs:
101143
- name: Install library
102144
run: poetry install --no-interaction --all-extras
103145
#----------------------------------------------
146+
# override with custom dependency versions
147+
#----------------------------------------------
148+
- name: Install Python tools for custom versions
149+
if: matrix.dependency-version != 'default'
150+
run: poetry run pip install toml packaging
151+
152+
- name: Generate requirements file with pyarrow
153+
if: matrix.dependency-version != 'default'
154+
run: |
155+
poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}-arrow.txt
156+
echo "Generated requirements for ${{ matrix.dependency-version }} versions with PyArrow:"
157+
cat requirements-${{ matrix.dependency-version }}-arrow.txt
158+
159+
- name: Override with custom dependency versions
160+
if: matrix.dependency-version != 'default'
161+
run: poetry run pip install -r requirements-${{ matrix.dependency-version }}-arrow.txt
162+
#----------------------------------------------
104163
# run test suite
105164
#----------------------------------------------
165+
- name: Show installed versions
166+
run: |
167+
echo "=== Dependency Version: ${{ matrix.dependency-version }} with PyArrow ==="
168+
poetry run pip list
169+
106170
- name: Run tests
107171
run: poetry run python -m pytest tests/unit
108172
check-linting:

‎.github/workflows/coverage-check.yml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: Code Coverage
2+
3+
permissions:
4+
contents: read
5+
6+
on: [pull_request, workflow_dispatch]
7+
8+
jobs:
9+
coverage:
10+
runs-on: ubuntu-latest
11+
environment: azure-prod
12+
env:
13+
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
14+
DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
15+
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
16+
DATABRICKS_CATALOG: peco
17+
DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
18+
steps:
19+
#----------------------------------------------
20+
# check-out repo and set-up python
21+
#----------------------------------------------
22+
- name: Check out repository
23+
uses: actions/checkout@v4
24+
with:
25+
fetch-depth: 0 # Needed for coverage comparison
26+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
27+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
28+
- name: Set up python
29+
id: setup-python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.10"
33+
#----------------------------------------------
34+
# ----- install & configure poetry -----
35+
#----------------------------------------------
36+
- name: Install Poetry
37+
uses: snok/install-poetry@v1
38+
with:
39+
virtualenvs-create: true
40+
virtualenvs-in-project: true
41+
installer-parallel: true
42+
43+
#----------------------------------------------
44+
# load cached venv if cache exists
45+
#----------------------------------------------
46+
- name: Load cached venv
47+
id: cached-poetry-dependencies
48+
uses: actions/cache@v4
49+
with:
50+
path: .venv
51+
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
52+
#----------------------------------------------
53+
# install dependencies if cache does not exist
54+
#----------------------------------------------
55+
- name: Install dependencies
56+
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
57+
run: poetry install --no-interaction --no-root
58+
#----------------------------------------------
59+
# install your root project, if required
60+
#----------------------------------------------
61+
- name: Install library
62+
run: poetry install --no-interaction --all-extras
63+
#----------------------------------------------
64+
# run all tests
65+
#----------------------------------------------
66+
- name: Run tests with coverage
67+
continue-on-error: true
68+
run: |
69+
poetry run python -m pytest \
70+
tests/unit tests/e2e \
71+
--cov=src --cov-report=xml --cov-report=term -v
72+
#----------------------------------------------
73+
# check for coverage override
74+
#----------------------------------------------
75+
- name: Check for coverage override
76+
id: override
77+
run: |
78+
OVERRIDE_COMMENT=$(echo "${{ github.event.pull_request.body }}" | grep -E "SKIP_COVERAGE_CHECK\s*=" || echo "")
79+
if [ -n "$OVERRIDE_COMMENT" ]; then
80+
echo "override=true" >> $GITHUB_OUTPUT
81+
REASON=$(echo "$OVERRIDE_COMMENT" | sed -E 's/.*SKIP_COVERAGE_CHECK\s*=\s*(.+)/\1/')
82+
echo "reason=$REASON" >> $GITHUB_OUTPUT
83+
echo "Coverage override found in PR description: $REASON"
84+
else
85+
echo "override=false" >> $GITHUB_OUTPUT
86+
echo "No coverage override found"
87+
fi
88+
#----------------------------------------------
89+
# check coverage percentage
90+
#----------------------------------------------
91+
- name: Check coverage percentage
92+
if: steps.override.outputs.override == 'false'
93+
run: |
94+
COVERAGE_FILE="coverage.xml"
95+
if [ ! -f "$COVERAGE_FILE" ]; then
96+
echo "ERROR: Coverage file not found at $COVERAGE_FILE"
97+
exit 1
98+
fi
99+
100+
# Install xmllint if not available
101+
if ! command -v xmllint &> /dev/null; then
102+
sudo apt-get update && sudo apt-get install -y libxml2-utils
103+
fi
104+
105+
COVERED=$(xmllint --xpath "string(//coverage/@lines-covered)" "$COVERAGE_FILE")
106+
TOTAL=$(xmllint --xpath "string(//coverage/@lines-valid)" "$COVERAGE_FILE")
107+
PERCENTAGE=$(python3 -c "covered=${COVERED}; total=${TOTAL}; print(round((covered/total)*100, 2))")
108+
109+
echo "Branch Coverage: $PERCENTAGE%"
110+
echo "Required Coverage: 85%"
111+
112+
# Use Python to compare the coverage with 85
113+
python3 -c "import sys; sys.exit(0 if float('$PERCENTAGE') >= 85 else 1)"
114+
if [ $? -eq 1 ]; then
115+
echo "ERROR: Coverage is $PERCENTAGE%, which is less than the required 85%"
116+
exit 1
117+
else
118+
echo "SUCCESS: Coverage is $PERCENTAGE%, which meets the required 85%"
119+
fi
120+
121+
#----------------------------------------------
122+
# coverage enforcement summary
123+
#----------------------------------------------
124+
- name: Coverage enforcement summary
125+
run: |
126+
if [ "${{ steps.override.outputs.override }}" == "true" ]; then
127+
echo "⚠️ Coverage checks bypassed: ${{ steps.override.outputs.reason }}"
128+
echo "Please ensure this override is justified and temporary"
129+
else
130+
echo "✅ Coverage checks enforced - minimum 85% required"
131+
fi

‎CHANGELOG.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,32 @@
11
# Release History
22

3+
# 4.1.0 (2025-08-18)
4+
- Removed Codeowners (databricks/databricks-sql-python#623 by @jprakash-db)
5+
- Azure Service Principal Credential Provider (databricks/databricks-sql-python#621 by @jprakash-db)
6+
- Add optional telemetry support to the python connector (databricks/databricks-sql-python#628 by @saishreeeee)
7+
- Fix potential resource leak in `CloudFetchQueue` (databricks/databricks-sql-python#624 by @varun-edachali-dbx)
8+
- Generalise Backend Layer (databricks/databricks-sql-python#604 by @varun-edachali-dbx)
9+
- Arrow performance optimizations (databricks/databricks-sql-python#638 by @jprakash-db)
10+
- Connection errors to unauthenticated telemetry endpoint (databricks/databricks-sql-python#619 by @saishreeeee)
11+
- SEA: Execution Phase (databricks/databricks-sql-python#645 by @varun-edachali-dbx)
12+
- Add retry mechanism to telemetry requests (databricks/databricks-sql-python#617 by @saishreeeee)
13+
- SEA: Fetch Phase (databricks/databricks-sql-python#650 by @varun-edachali-dbx)
14+
- added logs for cloud fetch speed (databricks/databricks-sql-python#654 by @shivam2680)
15+
- Make telemetry batch size configurable and add time-based flush (databricks/databricks-sql-python#622 by @saishreeeee)
16+
- Normalise type code (databricks/databricks-sql-python#652 by @varun-edachali-dbx)
17+
- Testing for telemetry (databricks/databricks-sql-python#616 by @saishreeeee)
18+
- Bug fixes in telemetry (databricks/databricks-sql-python#659 by @saishreeeee)
19+
- Telemetry server-side flag integration (databricks/databricks-sql-python#646 by @saishreeeee)
20+
- Enhance SEA HTTP Client (databricks/databricks-sql-python#618 by @varun-edachali-dbx)
21+
- SEA: Allow large metadata responses (databricks/databricks-sql-python#653 by @varun-edachali-dbx)
22+
- Added code coverage workflow to test the code coverage from unit and e2e tests (databricks/databricks-sql-python#657 by @msrathore-db)
23+
- Concat tables to be backward compatible (databricks/databricks-sql-python#647 by @jprakash-db)
24+
- Refactor codebase to use a unified http client (databricks/databricks-sql-python#673 by @vikrantpuppala)
25+
- Add kerberos support for proxy auth (databricks/databricks-sql-python#675 by @vikrantpuppala)
26+
27+
# 4.0.5 (2025-06-24)
28+
- Fix: Reverted change in cursor close handling which led to errors impacting users (databricks/databricks-sql-python#613 by @madhav-db)
29+
330
# 4.0.4 (2025-06-16)
431

532
- Update thrift client library after cleaning up unused fields and structs (databricks/databricks-sql-python#553 by @vikrantpuppala)

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ The Databricks SQL Connector for Python allows you to develop Python application
77

88
This connector uses Arrow as the data-exchange format, and supports APIs (e.g. `fetchmany_arrow`) to directly fetch Arrow tables. Arrow tables are wrapped in the `ArrowQueue` class to provide a natural API to get several rows at a time. [PyArrow](https://arrow.apache.org/docs/python/index.html) is required to enable this and use these APIs, you can install it via `pip install pyarrow` or `pip install databricks-sql-connector[pyarrow]`.
99

10+
The connector includes built-in support for HTTP/HTTPS proxy servers with multiple authentication methods including basic authentication and Kerberos/Negotiate authentication. See `docs/proxy.md` and `examples/proxy_authentication.py` for details.
11+
1012
You are welcome to file an issue here for general use cases. You can also contact Databricks Support [here](help.databricks.com).
1113

1214
## Requirements

0 commit comments

Comments
 (0)