Skip to content

Commit cc85898

Browse files
Add CI actions to improve dependency checks
Signed-off-by: Vikrant Puppala <vikrant.puppala@databricks.com>
1 parent 2f982bc commit cc85898

File tree

3 files changed

+225
-0
lines changed

3 files changed

+225
-0
lines changed

.github/workflows/code-quality-checks.yml

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,16 @@ jobs:
88
strategy:
99
matrix:
1010
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
11+
dependency-version: ["default", "minimum"]
12+
# Optimize matrix - test min/max on subset of Python versions
13+
exclude:
14+
- python-version: "3.12"
15+
dependency-version: "minimum"
16+
- python-version: "3.13"
17+
dependency-version: "minimum"
18+
19+
name: "Unit Tests (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)"
20+
1121
steps:
1222
#----------------------------------------------
1323
# check-out repo and set-up python
@@ -50,15 +60,47 @@ jobs:
5060
- name: Install library
5161
run: poetry install --no-interaction
5262
#----------------------------------------------
63+
# override with custom dependency versions
64+
#----------------------------------------------
65+
- name: Install Python tools for custom versions
66+
if: matrix.dependency-version != 'default'
67+
run: poetry run pip install toml packaging
68+
69+
- name: Generate requirements file
70+
if: matrix.dependency-version != 'default'
71+
run: |
72+
poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --output requirements-${{ matrix.dependency-version }}.txt
73+
echo "Generated requirements for ${{ matrix.dependency-version }} versions:"
74+
cat requirements-${{ matrix.dependency-version }}.txt
75+
76+
- name: Override with custom dependency versions
77+
if: matrix.dependency-version != 'default'
78+
run: poetry run pip install -r requirements-${{ matrix.dependency-version }}.txt
79+
80+
#----------------------------------------------
5381
# run test suite
5482
#----------------------------------------------
83+
- name: Show installed versions
84+
run: |
85+
echo "=== Dependency Version: ${{ matrix.dependency-version }} ==="
86+
poetry run pip list
87+
5588
- name: Run tests
5689
run: poetry run python -m pytest tests/unit
5790
run-unit-tests-with-arrow:
5891
runs-on: ubuntu-latest
5992
strategy:
6093
matrix:
6194
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
95+
dependency-version: ["default", "minimum"]
96+
exclude:
97+
- python-version: "3.12"
98+
dependency-version: "minimum"
99+
- python-version: "3.13"
100+
dependency-version: "minimum"
101+
102+
name: "Unit Tests + PyArrow (Python ${{ matrix.python-version }}, ${{ matrix.dependency-version }} deps)"
103+
62104
steps:
63105
#----------------------------------------------
64106
# check-out repo and set-up python
@@ -101,8 +143,30 @@ jobs:
101143
- name: Install library
102144
run: poetry install --no-interaction --all-extras
103145
#----------------------------------------------
146+
# override with custom dependency versions
147+
#----------------------------------------------
148+
- name: Install Python tools for custom versions
149+
if: matrix.dependency-version != 'default'
150+
run: poetry run pip install toml packaging
151+
152+
- name: Generate requirements file with pyarrow
153+
if: matrix.dependency-version != 'default'
154+
run: |
155+
poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --include-optional --output requirements-${{ matrix.dependency-version }}-arrow.txt
156+
echo "Generated requirements for ${{ matrix.dependency-version }} versions with PyArrow:"
157+
cat requirements-${{ matrix.dependency-version }}-arrow.txt
158+
159+
- name: Override with custom dependency versions
160+
if: matrix.dependency-version != 'default'
161+
run: poetry run pip install -r requirements-${{ matrix.dependency-version }}-arrow.txt
162+
#----------------------------------------------
104163
# run test suite
105164
#----------------------------------------------
165+
- name: Show installed versions
166+
run: |
167+
echo "=== Dependency Version: ${{ matrix.dependency-version }} with PyArrow ==="
168+
poetry run pip list
169+
106170
- name: Run tests
107171
run: poetry run python -m pytest tests/unit
108172
check-linting:

.github/workflows/integration.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ jobs:
1010
run-e2e-tests:
1111
runs-on: ubuntu-latest
1212
environment: azure-prod
13+
strategy:
14+
matrix:
15+
dependency-version: ["default", "minimum"]
16+
17+
name: "E2E Tests (${{ matrix.dependency-version }} deps)"
18+
1319
env:
1420
DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
1521
DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
@@ -52,7 +58,29 @@ jobs:
5258
- name: Install dependencies
5359
run: poetry install --no-interaction --all-extras
5460
#----------------------------------------------
61+
# override with custom dependency versions
62+
#----------------------------------------------
63+
- name: Install Python tools for custom versions
64+
if: matrix.dependency-version != 'default'
65+
run: poetry run pip install toml packaging
66+
67+
- name: Generate requirements file for e2e
68+
if: matrix.dependency-version != 'default'
69+
run: |
70+
poetry run python scripts/dependency_manager.py ${{ matrix.dependency-version }} --include-optional --output requirements-${{ matrix.dependency-version }}-e2e.txt
71+
echo "Generated requirements for ${{ matrix.dependency-version }} versions (E2E):"
72+
cat requirements-${{ matrix.dependency-version }}-e2e.txt
73+
74+
- name: Override with custom dependency versions
75+
if: matrix.dependency-version != 'default'
76+
run: poetry run pip install -r requirements-${{ matrix.dependency-version }}-e2e.txt
77+
#----------------------------------------------
5578
# run test suite
5679
#----------------------------------------------
80+
- name: Show installed versions
81+
run: |
82+
echo "=== E2E Dependency Version: ${{ matrix.dependency-version }} ==="
83+
poetry run pip list
84+
5785
- name: Run e2e tests
5886
run: poetry run python -m pytest tests/e2e

scripts/dependency_manager.py

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
"""
2+
Dependency version management for testing.
3+
Generates requirements files for min, max, and default dependency versions.
4+
"""
5+
6+
import toml
7+
import sys
8+
import argparse
9+
from packaging.specifiers import SpecifierSet
10+
from pathlib import Path
11+
12+
class DependencyManager:
13+
def __init__(self, pyproject_path="pyproject.toml"):
14+
self.pyproject_path = Path(pyproject_path)
15+
self.dependencies = self._load_dependencies()
16+
17+
def _load_dependencies(self):
18+
"""Load dependencies from pyproject.toml"""
19+
with open(self.pyproject_path, 'r') as f:
20+
pyproject = toml.load(f)
21+
return pyproject['tool']['poetry']['dependencies']
22+
23+
def _parse_constraint(self, name, constraint):
24+
"""Parse a dependency constraint into version info"""
25+
if isinstance(constraint, str):
26+
return constraint, False # version_constraint, is_optional
27+
elif isinstance(constraint, list):
28+
# Handle complex constraints like pandas
29+
return constraint[0]['version'], False
30+
elif isinstance(constraint, dict):
31+
if 'version' in constraint:
32+
return constraint['version'], constraint.get('optional', False)
33+
return None, False
34+
35+
def _extract_versions_from_specifier(self, spec_set_str):
36+
"""Extract minimum version from a specifier set"""
37+
try:
38+
# Handle caret (^) and tilde (~) constraints that packaging doesn't support
39+
if spec_set_str.startswith('^'):
40+
# ^1.2.3 means >=1.2.3, <2.0.0
41+
min_version = spec_set_str[1:] # Remove ^
42+
return min_version, None
43+
elif spec_set_str.startswith('~'):
44+
# ~1.2.3 means >=1.2.3, <1.3.0
45+
min_version = spec_set_str[1:] # Remove ~
46+
return min_version, None
47+
48+
spec_set = SpecifierSet(spec_set_str)
49+
min_version = None
50+
51+
for spec in spec_set:
52+
if spec.operator in ('>=', '=='):
53+
min_version = spec.version
54+
break
55+
56+
return min_version, None
57+
except Exception as e:
58+
print(f"Warning: Could not parse constraint '{spec_set_str}': {e}", file=sys.stderr)
59+
return None, None
60+
61+
def generate_requirements(self, version_type="min", include_optional=False):
62+
"""
63+
Generate requirements for specified version type.
64+
65+
Args:
66+
version_type: "min" or "default"
67+
include_optional: Whether to include optional dependencies
68+
"""
69+
requirements = []
70+
71+
for name, constraint in self.dependencies.items():
72+
if name == 'python':
73+
continue
74+
75+
version_constraint, is_optional = self._parse_constraint(name, constraint)
76+
if not version_constraint:
77+
continue
78+
79+
if is_optional and not include_optional:
80+
continue
81+
82+
if version_type == "default":
83+
# For default, just use the constraint as-is (let poetry resolve)
84+
requirements.append(f"{name}{version_constraint}")
85+
elif version_type == "min":
86+
min_version, _ = self._extract_versions_from_specifier(version_constraint)
87+
if min_version:
88+
requirements.append(f"{name}=={min_version}")
89+
90+
return requirements
91+
92+
93+
def write_requirements_file(self, filename, version_type="min", include_optional=False):
94+
"""Write requirements to a file"""
95+
requirements = self.generate_requirements(version_type, include_optional)
96+
97+
with open(filename, 'w') as f:
98+
f.write(f"# {version_type.title()} dependency versions generated from pyproject.toml\n")
99+
for req in sorted(requirements):
100+
f.write(f"{req}\n")
101+
102+
print(f"Generated {filename} with {len(requirements)} dependencies")
103+
return requirements
104+
105+
def main():
106+
parser = argparse.ArgumentParser(description="Manage dependency versions for testing")
107+
parser.add_argument("version_type", choices=["min", "default"],
108+
help="Type of versions to generate")
109+
parser.add_argument("--output", "-o", default=None,
110+
help="Output requirements file (default: requirements-{version_type}.txt)")
111+
parser.add_argument("--include-optional", action="store_true",
112+
help="Include optional dependencies")
113+
parser.add_argument("--pyproject", default="pyproject.toml",
114+
help="Path to pyproject.toml file")
115+
116+
args = parser.parse_args()
117+
118+
if args.output is None:
119+
args.output = f"requirements-{args.version_type}.txt"
120+
121+
manager = DependencyManager(args.pyproject)
122+
requirements = manager.write_requirements_file(
123+
args.output,
124+
args.version_type,
125+
args.include_optional
126+
)
127+
128+
# Also print to stdout for GitHub Actions
129+
for req in requirements:
130+
print(req)
131+
132+
if __name__ == "__main__":
133+
main()

0 commit comments

Comments
 (0)