PySQL Connector split into connector and sqlalchemy (#444)

jprakash-db · jackyhu-db · varun-edachali-dbx · commit 904c304e9ae3 · 2025-05-28T03:42:28.000Z
* Modified the gitignore file to not have .idea file * [PECO-1803] Splitting the PySql connector into the core and the non core part (#417) * Implemented ColumnQueue to test the fetchall without pyarrow Removed token removed token * order of fields in row corrected * Changed the folder structure and tested the basic setup to work * Refractored the code to make connector to work * Basic Setup of connector, core and sqlalchemy is working * Basic integration of core, connect and sqlalchemy is working * Setup working dynamic change from ColumnQueue to ArrowQueue * Refractored the test code and moved to respective folders * Added the unit test for column_queue Fixed __version__ Fix * venv_main added to git ignore * Added code for merging columnar table * Merging code for columnar * Fixed the retry_close sesssion test issue with logging * Fixed the databricks_sqlalchemy tests and introduced pytest.ini for the sqla_testing * Added pyarrow_test mark on pytest * Fixed databricks.sqlalchemy to databricks_sqlalchemy imports * Added poetry.lock * Added dist folder * Changed the pyproject.toml * Minor Fix * Added the pyarrow skip tag on unit tests and tested their working * Fixed the Decimal and timestamp conversion issue in non arrow pipeline * Removed not required files and reformatted * Fixed test_retry error * Changed the folder structure to src / databricks * Removed the columnar non arrow flow to another PR * Moved the README to the root * removed columnQueue instance * Revmoved databricks_sqlalchemy dependency in core * Changed the pysql_supports_arrow predicate, introduced changes in the pyproject.toml * Ran the black formatter with the original version * Extra .py removed from all the __init__.py files names * Undo formatting check * Check * Check * Check * Check * Check * Check * Check * Check * Check * Check * Check * Check * Check * Check * BIG UPDATE * Refeactor code * Refractor * Fixed versioning * Minor refractoring * Minor refractoring * Changed the folder structure such that sqlalchemy has not reference here * Fixed README.md and CONTRIBUTING.md * Added manual publish * On push trigger added * Manually setting the publish step * Changed versioning in pyproject.toml * Bumped up the version to 4.0.0.b3 and also changed the structure to have pyarrow as optional * Removed the sqlalchemy tests from integration.yml file * [PECO-1803] Print warning message if pyarrow is not installed (#468) Print warning message if pyarrow is not installed Signed-off-by: Jacky Hu <jacky.hu@databricks.com> * [PECO-1803] Remove sqlalchemy and update README.md (#469) Remove sqlalchemy and update README.md Signed-off-by: Jacky Hu <jacky.hu@databricks.com> * Removed all sqlalchemy related stuff * generated the lock file * Fixed failing tests * removed poetry.lock * Updated the lock file * Fixed poetry numpy 2.2.2 issue * Workflow fixes --------- Signed-off-by: Jacky Hu <jacky.hu@databricks.com> Co-authored-by: Jacky Hu <jacky.hu@databricks.com> Signed-off-by: varun-edachali-dbx <varun.edachali@databricks.com>
diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -93,7 +93,7 @@ jobs:
       #----------------------------------------------
       -   name: Load cached venv
           id: cached-poetry-dependencies
-          uses: actions/cache@v4
+          uses: actions/cache@v2
           with:
             path: .venv-pyarrow
             key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,41 +1,10 @@
 # Release History
 
-# 4.0.3 (2025-04-22)
-
-- Fix: Removed `packaging` dependency in favour of default libraries, for `urllib3` version checks (databricks/databricks-sql-python#547 by @jprakash-db)
-- 
-# 4.0.2 (2025-04-01)
-
-- Fix: Relaxed the pin for `python-dateutil` to be `^2.8.0` (databricks/databricks-sql-python#538 by @jprakash-db)
-
-# 4.0.1 (2025-03-19)
-
-**Note: this release was yanked from Pypi on 19 March 2025 due to compatibility issues with `dbt-databricks<1.5.3`**
-
-- Support for multiple timestamp formats parsing (databricks/databricks-sql-python#533 by @jprakash-db)
-- Rename `_user_agent_entry` in connect call to `user_agent_entry` to expose it as a public parameter. (databricks/databricks-sql-python#530 by @shivam2680)
-- Fix: compatibility with urllib3 versions less than 2.x. (databricks/databricks-sql-python#526 by @shivam2680)
-- Support for Python 3.13 and updated dependencies (databricks/databricks-sql-python#510 by @dhirschfeld and @dbaxa)
-
-# 4.0.0 (2025-01-19)
+# 4.0.0 (TBD)
 
 - Split the connector into two separate packages: `databricks-sql-connector` and `databricks-sqlalchemy`. The `databricks-sql-connector` package contains the core functionality of the connector, while the `databricks-sqlalchemy` package contains the SQLAlchemy dialect for the connector. 
 - Pyarrow dependency is now optional in `databricks-sql-connector`. Users needing arrow are supposed to explicitly install pyarrow
 
-# 3.7.3 (2025-03-28)
-
-- Fix: Unable to poll small results in execute_async function (databricks/databricks-sql-python#515 by @jprakash-db) 
-- Updated log messages to show the status code and error messages of requests (databricks/databricks-sql-python#511 by @jprakash-db) 
-- Fix: Incorrect metadata was fetched in case of queries with the same alias (databricks/databricks-sql-python#505 by @jprakash-db)
-
-# 3.7.2 (2025-01-31)
-
-- Updated the retry_dela_max and retry_timeout (databricks/databricks-sql-python#497 by @jprakash-db)
-
-# 3.7.1 (2025-01-07)
-
-- Relaxed the number of Http retry attempts (databricks/databricks-sql-python#486 by @jprakash-db)
-
 # 3.7.0 (2024-12-23)
 
 - Fix: Incorrect number of rows fetched in inline results when fetching results with FETCH_NEXT orientation (databricks/databricks-sql-python#479 by @jprakash-db)
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "4.0.3"
+version = "4.0.0"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
 license = "Apache-2.0"
@@ -18,13 +18,13 @@ pandas = [
 lz4 = "^4.0.2"
 requests = "^2.18.1"
 oauthlib = "^3.1.0"
+numpy = [
+    { version = "^1.16.6", python = ">=3.8,<3.11" },
+    { version = "^1.23.4", python = ">=3.11" },
+]
 openpyxl = "^3.0.10"
 urllib3 = ">=1.26"
-pyarrow = [
-    { version = ">=14.0.1", python = ">=3.8,<3.13", optional=true },
-    { version = ">=18.0.0", python = ">=3.13", optional=true }
-]
-python-dateutil = "^2.8.0"
+pyarrow = { version = ">=14.0.1", optional=true }
 
 [tool.poetry.extras]
 pyarrow = ["pyarrow"]
diff --git a/tests/unit/test_arrow_queue.py b/tests/unit/test_arrow_queue.py
@@ -1,13 +1,11 @@
 import unittest
 import pytest
-
 try:
     import pyarrow as pa
 except ImportError:
     pa = None
 from databricks.sql.utils import ArrowQueue
 
-
 @pytest.mark.skipif(pa is None, reason="PyArrow is not installed")
 class ArrowQueueSuite(unittest.TestCase):
     @staticmethod
diff --git a/tests/unit/test_cloud_fetch_queue.py b/tests/unit/test_cloud_fetch_queue.py
@@ -10,7 +10,6 @@
 import databricks.sql.utils as utils
 from databricks.sql.types import SSLOptions
 
-
 @pytest.mark.skipif(pyarrow is None, reason="PyArrow is not installed")
 class CloudFetchQueueSuite(unittest.TestCase):
     def create_result_link(
diff --git a/tests/unit/test_fetches.py b/tests/unit/test_fetches.py
@@ -5,14 +5,13 @@
 try:
     import pyarrow as pa
 except ImportError:
-    pa = None
+    pa=None
 
 import databricks.sql.client as client
 from databricks.sql.utils import ExecuteResponse, ArrowQueue
 from databricks.sql.backend.thrift_backend import ThriftDatabricksClient
 from databricks.sql.result_set import ThriftResultSet
 
-
 @pytest.mark.skipif(pa is None, reason="PyArrow is not installed")
 class FetchTests(unittest.TestCase):
     """
diff --git a/tests/unit/test_fetches_bench.py b/tests/unit/test_fetches_bench.py
@@ -1,18 +1,16 @@
 import unittest
 from unittest.mock import Mock
-
 try:
     import pyarrow as pa
 except ImportError:
-    pa = None
+    pa=None
 import uuid
 import time
 import pytest
 
 import databricks.sql.client as client
 from databricks.sql.utils import ExecuteResponse, ArrowQueue
 
-
 @pytest.mark.skipif(pa is None, reason="PyArrow is not installed")
 class FetchBenchmarkTests(unittest.TestCase):
     """
diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py
@@ -6,11 +6,10 @@
 from unittest.mock import patch, MagicMock, Mock
 from ssl import CERT_NONE, CERT_REQUIRED
 from urllib3 import HTTPSConnectionPool
-
 try:
     import pyarrow
 except ImportError:
-    pyarrow = None
+    pyarrow=None
 import databricks.sql
 from databricks.sql import utils
 from databricks.sql.types import SSLOptions
@@ -30,8 +29,7 @@ def retry_policy_factory():
         "_retry_delay_default": (float, 5, 1, 60),
     }
 
-
-@pytest.mark.skipif(pyarrow is None, reason="PyArrow is not installed")
+@pytest.mark.skipif(pyarrow is None,reason="PyArrow is not installed")
 class ThriftBackendTestSuite(unittest.TestCase):
     okay_status = ttypes.TStatus(statusCode=ttypes.TStatusCode.SUCCESS_STATUS)
 
