Workflow fixes

Author: jprakash-db

.github/workflows/code-quality-checks.yml

@@ -52,12 +52,63 @@ jobs:
       # install your root project, if required
       #----------------------------------------------
       - name: Install library
-        run: poetry install --no-interaction --all-extras
+        run: poetry install --no-interaction
       #----------------------------------------------
       #              run test suite
       #----------------------------------------------
       - name: Run tests
         run: poetry run python -m pytest tests/unit
+  run-unit-tests-with-arrow:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ 3.8, 3.9, "3.10", "3.11" ]
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      -   name: Check out repository
+          uses: actions/checkout@v2
+      -   name: Set up python ${{ matrix.python-version }}
+          id: setup-python
+          uses: actions/setup-python@v2
+          with:
+            python-version: ${{ matrix.python-version }}
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      -   name: Install Poetry
+          uses: snok/install-poetry@v1
+          with:
+            virtualenvs-create: true
+            virtualenvs-in-project: true
+            installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      -   name: Load cached venv
+          id: cached-poetry-dependencies
+          uses: actions/cache@v2
+          with:
+            path: .venv-pyarrow
+            key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      -   name: Install dependencies
+          if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+          run: poetry install --no-interaction --no-root
+      #----------------------------------------------
+      # install your root project, if required
+      #----------------------------------------------
+      -   name: Install library
+          run: poetry install --no-interaction --all-extras
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+      -   name: Run tests
+          run: poetry run python -m pytest tests/unit
   check-linting:
     runs-on: ubuntu-latest
     strategy:

tests/unit/test_arrow_queue.py

@@ -1,10 +1,12 @@
 import unittest
-
-import pyarrow as pa
-
+import pytest
+try:
+    import pyarrow as pa
+except ImportError:
+    pa = None
 from databricks.sql.utils import ArrowQueue
 
-
+@pytest.mark.skipif(pa is None, reason="PyArrow is not installed")
 class ArrowQueueSuite(unittest.TestCase):
     @staticmethod
     def make_arrow_table(batch):

tests/unit/test_cloud_fetch_queue.py

@@ -1,12 +1,16 @@
-import pyarrow
+try:
+    import pyarrow
+except ImportError:
+    pyarrow = None
 import unittest
+import pytest
 from unittest.mock import MagicMock, patch
 
 from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink
 import databricks.sql.utils as utils
 from databricks.sql.types import SSLOptions
 
-
+@pytest.mark.skipif(pyarrow is None, reason="PyArrow is not installed")
 class CloudFetchQueueSuite(unittest.TestCase):
     def create_result_link(
         self,

tests/unit/test_fetches.py

@@ -1,12 +1,16 @@
 import unittest
+import pytest
 from unittest.mock import Mock
 
-import pyarrow as pa
+try:
+    import pyarrow as pa
+except ImportError:
+    pa=None
 
 import databricks.sql.client as client
 from databricks.sql.utils import ExecuteResponse, ArrowQueue
 
-
+@pytest.mark.skipif(pa is None, reason="PyArrow is not installed")
 class FetchTests(unittest.TestCase):
     """
     Unit tests for checking the fetch logic.

tests/unit/test_fetches_bench.py

@@ -1,15 +1,17 @@
 import unittest
 from unittest.mock import Mock
-
-import pyarrow as pa
+try:
+    import pyarrow as pa
+except ImportError:
+    pa=None
 import uuid
 import time
 import pytest
 
 import databricks.sql.client as client
 from databricks.sql.utils import ExecuteResponse, ArrowQueue
 
-
+@pytest.mark.skipif(pa is None, reason="PyArrow is not installed")
 class FetchBenchmarkTests(unittest.TestCase):
     """
     Micro benchmark test for Arrow result handling.

tests/unit/test_thrift_backend.py

@@ -2,12 +2,14 @@
 from decimal import Decimal
 import itertools
 import unittest
+import pytest
 from unittest.mock import patch, MagicMock, Mock
 from ssl import CERT_NONE, CERT_REQUIRED
 from urllib3 import HTTPSConnectionPool
-
-import pyarrow
-
+try:
+    import pyarrow
+except ImportError:
+    pyarrow=None
 import databricks.sql
 from databricks.sql import utils
 from databricks.sql.types import SSLOptions
@@ -26,7 +28,7 @@ def retry_policy_factory():
         "_retry_delay_default": (float, 5, 1, 60),
     }
 
-
+@pytest.mark.skipif(pyarrow is None,reason="PyArrow is not installed")
 class ThriftBackendTestSuite(unittest.TestCase):
     okay_status = ttypes.TStatus(statusCode=ttypes.TStatusCode.SUCCESS_STATUS)