Resolve compatibility issue for kubernetes cases under CGroup2 (#3303)

Author: wjsi

.github/workflows/benchmark-ci.yml

@@ -35,7 +35,7 @@ jobs:
         shell: bash
         run: |
           source ./ci/install-conda.sh
-          python -m pip install --upgrade pip "setuptools<64" wheel coverage;
+          python -m pip install --upgrade pip setuptools wheel coverage;
 
       - name: Install dependencies
         id: build

.github/workflows/core-ci.yml

@@ -35,7 +35,7 @@ jobs:
       shell: bash
       run: |
         source ./ci/install-conda.sh
-        python -m pip install --upgrade pip "setuptools<64" wheel coverage;
+        python -m pip install --upgrade pip setuptools wheel coverage;
 
     - name: Install dependencies
       env:

.github/workflows/os-compat-ci.yml

@@ -32,7 +32,7 @@ jobs:
       shell: bash
       run: |
         source ./ci/install-conda.sh
-        python -m pip install --upgrade pip "setuptools<64" wheel coverage;
+        python -m pip install --upgrade pip setuptools wheel coverage;
 
     - name: Install dependencies
       env:

.github/workflows/platform-ci.yml

@@ -18,9 +18,9 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8-ray, 3.8-ray-deploy, 3.8-ray-dag, 3.8-vineyard, 3.8-dask]
+        python-version: [3.8-kubernetes, 3.8-ray, 3.8-ray-deploy, 3.8-ray-dag, 3.8-vineyard, 3.8-dask]
         include:
-          - { os: ubuntu-20.04, python-version: 3.8-kubernetes, no-common-tests: 1,
+          - { os: ubuntu-latest, python-version: 3.8-kubernetes, no-common-tests: 1,
               no-deploy: 1, with-kubernetes: "with Kubernetes" }
           - { os: ubuntu-20.04, python-version: 3.8-hadoop, no-common-tests: 1,
               no-deploy: 1, with-hadoop: "with hadoop" }
@@ -47,13 +47,14 @@ jobs:
         shell: bash
         run: |
           source ./ci/install-conda.sh
-          python -m pip install --upgrade pip "setuptools<64" wheel coverage;
+          python -m pip install --upgrade pip setuptools wheel coverage
 
       - name: Start minikube
         if: ${{ matrix.with-kubernetes }}
-        with:
-          driver: none
         uses: medyagh/setup-minikube@master
+        with:
+          driver: docker
+          mount-path: '/home/runner:/home/runner'
 
       - name: Install dependencies
         env:
@@ -125,6 +126,7 @@ jobs:
           RUN_DASK: ${{ matrix.run-dask }}
           NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
           NUMPY_EXPERIMENTAL_ARRAY_FUNCTION: 1
+          USE_MINIKUBE_DOCKER_ENV: true
           CHANGE_MINIKUBE_NONE_USER: true
         shell: bash
         run: |

azure-pipelines.yml

@@ -94,7 +94,7 @@ jobs:
 
         # do compatibility test for earliest supported pandas release
         if [[ "$(mars.test.module)" == "dataframe" ]]; then
-          pip install numpy\<1.24.0
+          pip install numpy\<1.24.0 sqlalchemy\<2.0
           pip install -i https://pkgs.dev.azure.com/mars-project/mars/_packaging/pandas/pypi/simple/ pandas==1.0.5
           pytest $PYTEST_CONFIG -m pd_compat mars/dataframe
           mv .coverage build/.coverage.pd_compat.file

benchmarks/tpch/gen_data.py

@@ -102,7 +102,6 @@ def to_parquet(args):
 def generate(
     tables, SCALE_FACTOR, folder, upload_to_s3, validate_dataset, num_processes
 ):
-
     if upload_to_s3:
         assert "AWS_ACCESS_KEY_ID" in os.environ, "AWS credentials not set"
     else:
@@ -117,7 +116,6 @@ def generate(
             fs = s3fs.S3FileSystem()
 
     for table_name, (table_short, num_pieces, load_func) in tables.items():
-
         if upload_to_s3:
             output_prefix = f"s3://{folder}/{table_name}.pq"
         else:

mars/core/entity/executable.py

@@ -56,7 +56,9 @@ def _thread_body(self):
                 fut.set_result(None)
             except (RuntimeError, ConnectionError, KeyError, ActorNotExist):
                 fut.set_result(None)
-            except Exception as ex:  # pragma: no cover  # noqa: E722  # nosec  # pylint: disable=bare-except
+            except (
+                Exception
+            ) as ex:  # pragma: no cover  # noqa: E722  # nosec  # pylint: disable=bare-except
                 fut.set_exception(ex)
             finally:
                 del session

mars/dataframe/base/shift.py

@@ -299,9 +299,7 @@ def _tile_series(cls, op):
                 to_concats = [c]
                 left = abs(op.periods)
                 while left > 0 and 0 <= prev_i < inp.chunk_shape[0]:
-                    prev_chunk = inp.cix[
-                        prev_i,
-                    ]
+                    prev_chunk = inp.cix[prev_i,]
                     size = min(left, prev_chunk.shape[0])
                     left -= size
                     prev_i = prev_i - 1 if inc else prev_i + 1

mars/dataframe/base/to_numeric.py

@@ -23,7 +23,6 @@
 
 
 class DataFrameToNumeric(DataFrameOperand, DataFrameOperandMixin):
-
     errors = StringField("errors")
     downcast = StringField("downcast")
 

mars/dataframe/datasource/from_tensor.py

@@ -310,9 +310,7 @@ def _tile_input_1d_tileables(cls, op: "DataFrameFromTensor"):
                 index_value = parse_index(pd_index, store_data=True)
             else:
                 assert op.index is not None
-                index_chunk = in_tensors[-1].cix[
-                    i,
-                ]
+                index_chunk = in_tensors[-1].cix[i,]
                 index_value = parse_index(
                     pd.Index([], dtype=index_chunk.dtype),
                     index_chunk,