#327/#328: Updated ITDE to 5.0.0 and updated Github secrets for Docker credentials (#329)

fixes #327
fixes #328

Author: tomuben

.github/workflows/integration-tests.yml

@@ -3,9 +3,9 @@ name: Integration Tests
 on:
   workflow_call:
     secrets:
-      DOCKER_USERNAME:
+      CI4_DOCKERHUB_USERNAME:
           required: true
-      DOCKER_PASSWORD:
+      CI4_DOCKERHUB_TOKEN:
           required: true
 jobs:
 
@@ -53,5 +53,5 @@ jobs:
       - name: Run test ${{ matrix.test-path.name }}
         run: poetry run -- python ${{ matrix.test-path.path }}
         env: # Set the secret as an input
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWD: ${{ secrets.DOCKER_PASSWORD }}
+          DOCKER_USER: ${{ secrets.CI4_DOCKERHUB_USERNAME }}
+          DOCKER_PASSWD: ${{ secrets.CI4_DOCKERHUB_TOKEN }}

.github/workflows/slow-checks.yml

@@ -3,9 +3,9 @@ name: Slow-Checks
 on:
   workflow_call:
     secrets:
-      DOCKER_USERNAME:
+      CI4_DOCKERHUB_USERNAME:
           required: true
-      DOCKER_PASSWORD:
+      CI4_DOCKERHUB_TOKEN:
           required: true
 jobs:
   integration-tests:

doc/changes/unreleased.md

@@ -7,4 +7,9 @@
 ## Refactorings
 
  - #313: Updated GPU Test Query and updated Poetry dependencies and PTB GH action
- - #320: Updated poetry dependencies and Github workflows by PTB 1.12.0. 
+ - #320: Updated poetry dependencies and Github workflows by PTB 1.12.0.
+ - #328: Updated Github secrets for Docker credentials
+
+## Dependencies
+
+ - #327: Updated ITDE to 5.0.0

exasol/slc/api/run_db_tests.py

@@ -26,6 +26,7 @@
 from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import (  # pylint: disable=line-too-long
     DbOsAccess,
 )
+from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports
 from exasol_integration_test_docker_environment.lib.utils.api_function_decorators import (
     cli_function,
 )
@@ -60,8 +61,8 @@ def run_db_test(
     additional_db_parameter: tuple[str, ...] = tuple(),
     docker_environment_variable: tuple[str, ...] = tuple(),
     external_exasol_db_host: Optional[str] = None,
-    external_exasol_db_port: int = 8563,
-    external_exasol_bucketfs_port: int = 2580,
+    external_exasol_db_port: int = Ports.external.database,
+    external_exasol_bucketfs_port: int = Ports.external.bucketfs_http,
     external_exasol_ssh_port: Optional[int] = None,
     external_exasol_db_user: Optional[str] = None,
     external_exasol_db_password: Optional[str] = None,
@@ -104,6 +105,7 @@ def run_db_test(
     use_job_specific_log_file: bool = True,
     compression_strategy: CompressionStrategy = defaultCompressionStrategy(),
     accelerator: Accelerator = defaultAccelerator(),
+    external_exasol_bucketfs_https_port: int = Ports.external.bucketfs_https,
 ) -> AllTestsResult:
     """
     This command runs the integration tests in local docker-db.
@@ -157,10 +159,12 @@ def run_db_test(
             raise api_errors.MissingArgumentError("external_exasol_ssh_port")
 
     docker_runtime = None
+    itde_accelerator: tuple[str, ...] = ()
     if accelerator == Accelerator.NVIDA:
-        additional_db_parameter += ("-enableAcceleratorDeviceDetection=1",)
-        docker_runtime = "nvidia"
-        docker_environment_variable += ("NVIDIA_VISIBLE_DEVICES=all",)
+        itde_accelerator = ("nvidia=all",)
+        add_db_param_accel_detection = "-enableAcceleratorDeviceDetection=1"
+        if add_db_param_accel_detection not in additional_db_parameter:
+            additional_db_parameter += (add_db_param_accel_detection,)
 
     def root_task_generator() -> DependencyLoggerBaseTask:
         return generate_root_task(
@@ -191,7 +195,8 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             max_start_attempts=max_start_attempts,
             external_exasol_db_host=external_exasol_db_host,
             external_exasol_db_port=external_exasol_db_port,
-            external_exasol_bucketfs_port=external_exasol_bucketfs_port,
+            external_exasol_bucketfs_http_port=external_exasol_bucketfs_port,
+            external_exasol_bucketfs_https_port=external_exasol_bucketfs_https_port,
             external_exasol_db_user=external_exasol_db_user,
             external_exasol_db_password=external_exasol_db_password,
             external_exasol_ssh_port=external_exasol_ssh_port,
@@ -207,6 +212,7 @@ def root_task_generator() -> DependencyLoggerBaseTask:
             compression_strategy=compression_strategy,
             docker_runtime=docker_runtime,
             docker_environment_variables=docker_environment_variable,
+            accelerator=itde_accelerator,
         )
 
     return run_task(

exasol/slc/internal/tasks/test/test_runner_db_test_base_task.py

@@ -57,8 +57,6 @@
 from exasol.slc.internal.tasks.upload.language_definition import LanguageDefinition
 from exasol.slc.models.flavor_ci_model import (
     FlavorCiConfig,
-    TestConfig,
-    TestSet,
 )
 from exasol.slc.models.run_db_test_result import RunDBTestsInTestConfigResult
 

exasol/slc/models/accelerator.py

@@ -1,5 +1,4 @@
 from enum import Enum
-from typing import List
 
 
 class Accelerator(Enum):

exasol/slc/models/flavor_ci_model.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Optional
 
 from pydantic import BaseModel
 

exasol/slc/tool/commands/run_db_tests.py

@@ -12,13 +12,13 @@
     system_options,
 )
 from exasol_integration_test_docker_environment.cli.options.test_environment_options import (
-    docker_db_options,
     external_db_options,
     test_environment_options,
 )
 from exasol_integration_test_docker_environment.cli.termination_handler import (
     TerminationHandler,
 )
+from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports
 from exasol_integration_test_docker_environment.lib.utils.cli_function_decorators import (
     add_options,
 )
@@ -36,6 +36,7 @@
 from exasol.slc.tool.options.flavor_options import flavor_options
 from exasol.slc.tool.options.goal_options import release_options
 from exasol.slc.tool.options.test_container_options import test_container_options
+from exasol.slc.tool.options.test_environment_options import docker_db_options
 
 
 @cli.command(short_help="Runs integration tests.")
@@ -152,6 +153,12 @@
     default=defaultAccelerator().value,
     help=f"""Accelerator to be enabled for tests in docker-db. Possible values: {acceleratorValues()}""",
 )
+@click.option(
+    "--external-exasol-bucketfs-https-port",
+    type=int,
+    default=Ports.external.bucketfs_https,
+    help="""Bucketfs port of external Exasol DB.""",
+)
 def run_db_test(
     flavor_path: tuple[str, ...],
     release_goal: tuple[str, ...],
@@ -212,6 +219,7 @@ def run_db_test(
     use_job_specific_log_file: bool,
     compression_strategy: str,
     accelerator: str,
+    external_exasol_bucketfs_https_port: int,
 ):
     """
     This command runs the integration tests in local docker-db.
@@ -282,6 +290,7 @@ def run_db_test(
                 compression_strategy=CompressionStrategy[compression_strategy.upper()],
                 docker_environment_variable=docker_environment_variable,
                 accelerator=Accelerator[accelerator.upper()],
+                external_exasol_bucketfs_https_port=external_exasol_bucketfs_https_port,
             )
             if result.command_line_output_path.exists():
                 with result.command_line_output_path.open("r") as f:

exasol/slc/tool/options/test_environment_options.py

@@ -0,0 +1,58 @@
+import click
+from exasol_integration_test_docker_environment.cli.options.test_environment_options import (
+    LATEST_DB_VERSION,
+)
+from exasol_integration_test_docker_environment.lib.test_environment.parameter.docker_db_test_environment_parameter import (
+    DbOsAccess,
+)
+
+docker_db_options = [
+    click.option(
+        "--docker-db-image-version",
+        type=str,
+        default=LATEST_DB_VERSION,
+        show_default=True,
+        help="""Docker DB Image Version against which the tests should run.""",
+    ),
+    click.option(
+        "--docker-db-image-name",
+        type=str,
+        default="""exasol/docker-db""",
+        show_default=True,
+        help="""Docker DB Image Name against which the tests should run.""",
+    ),
+    click.option(
+        "--db-os-access",
+        type=click.Choice([e.name for e in DbOsAccess]),
+        metavar="METHOD",
+        default="""DOCKER_EXEC""",
+        show_default=True,
+        help="""How to access file system and command line of the
+     		 database operating system. Experimental option, will show no
+     		 effect until implementation of feature SSH access is
+     		 completed.""",
+    ),
+    click.option(
+        "--create-certificates/--no-create-certificates",
+        default=False,
+        help="""Creates and injects SSL certificates to the Docker DB container.""",
+    ),
+    click.option(
+        "--additional-db-parameter",
+        "-p",
+        type=str,
+        multiple=True,
+        default=(),
+        help="""Additional database parameter which will be injected to EXAConf. Value should have format '-param=value'.""",
+    ),
+    click.option(
+        "--docker-environment-variable",
+        type=str,
+        multiple=True,
+        default=[],
+        help="""An environment variable which will be added to the docker-db.
+                The variable needs to have format "key=value".
+                For example "HTTPS_PROXY=192.168.1.5".
+                You can repeat this option to add further environment variables.""",
+    ),
+]