singularity dockerFile support, add tests and fixes

Author: mr-c

cwltool/singularity.py

@@ -1,6 +1,7 @@
 """Support for executing Docker format containers using Singularity {2,3}.x or Apptainer 1.x."""
 
 import copy
+import hashlib
 import logging
 import os
 import os.path
@@ -13,6 +14,7 @@
 from typing import cast
 
 from schema_salad.sourceline import SourceLine
+from schema_salad.utils import json_dumps
 from spython.main import Client
 from spython.main.parse.parsers.docker import DockerParser
 from spython.main.parse.writers.singularity import SingularityWriter
@@ -206,41 +208,46 @@ def get_image(
             absolute_path = os.path.abspath(cache_folder)
             if "dockerImageId" in docker_req:
                 image_name = docker_req["dockerImageId"]
-                image_path = os.path.join(absolute_path, image_name)
-                if os.path.exists(image_path):
-                    found = True
+            else:
+                image_name = hashlib.md5(  # nosec
+                    json_dumps(dockerRequirement, separators=(",", ":"), sort_keys=True).encode(
+                        "utf-8"
+                    )
+                ).hexdigest()
+            if is_version_3_or_newer():
+                image_name = _normalize_sif_id(image_name)
+            else:
+                image_name = _normalize_image_id(image_name)
+            image_name = os.path.join(absolute_path, image_name)
+            docker_req["dockerImageId"] = image_name
+            if os.path.exists(image_name):
+                found = True
             if found is False:
                 dockerfile_path = os.path.join(absolute_path, "Dockerfile")
                 singularityfile_path = dockerfile_path + ".def"
-                # if you do not set APPTAINER_TMPDIR will crash
-                # WARNING: 'nodev' mount option set on /tmp, it could be a
-                #          source of failure during build process
-                # FATAL:   Unable to create build: 'noexec' mount option set on
-                #          /tmp, temporary root filesystem won't be usable at this location
                 with open(dockerfile_path, "w") as dfile:
                     dfile.write(docker_req["dockerFile"])
 
-                singularityfile = SingularityWriter(DockerParser(dockerfile_path).parse()).convert()
+                docker_recipe = DockerParser(dockerfile_path).parse()
+                docker_recipe["spython-base"].entrypoint = ""
+                singularityfile = SingularityWriter(docker_recipe).convert()
                 with open(singularityfile_path, "w") as file:
                     file.write(singularityfile)
 
+                # if you do not set APPTAINER_TMPDIR will crash
+                # WARNING: 'nodev' mount option set on /tmp, it could be a
+                #          source of failure during build process
+                # FATAL:   Unable to create build: 'noexec' mount option set on
+                #          /tmp, temporary root filesystem won't be usable at this location
                 os.environ["APPTAINER_TMPDIR"] = absolute_path
                 singularity_options = ["--fakeroot"] if not shutil.which("proot") else []
-                if "dockerImageId" in docker_req:
-                    Client.build(
-                        recipe=singularityfile_path,
-                        build_folder=absolute_path,
-                        image=docker_req["dockerImageId"],
-                        sudo=False,
-                        options=singularity_options,
-                    )
-                else:
-                    Client.build(
-                        recipe=singularityfile_path,
-                        build_folder=absolute_path,
-                        sudo=False,
-                        options=singularity_options,
-                    )
+                Client.build(
+                    recipe=singularityfile_path,
+                    build_folder=absolute_path,
+                    image=image_name,
+                    sudo=False,
+                    options=singularity_options,
+                )
                 found = True
         elif "dockerImageId" not in docker_req and "dockerPull" in docker_req:
             match = re.search(pattern=r"([a-z]*://)", string=docker_req["dockerPull"])

tests/sing_dockerfile_named_test.cwl

@@ -0,0 +1,18 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerFile: |
+      FROM docker.io/debian:stable-slim
+    dockerImageId: customDebian
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo
+arguments:
+  - $(inputs.message)

tests/sing_dockerfile_test.cwl

@@ -0,0 +1,17 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+
+requirements:
+  DockerRequirement:
+    dockerFile: |
+      FROM docker.io/debian:stable-slim
+
+inputs:
+  message: string
+
+outputs: []
+
+baseCommand: echo
+arguments:
+  - $(inputs.message)

tests/test_singularity.py

@@ -6,6 +6,7 @@
 import pytest
 
 from cwltool.main import main
+from cwltool.singularity import _IMAGES, _IMAGES_LOCK
 
 from .util import (
     get_data,
@@ -17,6 +18,12 @@
 )
 
 
+@pytest.fixture(autouse=True)
+def clear_singularity_image_cache() -> None:
+    with _IMAGES_LOCK:
+        _IMAGES.clear()
+
+
 @needs_singularity_2_6
 def test_singularity_pullfolder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     """Test singularity respects SINGULARITY_PULLFOLDER."""
@@ -149,7 +156,7 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None:
                 "hello",
             ]
         )
-        assert result_code == 0
+        assert result_code == 0, stderr
         result_code1, stdout, stderr = get_main_output(
             [
                 "--singularity",
@@ -159,3 +166,89 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None:
             ]
         )
         assert result_code1 == 0
+
+
+@needs_singularity_3_or_newer
+def test_singularity_dockerfile_no_name_no_cache(tmp_path: Path) -> None:
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    with working_directory(workdir):
+        result_code, stdout, stderr = get_main_output(
+            [
+                "--singularity",
+                get_data("tests/sing_dockerfile_test.cwl"),
+                "--message",
+                "hello",
+            ]
+        )
+        assert result_code == 0, stderr
+    assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists()
+
+
+@needs_singularity_3_or_newer
+def test_singularity_dockerfile_no_name_with_cache(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    cachedir = tmp_path / "cache"
+    cachedir.mkdir()
+    monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(cachedir))
+    with working_directory(workdir):
+        result_code, stdout, stderr = get_main_output(
+            [
+                "--singularity",
+                get_data("tests/sing_dockerfile_test.cwl"),
+                "--message",
+                "hello",
+            ]
+        )
+        assert result_code == 0, stderr
+    assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists()
+    assert (cachedir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists()
+
+
+@needs_singularity_3_or_newer
+def test_singularity_dockerfile_with_name_no_cache(tmp_path: Path) -> None:
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    with working_directory(workdir):
+        result_code, stdout, stderr = get_main_output(
+            [
+                "--singularity",
+                get_data("tests/sing_dockerfile_named_test.cwl"),
+                "--message",
+                "hello",
+            ]
+        )
+        assert result_code == 0, stderr
+    print(list(workdir.iterdir()))
+    assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists()
+    assert not (workdir / "customDebian_latest.sif").exists()
+
+
+@needs_singularity_3_or_newer
+def test_singularity_dockerfile_with_name_with_cache(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    workdir = tmp_path / "working_dir"
+    workdir.mkdir()
+    cachedir = tmp_path / "cache"
+    cachedir.mkdir()
+    monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(cachedir))
+    with working_directory(workdir):
+        result_code, stdout, stderr = get_main_output(
+            [
+                "--singularity",
+                get_data("tests/sing_dockerfile_named_test.cwl"),
+                "--message",
+                "hello",
+            ]
+        )
+        print(list(workdir.iterdir()))
+        print(list(cachedir.iterdir()))
+        assert result_code == 0, stderr
+    assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists()
+    assert not (cachedir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists()
+    assert not (workdir / "customDebian_latest.sif").exists()
+    assert (cachedir / "customDebian_latest.sif").exists()