Convert pipeline container to a multi-part build

.dockerignore

@@ -1,9 +1,39 @@
-*
-!requirements.txt
-!Cargo.toml
-!Cargo.lock
-!utils
-!openssl
-!uv.lock
-!pyproject.toml
-!Makefile
+# Exclude build and development files
+.git/
+.github/
+*.nf
+workflows/
+tests/
+containers/
+Dockerfile.old
+.dockerignore
+.gitignore
+.pre-commit-config.yaml
+pytest.ini
+.python-version
+RELEASE.rst
+LICENSE
+README.md
+*.md
+
+# Python artifacts
+__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+.coverage
+htmlcov/
+*.egg-info/
+dist/
+build/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db

.github/workflows/main.yaml

@@ -22,22 +22,112 @@ jobs:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          MSG_MINIMAL: true
 
+  wait-for-rust-build:
+    runs-on: ubuntu-latest
+    outputs:
+      rust_changed: ${{ steps.check-rust.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Check if Rust files changed
+        id: check-rust
+        run: |
+          if git diff --name-only HEAD^ HEAD | grep -qE '^(utils/|Cargo\.(toml|lock))'; then
+            echo "changed=true" >> $GITHUB_OUTPUT
+            echo "Rust files changed, waiting for rust-container workflow..."
+          else
+            echo "changed=false" >> $GITHUB_OUTPUT
+            echo "No Rust changes, skipping wait"
+          fi
+
+      - name: Wait for Rust container build
+        if: steps.check-rust.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "Waiting for rust-container workflow to complete..."
+          sleep 30  # Give workflow time to start
+
+          # Wait up to 30 minutes for rust-container workflow to complete
+          timeout=1800
+          elapsed=0
+          while [ $elapsed -lt $timeout ]; do
+            # Check for running or completed rust-container workflows for this commit
+            status=$(gh run list \
+              --workflow=rust-container.yaml \
+              --commit=${{ github.sha }} \
+              --json status,conclusion \
+              --jq '.[0] | "\(.status):\(.conclusion)"')
+
+            if [ -z "$status" ]; then
+              echo "Warning: rust-container workflow not found yet, waiting..."
+              sleep 10
+              elapsed=$((elapsed + 10))
+              continue
+            fi
+
+            workflow_status=$(echo "$status" | cut -d: -f1)
+            workflow_conclusion=$(echo "$status" | cut -d: -f2)
+
+            if [ "$workflow_status" = "completed" ]; then
+              if [ "$workflow_conclusion" = "success" ]; then
+                echo "✓ Rust container build completed successfully"
+                exit 0
+              else
+                echo "✗ Rust container build failed with conclusion: $workflow_conclusion"
+                exit 1
+              fi
+            fi
+
+            echo "Rust container build status: $workflow_status (elapsed: ${elapsed}s)"
+            sleep 30
+            elapsed=$((elapsed + 30))
+          done
+
+          echo "✗ Timeout waiting for rust-container workflow"
+          exit 1
+
   create-docker-image:
+    needs: wait-for-rust-build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
+      - name: Read versions
+        id: versions
+        run: |
+          echo "pipeline=$(cat VERSION)" >> $GITHUB_OUTPUT
+          echo "infernal=$(cat containers/infernal/VERSION)" >> $GITHUB_OUTPUT
+          echo "samtools=$(cat containers/samtools/VERSION)" >> $GITHUB_OUTPUT
+
+          RUST_VERSION=$(cat containers/rust-utils/VERSION)
+          if [ "$RUST_VERSION" = "auto" ]; then
+            RUST_VERSION=$(git rev-parse --short=8 HEAD)
+          fi
+          echo "rust=$RUST_VERSION" >> $GITHUB_OUTPUT
+
       - name: docker login
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USER }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
 
       - name: docker build
-        run: docker build -f Dockerfile -t rnacentral/rnacentral-import-pipeline:latest .
+        run: |
+          docker build \
+            --build-arg INFERNAL_VERSION=${{ steps.versions.outputs.infernal }} \
+            --build-arg SAMTOOLS_VERSION=${{ steps.versions.outputs.samtools }} \
+            --build-arg RUST_VERSION=${{ steps.versions.outputs.rust }} \
+            -t rnacentral/rnacentral-import-pipeline:${{ steps.versions.outputs.pipeline }} \
+            -t rnacentral/rnacentral-import-pipeline:latest \
+            -f Dockerfile .
 
       - name: docker push
-        run: docker push rnacentral/rnacentral-import-pipeline:latest
+        run: |
+          docker push rnacentral/rnacentral-import-pipeline:${{ steps.versions.outputs.pipeline }}
+          docker push rnacentral/rnacentral-import-pipeline:latest
 
   finished-notification:
     needs:

.github/workflows/manual_build.yaml

@@ -19,22 +19,112 @@ jobs:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          MSG_MINIMAL: true
 
+  wait-for-rust-build:
+    runs-on: ubuntu-latest
+    outputs:
+      rust_changed: ${{ steps.check-rust.outputs.changed }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - name: Check if Rust files changed
+        id: check-rust
+        run: |
+          if git diff --name-only HEAD^ HEAD | grep -qE '^(utils/|Cargo\.(toml|lock))'; then
+            echo "changed=true" >> $GITHUB_OUTPUT
+            echo "Rust files changed, waiting for rust-container workflow..."
+          else
+            echo "changed=false" >> $GITHUB_OUTPUT
+            echo "No Rust changes, skipping wait"
+          fi
+
+      - name: Wait for Rust container build
+        if: steps.check-rust.outputs.changed == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          echo "Waiting for rust-container workflow to complete..."
+          sleep 30  # Give workflow time to start
+
+          # Wait up to 30 minutes for rust-container workflow to complete
+          timeout=1800
+          elapsed=0
+          while [ $elapsed -lt $timeout ]; do
+            # Check for running or completed rust-container workflows for this commit
+            status=$(gh run list \
+              --workflow=rust-container.yaml \
+              --commit=${{ github.sha }} \
+              --json status,conclusion \
+              --jq '.[0] | "\(.status):\(.conclusion)"')
+
+            if [ -z "$status" ]; then
+              echo "Warning: rust-container workflow not found yet, waiting..."
+              sleep 10
+              elapsed=$((elapsed + 10))
+              continue
+            fi
+
+            workflow_status=$(echo "$status" | cut -d: -f1)
+            workflow_conclusion=$(echo "$status" | cut -d: -f2)
+
+            if [ "$workflow_status" = "completed" ]; then
+              if [ "$workflow_conclusion" = "success" ]; then
+                echo "✓ Rust container build completed successfully"
+                exit 0
+              else
+                echo "✗ Rust container build failed with conclusion: $workflow_conclusion"
+                exit 1
+              fi
+            fi
+
+            echo "Rust container build status: $workflow_status (elapsed: ${elapsed}s)"
+            sleep 30
+            elapsed=$((elapsed + 30))
+          done
+
+          echo "✗ Timeout waiting for rust-container workflow"
+          exit 1
+
   create-docker-image:
+    needs: wait-for-rust-build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
+      - name: Read versions
+        id: versions
+        run: |
+          echo "pipeline=$(cat VERSION)" >> $GITHUB_OUTPUT
+          echo "infernal=$(cat containers/infernal/VERSION)" >> $GITHUB_OUTPUT
+          echo "samtools=$(cat containers/samtools/VERSION)" >> $GITHUB_OUTPUT
+
+          RUST_VERSION=$(cat containers/rust-utils/VERSION)
+          if [ "$RUST_VERSION" = "auto" ]; then
+            RUST_VERSION=$(git rev-parse --short=8 HEAD)
+          fi
+          echo "rust=$RUST_VERSION" >> $GITHUB_OUTPUT
+
       - name: docker login
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USER }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
         run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
 
       - name: docker build
-        run: docker build -f Dockerfile -t rnacentral/rnacentral-import-pipeline:latest .
+        run: |
+          docker build \
+            --build-arg INFERNAL_VERSION=${{ steps.versions.outputs.infernal }} \
+            --build-arg SAMTOOLS_VERSION=${{ steps.versions.outputs.samtools }} \
+            --build-arg RUST_VERSION=${{ steps.versions.outputs.rust }} \
+            -t rnacentral/rnacentral-import-pipeline:${{ steps.versions.outputs.pipeline }} \
+            -t rnacentral/rnacentral-import-pipeline:latest \
+            -f Dockerfile .
 
       - name: docker push
-        run: docker push rnacentral/rnacentral-import-pipeline:latest
+        run: |
+          docker push rnacentral/rnacentral-import-pipeline:${{ steps.versions.outputs.pipeline }}
+          docker push rnacentral/rnacentral-import-pipeline:latest
 
   finished-notification:
     needs:

.github/workflows/rust-container.yaml

@@ -0,0 +1,94 @@
+# GitHub Actions workflow for building and pushing Rust utilities container
+# Rebuilds when Rust code changes (utils/**, Cargo.toml, Cargo.lock)
+
+name: Build Rust Utilities Container
+
+on:
+  push:
+    branches: ['master', 'dev']
+    paths:
+      - 'utils/**'
+      - 'Cargo.toml'
+      - 'Cargo.lock'
+      - 'containers/rust-utils/**'
+      - '.github/workflows/rust-container.yaml'
+  workflow_dispatch:
+    inputs:
+      force_rebuild:
+        description: 'Force rebuild Rust utilities container'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  build-rust-utils:
+    runs-on: ubuntu-latest
+    env:
+      FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Calculate Rust version
+        id: rust-version
+        run: |
+          VERSION_FILE="containers/rust-utils/VERSION"
+          VERSION_CONTENT=$(cat $VERSION_FILE)
+
+          if [ "$VERSION_CONTENT" = "auto" ]; then
+            GIT_SHA=$(git rev-parse --short=8 HEAD)
+
+            # Check for uncommitted changes
+            if ! git diff-index --quiet HEAD --; then
+              GIT_SHA="${GIT_SHA}-dirty"
+              echo "⚠️  Warning: Building with uncommitted changes"
+            fi
+
+            VERSION=$GIT_SHA
+          else
+            VERSION=$VERSION_CONTENT
+          fi
+
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "🔖 Building rust-utils:$VERSION"
+
+      - name: Docker login
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USER }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+      - name: Build Rust utilities container
+        run: |
+          VERSION=${{ steps.rust-version.outputs.version }}
+          if [ "$FORCE_REBUILD" = "true" ]; then
+            docker build --no-cache \
+              --build-arg RUST_VERSION=${VERSION} \
+              -t rnacentral/rust-utils:${VERSION} \
+              -f containers/rust-utils/Dockerfile .
+          else
+            docker build \
+              --build-arg RUST_VERSION=${VERSION} \
+              -t rnacentral/rust-utils:${VERSION} \
+              -f containers/rust-utils/Dockerfile .
+          fi
+
+      - name: Tag as latest
+        run: |
+          VERSION=${{ steps.rust-version.outputs.version }}
+          docker tag rnacentral/rust-utils:${VERSION} rnacentral/rust-utils:latest
+
+      - name: Push versioned tag
+        run: |
+          VERSION=${{ steps.rust-version.outputs.version }}
+          docker push rnacentral/rust-utils:${VERSION}
+
+      - name: Push latest tag
+        run: docker push rnacentral/rust-utils:latest
+
+      - name: Slack notification
+        if: always()
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_MESSAGE: 'Rust utilities ${{ steps.rust-version.outputs.version }} container built and pushed: ${{ job.status }}'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          MSG_MINIMAL: true