Revise Linux GPU unittests CI workflow and remove ffmpeg CI workflow.

.github/workflows/unittest-linux-gpu.yml

@@ -1,98 +1,189 @@
 name: Unit-tests on Linux GPU
 
 on:
-  pull_request:
-  push:
-    branches:
-      - nightly
-      - main
-      - release/*
-  workflow_dispatch:
+    pull_request:
+    push:
+        branches:
+            - nightly
+            - main
+            - release/*
+    workflow_dispatch:
 
 jobs:
-  tests:
-    strategy:
-      matrix:
-        # TODO add up to 3.13
-        python_version: ["3.10"]
-        cuda_arch_version: ["12.6"]
-      fail-fast: false
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      runner: linux.g5.4xlarge.nvidia.gpu
-      repository: pytorch/audio
-      gpu-arch-type: cuda
-      gpu-arch-version: ${{ matrix.cuda_arch_version }}
-      timeout: 120
-
-      script: |
-        set -ex
-        # Set up Environment Variables
-        export PYTHON_VERSION="${{ matrix.python_version }}"
-        export PIP_PROGRESS_BAR=off
-        export CONDA_QUIET=1
-        export CU_VERSION="${{ matrix.cuda_arch_version }}"
-        export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true
-        export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true
-        # Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility
-        export CUBLAS_WORKSPACE_CONFIG=:4096:8
-
-        # Set UPLOAD_CHANNEL
-        if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
-          export UPLOAD_CHANNEL=test
-        else
-          export UPLOAD_CHANNEL=nightly
-        fi
-
-        echo "::group::Create conda env"
-        # Mark Build Directory Safe
-        git config --global --add safe.directory /__w/audio/audio
-        conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}"
-        conda activate ./ci_env
-
-        echo "::endgroup::"
-        echo "::group::Install PyTorch"
-        export GPU_ARCH_ID="cu126"  # TODO this is currently hardcoded, should depend on matrix's value.
-        PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/${GPU_ARCH_ID}"
-        pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
-        echo "::endgroup::"
-
-        echo "::group::Install TorchAudio"
-        conda install --quiet --yes cmake ninja
-        pip3 install --progress-bar off -v -e . --no-use-pep517
-
-        echo "::endgroup::"
-        echo "::group::Build FFmpeg"
-        .github/scripts/ffmpeg/build_gpu.sh
-
-        echo "::endgroup::"
-        echo "::group::Install other Dependencies"
-        
-        pip3 install parameterized requests coverage pytest pytest-cov scipy numpy expecttest 
-
-        echo "::endgroup::"
-        echo "::group::Run tests"
-        export PATH="${PWD}/third_party/install/bin/:${PATH}"
-
-        declare -a args=(
-            '-v'
-            '--cov=torchaudio'
-            "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml"
-            '--durations' '100'
-            '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency'
-        )
-
-        cd test
-        python3 -m torch.utils.collect_env
-        env | grep TORCHAUDIO || true
-        pytest "${args[@]}" torchaudio_unittest
-        coverage html
+    build:
+        # Do not use matrix here to parameterize Python/CUDA versions.
+        # This job is required to pass for each PR.
+        # The name of the required job is sensitive to matrix parameter.
+        uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+        permissions:
+            id-token: write
+            contents: read
+        with:
+            job-name: Run tests
+            runner: linux.g5.4xlarge.nvidia.gpu
+            repository: pytorch/audio
+            gpu-arch-type: cuda
+            gpu-arch-version: "12.6" # See GPU_ARCH_ID below
+            timeout: 120
+
+            script: |
+                set -ex
+
+                # Set up Environment Variables
+                export PYTHON_VERSION="3.10"
+                export CU_VERSION="11.8"
+                export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
+                export PIP_PROGRESS_BAR=off
+                export CONDA_QUIET=1
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_CUDA_SMALL_MEMORY=true
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_ON_PYTHON_310=true
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_TEMPORARY_DISABLED=true
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_DECODER=true
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_SOX_ENCODER=true
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=true
+                export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_MOD_demucs=true
+                # Avoid reproducibility errors with CUBLAS: https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility
+                export CUBLAS_WORKSPACE_CONFIG=:4096:8
+
+                # Set CHANNEL
+                if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
+                  export CHANNEL=test
+                  export BUILD_VERSION="$( cut -f 1 -d a version.txt )"
+                else
+                  export CHANNEL=nightly
+                  export BUILD_VERSION="$( cut -f 1 -d a version.txt )".dev"$(date "+%Y%m%d")"
+                fi
+
+                # Install miniforge
+                wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+                bash Miniforge3.sh -b -p "${HOME}/conda"
+                source "${HOME}/conda/etc/profile.d/conda.sh"
+
+                echo "::group::Create conda env"
+                # Mark Build Directory Safe
+                git config --global --add safe.directory /__w/audio/audio
+                conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}"
+                conda activate ./ci_env
+                conda install -q -y pip numpy
+
+                echo "::endgroup::"
+                echo "::group::Install PyTorch"
+
+                GPU_ARCH_ID=cu126  # This is hard-coded and must be consistent with gpu-arch-version.
+                PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}"
+                pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
+
+                echo "::endgroup::"
+                echo "::group::Install TorchAudio"
+                conda install --quiet --yes cmake>=3.18.0 ninja
+                pip install --progress-bar off -v . --no-build-isolation
+
+                # TODO: Need to rely on torchcodec instead of building ffmpeg from source.
+                echo "::endgroup::"
+                echo "::group::Build FFmpeg"
+                conda install -q -y "ffmpeg<=7"
+
+                echo "::endgroup::"
+                echo "::group::Install other dependencies"
+
+                pip install --progress-bar off parameterized requests coverage pytest pytest-cov scipy numpy expecttest
+
+                declare -a args=(
+                    '-v'
+                    '--cov=torchaudio'
+                    "--junitxml=${RUNNER_TEST_RESULTS_DIR}/junit.xml"
+                    '--durations' '100'
+                    '-k' '(cuda or gpu) and not (torchscript and rnnt) and not torchscript_consistency'
+                )
+
+                cd test
+                python3 -m torch.utils.collect_env
+                env | grep TORCHAUDIO || true
+                pytest "${args[@]}" torchaudio_unittest
+                coverage html
+
+    commit-main:
+        if: github.ref_name == 'main'
+        permissions:
+            # Required for `git push`
+            # Note:
+            #   This is not effective from fork.
+            #   When you debug this, make sure to make a branch on pytorch and
+            #   make PR from there.
+            contents: write
+        runs-on: ubuntu-latest
+        needs: build
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  ref: gh-pages
+                  fetch-depth: 5
+            - uses: actions/download-artifact@v3
+              with:
+                  name: docs
+            - name: Update main doc
+              run: |
+                  set -x
+
+                  git config user.name "pytorchbot"
+                  git config user.email "soumith+bot@pytorch.org"
+                  # When `git clone`, `gh-pages` branch is fetched by default.
+                  # The size of gh-pages grows significantly, so we use ammend and force push
+                  # We add a new commit once a week
+                  if [ "$(date +%d)" = "1" ]; then
+                     git commit --allow-empty -m "placeholder"
+                  fi
+
+                  # TODO: add tag-based process (need to handle the main directory name)
+                  # Update the main doc
+                  rm -rf main
+                  mv html main
+                  git add --all main || true
+
+                  git commit --amend -m "auto-generating sphinx docs" || true
+                  git push -f
+
+    # Push for release
+    # Make sure that version.txt is updated first (alpha suffix is removed)
+    commit-release:
+        if: startsWith(github.ref_name, 'release/') && ( github.event_name == 'workflow_dispatch' )
+        permissions:
+            # Required for `git push`
+            # Note:
+            #   This is not effective from fork.
+            #   When you debug this, make sure to make a branch on pytorch and
+            #   make PR from there.
+            contents: write
+        runs-on: ubuntu-latest
+        needs: build
+        steps:
+            - uses: actions/checkout@v3
+              with:
+                  ref: gh-pages
+                  fetch-depth: 5
+            - uses: actions/checkout@v4
+              with:
+                  path: _src
+            - uses: actions/download-artifact@v3
+              with:
+                  name: docs
+            - name: Update doc
+              run: |
+                  set -x
+
+                  git config user.name "pytorchbot"
+                  git config user.email "soumith+bot@pytorch.org"
+                  # When `git clone`, `gh-pages` branch is fetched by default.
+                  # The size of gh-pages grows significantly, so we use ammend and force push
+                  # We add a new commit once a week
+                  if [ "$(date +%d)" = "1" ]; then
+                     git commit --allow-empty -m "placeholder"
+                  fi
+
+                  dirname="$(cat _src/version.txt)"
+                  rm -rf "${dirname}"
+                  mv html "${dirname}"
+                  git add --all "${dirname}" || true
+
+                  git commit --amend -m "auto-generating sphinx docs" || true
+                  git push -f