diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml
deleted file mode 100644
index effc44ced5..0000000000
--- a/.github/workflows/build_docs.yml
+++ /dev/null
@@ -1,176 +0,0 @@
-name: Build documentation
-
-# Documentation is deployed in the following ways
-# 1. Each time a commit is pushed to main branch
-# 2. Nightly (so that docs are updated even when there is no commit to main branch on the day)
-# 3. Manual trigger for release
-#    Because release requires changing the version number, which is used as a directory name,
-#    automating release doc deployment is tricky.
-#    There is no reliable way to know if there should have been a minor version bump.
-#
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-  workflow_dispatch:
-  schedule:
-    - cron: '0 0 * * *'
-
-jobs:
-  build:
-    # Do not use matrix here to parameterize Python/CUDA versions.
-    # This job is required to pass for each PR.
-    # The name of the required job is sensitive to matrix parameter.
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      job-name: Build doc
-      runner: linux.g5.4xlarge.nvidia.gpu
-      repository: pytorch/audio
-      gpu-arch-type: cuda
-      gpu-arch-version: "12.6"  # See GPU_ARCH_ID below
-      timeout: 120
-      upload-artifact: docs
-
-      script: |
-        set -ex
-        # Set up Environment Variables
-        export PYTHON_VERSION="3.10"
-        export CU_VERSION="11.8"
-        export CUDATOOLKIT="pytorch-cuda=${CU_VERSION}"
-
-        # Set CHANNEL
-        if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
-          export CHANNEL=test
-          export BUILD_VERSION="$( cut -f 1 -d a version.txt )"
-        else
-          export CHANNEL=nightly
-          export BUILD_VERSION="$( cut -f 1 -d a version.txt )".dev"$(date "+%Y%m%d")"
-        fi
-
-        echo "::group::Create conda env"
-        # Mark Build Directory Safe
-        git config --global --add safe.directory /__w/audio/audio
-        conda create --quiet -y --prefix ci_env python="${PYTHON_VERSION}"
-        conda activate ./ci_env
-
-        echo "::endgroup::"
-        echo "::group::Install PyTorch"
-
-        GPU_ARCH_ID=cu126  # This is hard-coded and must be consistent with gpu-arch-version.
-        PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${CHANNEL}/${GPU_ARCH_ID}"
-        pip install --progress-bar=off --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
-
-        echo "::endgroup::"
-        echo "::group::Install TorchAudio"
-        conda install --quiet --yes cmake>=3.18.0 ninja
-        pip3 install --progress-bar off -v -e . --no-use-pep517
-
-        # TODO: Need to rely on torchcodec instead of building ffmpeg from source.
-        echo "::endgroup::"
-        echo "::group::Build FFmpeg"
-        .github/scripts/ffmpeg/build_gpu.sh
-
-        echo "::endgroup::"
-        echo "::group::Install other dependencies"
-
-        pip install --progress-bar off -r docs/requirements.txt -r docs/requirements-tutorials.txt
-
-        echo "::endgroup::"
-        echo "::group::Build documentation"
-        export BUILD_GALLERY=true
-        (cd docs && make html)
-
-        echo "::endgroup::"
-        echo "::group::Copy artifact"
-        cp -rf docs/build/html/* "${RUNNER_DOCS_DIR}"
-        mv docs/build/html /artifacts/
-
-  commit-main:
-    if: github.ref_name == 'main'
-    permissions:
-      # Required for `git push`
-      # Note:
-      #   This is not effective from fork.
-      #   When you debug this, make sure to make a branch on pytorch and
-      #   make PR from there.
-      contents: write
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: gh-pages
-        fetch-depth: 5
-    - uses: actions/download-artifact@v3
-      with:
-        name: docs
-    - name: Update main doc
-      run: |
-        set -x
-
-        git config user.name "pytorchbot"
-        git config user.email "soumith+bot@pytorch.org"
-        # When `git clone`, `gh-pages` branch is fetched by default.
-        # The size of gh-pages grows significantly, so we use ammend and force push
-        # We add a new commit once a week
-        if [ "$(date +%d)" = "1" ]; then
-           git commit --allow-empty -m "placeholder"
-        fi
-
-        # TODO: add tag-based process (need to handle the main directory name)
-        # Update the main doc
-        rm -rf main
-        mv html main
-        git add --all main || true
-
-        git commit --amend -m "auto-generating sphinx docs" || true
-        git push -f
-
-  # Push for release
-  # Make sure that version.txt is updated first (alpha suffix is removed)
-  commit-release:
-    if: startsWith(github.ref_name, 'release/') && ( github.event_name == 'workflow_dispatch' )
-    permissions:
-      # Required for `git push`
-      # Note:
-      #   This is not effective from fork.
-      #   When you debug this, make sure to make a branch on pytorch and
-      #   make PR from there.
-      contents: write
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: gh-pages
-        fetch-depth: 5
-    - uses: actions/checkout@v4
-      with:
-        path: _src
-    - uses: actions/download-artifact@v3
-      with:
-        name: docs
-    - name: Update doc
-      run: |
-        set -x
-
-        git config user.name "pytorchbot"
-        git config user.email "soumith+bot@pytorch.org"
-        # When `git clone`, `gh-pages` branch is fetched by default.
-        # The size of gh-pages grows significantly, so we use ammend and force push
-        # We add a new commit once a week
-        if [ "$(date +%d)" = "1" ]; then
-           git commit --allow-empty -m "placeholder"
-        fi
-
-        dirname="$(cat _src/version.txt)"
-        rm -rf "${dirname}"
-        mv html "${dirname}"
-        git add --all "${dirname}" || true
-
-        git commit --amend -m "auto-generating sphinx docs" || true
-        git push -f
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
new file mode 100644
index 0000000000..26bcbf8083
--- /dev/null
+++ b/.github/workflows/docs.yaml
@@ -0,0 +1,80 @@
+name: Docs
+
+on:
+  pull_request:
+  push:
+    branches:
+      - nightly
+      - main
+      - release/*
+  workflow_dispatch:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+        cuda-version: ["12.6"]
+        ffmpeg-version: ["7"]
+      fail-fast: false
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.4xlarge.nvidia.gpu
+      repository: pytorch/audio
+      gpu-arch-type: cuda
+      gpu-arch-version: ${{ matrix.cuda-version }}
+      timeout: 120
+      job-name: docs
+
+      script: |
+        set -ex
+        # Set up Environment Variables
+        export PYTHON_VERSION="${{ matrix.python-version }}"
+        export FFMPEG_VERSION="${{ matrix.ffmpeg-version }}"
+        export CU_VERSION_WITHOUT_PERIODS="$(echo "${{ matrix.cuda-version }}" | sed 's/\.//g')"
+        export PIP_PROGRESS_BAR=off
+        export CONDA_QUIET=1
+
+        # Set UPLOAD_CHANNEL
+        if [[(${GITHUB_EVENT_NAME} = 'pull_request' && (${GITHUB_BASE_REF} = 'release'*)) || (${GITHUB_REF} = 'refs/heads/release'*) ]]; then
+          export UPLOAD_CHANNEL=test
+        else
+          export UPLOAD_CHANNEL=nightly
+        fi
+
+        echo "::group::Create conda env"
+        # Mark Build Directory Safe
+        git config --global --add safe.directory /__w/audio/audio
+        conda create -c conda-forge --strict-channel-priority -y --prefix ci_env python="${PYTHON_VERSION}" ffmpeg="${FFMPEG_VERSION}" cmake ninja
+        conda activate ./ci_env
+        conda info
+        python -m pip install --upgrade pip
+        echo "::endgroup::"
+
+        echo "::group::Install PyTorch and TorchCodec"
+        PYTORCH_WHEEL_INDEX="https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cu${CU_VERSION_WITHOUT_PERIODS}"
+        python -m pip install numpy
+        python -m pip install --pre torch torchcodec --index-url="${PYTORCH_WHEEL_INDEX}"
+        python -c 'import torch; print(f"{torch.__version__}"); print(f"{torch.__file__}"); print(f"{torch.cuda.is_available()=}")'
+        echo "::endgroup::"
+
+        echo "::group::Build and install TorchAudio"
+        python -m pip install . -v --no-build-isolation
+        echo "::endgroup::"
+
+        echo "::group::Install TorchAudio documentation build requirements"
+        python -m pip install -r docs/requirements.txt -r docs/requirements-tutorials.txt
+        echo "::endgroup::"
+
+        echo "::group::Build TorchAudio documentation"
+        export BUILD_GALLERY=true
+        (cd docs && make html)
+        echo "::endgroup::"
+
+        echo "::group::Copy artifacts"
+        cp -rf docs/build/html/* "${RUNNER_DOCS_DIR}"
+        mv docs/build/html /artifacts/
+        echo "::endgroup::"