ci: improve CI caching scheme (#1994)

I recently discovered that the caching of our ccache was either not
working at all, or was much less efficient/helpful than I thought. So
there are a number of improvements here:

- Split caching into separate restore/save, and rearrange the step
  order so we save cache before tests run. This allows us to save the
  ccache even for jobs with failed tests (where the job fails), or for
  jobs that we purposely end early (maybe because we see failing
  tests), not only when all tests finish and succeed. This makes
  subsequent builds, that we do on the same branch while attempting to
  fix failures, go faster by using the cache.

- Make subsequent pushes to the same PR or REF to cancel any previous
  jobs, so if we push updates before the last CI run of the same
  branch completes, it won't keep those going as useless zombie jobs.

- Change the cache key names we use to remove the github ref to
  encourage cache reuse across different commits or branches (I
  think), and to fix a couple errors where we had mis-named things and
  were sharing caches between jobs when we shouldn't or vice versa.

- Be more careful about the location of the cache, making sure we were
  for sure telling the GHA cache action to be saving/restoring the
  same directory location that we told ccache to use as its cache.

- Use CCACHE_COMPRESSION env var to be extra sure we're using
  compressed caches to be sure we get the best bang for buck on the
  limited cache storage we're allowed.

In all, there were a number of subtle things broken. Give it a good
scrubbing.  I'm not really sure exactly which one of these, or which
combination, was most responsible for getting things unstuck, but it
definitely works!  Some results from two test pushes in a row I did
after these changes:

                              build deps     build osl
    VFXP 2024 / cold            4:12            6:11
    VFXP 2024 / cached          1:42 (2.5x)     0:48 (7x)

    ABI check / cold            4:11            6:14 + 5:38(abi ref)
    ABI check / cached          0:57 (4.4x)     0:46 + 0:16 (11.5x)

    bleeding edge / cold        8:12            9:57
    bleeding edge / cached      1:39 (5x)       0:44 (13.6x)

It varies across jobs and from run to run, but let's call it something
in the neighbourhood of 4x speedup building dependencies and 10x
building OSL itself. It's not better because it doesn't speed up
linking, and not all of the dependencies' build systems are doing
things in a way that is amenable to ccache inserting itself into the
process. Still, this should be a big help in reducing the time of
our CI runs.

---------

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

.github/workflows/build-steps.yml

@@ -115,22 +115,23 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           fetch-depth: '0'
-      - name: Prepare ccache timestamp
-        id: ccache_cache_keys
-        shell: bash
-        run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT
-      - name: ccache
-        id: ccache
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
-        with:
-          path: ./ccache
-          key: ${{github.job}}-${{inputs.nametag}}-${{steps.ccache_cache_keys.outputs.date}}
-          restore-keys: ${{github.job}}-
       - name: Build setup
         shell: bash
         run: |
             ${{inputs.setenvs}}
             src/build-scripts/ci-startup.bash
+      - name: Prepare ccache timestamp
+        id: ccache_cache_keys
+        shell: bash
+        run: echo "date=`date -u +'%Y-%m-%dT%H:%M:%SZ'`" >> $GITHUB_OUTPUT
+      - name: ccache-restore
+        id: ccache-restore
+        uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          # path: ./ccache
+          key: ${{inputs.nametag}}-${{steps.ccache_cache_keys.outputs.date}}
+          restore-keys: ${{inputs.nametag}}
       - name: Install LLVM and Clang
         if: inputs.llvm_action_ver != ''
         uses: KyleMayes/install-llvm-action@6ba6e2cd3813def9879be378609d87cb3ef3bac3 # v2.0.6
@@ -162,6 +163,26 @@ jobs:
         if: inputs.skip_build != '1'
         shell: bash
         run: src/build-scripts/ci-build.bash
+      - name: Check out ABI standard
+        if: inputs.abi_check != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{inputs.abi_check}}
+          path: abi_standard
+      - name: Build ABI standard
+        if: inputs.abi_check != ''
+        shell: bash
+        run: |
+            mkdir -p abi_standard/build
+            pushd abi_standard
+            src/build-scripts/ci-build.bash
+            popd
+      - name: ccache-save
+        id: ccache-save
+        uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ${{inputs.nametag}}-${{steps.ccache_cache_keys.outputs.date}}
       - name: Testsuite
         if: inputs.skip_tests != '1'
         shell: bash
@@ -186,20 +207,6 @@ jobs:
             # sonar-scanner --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}"
             time sonar-scanner --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" --define sonar.cfamily.build-wrapper-output="$BUILD_WRAPPER_OUT_DIR" --define sonar.cfamily.gcov.reportsPath="_coverage" --define sonar.cfamily.threads="$PARALLEL"
         # Consult https://docs.sonarcloud.io/advanced-setup/ci-based-analysis/sonarscanner-cli/ for more information and options
-      - name: Check out ABI standard
-        if: inputs.abi_check != ''
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          ref: ${{inputs.abi_check}}
-          path: abi_standard
-      - name: Build ABI standard
-        if: inputs.abi_check != ''
-        shell: bash
-        run: |
-            mkdir -p abi_standard/build
-            pushd abi_standard
-            src/build-scripts/ci-build.bash
-            popd
       - name: Check ABI
         if: inputs.abi_check != ''
         shell: bash

.github/workflows/ci.yml

@@ -36,6 +36,11 @@ on:
 
 permissions: read-all
 
+# Allow subsequent pushes to the same PR or REF to cancel any previous jobs.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 
 jobs:
 
@@ -87,7 +92,7 @@ jobs:
             batched: b8_AVX2,b8_AVX512,b16_AVX512
             setenvs: USE_OPENVDB=0
           - desc: gcc9/C++17 llvm11 py3.9 exr3.1 oiio3.0 sse2 batch-b4sse2
-            nametag: linux-vfx2021
+            nametag: linux-vfx2022-clang
             runner: ubuntu-latest
             container: aswftesting/ci-osl:2022-clang13
             vfxyear: 2022
@@ -151,20 +156,21 @@ jobs:
       #   with:
       #     egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v3.0.2
+      - name: Build setup
+        run: |
+            ${{matrix.setenvs}}
+            src/build-scripts/ci-startup.bash
       - name: Prepare ccache timestamp
         id: ccache_cache_keys
         run: echo "::set-output name=date::`date -u +'%Y-%m-%dT%H:%M:%SZ'`"
       - name: ccache
         id: ccache
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
         with:
-          path: /tmp/ccache
-          key: ${{github.job}}-${{matrix.nametag}}-${{steps.ccache_cache_keys.outputs.date}}
-          restore-keys: ${{github.job}}-${{matrix.nametag}}-
-      - name: Build setup
-        run: |
-            ${{matrix.setenvs}}
-            src/build-scripts/ci-startup.bash
+          path: ${{ env.CCACHE_DIR }}
+          key: ${{matrix.nametag}}-${{steps.ccache_cache_keys.outputs.date}}
+          restore-keys: ${{matrix.nametag}}-
+          save-always: true
       - name: Remove existing OpenEXR
         if: matrix.openexr_ver != ''
         run: |
@@ -307,7 +313,7 @@ jobs:
           # break the ABI. Basically, we will build that version as well as
           # the current one, and compare the resulting libraries.
           - desc: abi check
-            nametag: linux-vfx2023
+            nametag: linux-abi
             runner: ubuntu-latest
             container: aswftesting/ci-osl:2023-clang15
             cc_compiler: gcc

src/build-scripts/ci-build.bash

@@ -41,6 +41,7 @@ cp -r ${OSL_BUILD_DIR}/CMake* ${OSL_BUILD_DIR}/*.cmake ${OSL_BUILD_DIR}/cmake-sa
 if [[ "$BUILDTARGET" != "none" ]] ; then
     echo "Parallel build ${CMAKE_BUILD_PARALLEL_LEVEL} of target ${BUILDTARGET}"
     time ${OSL_CMAKE_BUILD_WRAPPER} cmake --build ${OSL_BUILD_DIR} --target ${BUILDTARGET} --config ${OSL_CMAKE_BUILD_TYPE}
+    ccache --show-stats || true
 fi
 
 if [[ "${DEBUG_CI:=0}" != "0" ]] ; then

src/build-scripts/ci-startup.bash

@@ -13,6 +13,14 @@ export PATH=/usr/local/bin/_ccache:/usr/lib/ccache:$PATH
 export USE_CCACHE=${USE_CCACHE:=1}
 export CCACHE_CPP2=
 export CCACHE_DIR=$HOME/.ccache
+export CCACHE_COMPRESSION=yes
+if [[ "$(which ccache)" != "" ]] ; then
+    # Try to coax dependency building into also using ccache
+    # Wait, no, this breaks old OIIO!
+    # export CMAKE_CXX_COMPILER_LAUNCHER="ccache"
+    # export CMAKE_C_COMPILER_LAUNCHER="ccache"
+    ccache -z
+fi
 mkdir -p $CCACHE_DIR
 
 export DISTDIR=$PWD/dist

src/cmake/compiler.cmake

@@ -217,10 +217,6 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD"
 endif ()
 
 
-# We will use this for ccache and timing
-set (MY_RULE_LAUNCH "")
-
-
 ###########################################################################
 # Use ccache if found
 #
@@ -229,12 +225,18 @@ set (MY_RULE_LAUNCH "")
 # logic here makes it work even if the user is unaware of ccache. If it's
 # not found on the system, it will simply be silently not used.
 option (USE_CCACHE "Use ccache if found" ON)
-find_program (CCACHE_FOUND ccache)
-if (CCACHE_FOUND AND USE_CCACHE)
+find_program (CCACHE_EXE ccache)
+if (CCACHE_EXE AND USE_CCACHE)
     if (CMAKE_COMPILER_IS_CLANG AND USE_QT AND (NOT DEFINED ENV{CCACHE_CPP2}))
         message (STATUS "Ignoring ccache because clang + Qt + env CCACHE_CPP2 is not set")
     else ()
-        set (MY_RULE_LAUNCH ccache)
+        if (NOT ${CXX_COMPILER_LAUNCHER} MATCHES "ccache")
+            set (CXX_COMPILER_LAUNCHER ${CCACHE_EXR} ${CXX_COMPILER_LAUNCHER})
+        endif ()
+        if (NOT ${C_COMPILER_LAUNCHER} MATCHES "ccache")
+            set (C_COMPILER_LAUNCHER ${CCACHE_EXR} ${C_COMPILER_LAUNCHER})
+        endif ()
+        message (STATUS "ccache enabled: ${CCACHE_EXE}")
     endif ()
 endif ()
 
@@ -247,14 +249,8 @@ endif ()
 # set `-j 1` or CMAKE_BUILD_PARALLEL_LEVEL to 1.
 option (TIME_COMMANDS "Time each compile and link command" OFF)
 if (TIME_COMMANDS)
-    set (MY_RULE_LAUNCH "${CMAKE_COMMAND} -E time ${MY_RULE_LAUNCH}")
-endif ()
-
-
-# Note: This must be after any option that alters MY_RULE_LAUNCH
-if (MY_RULE_LAUNCH)
-    set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${MY_RULE_LAUNCH})
-    set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${MY_RULE_LAUNCH})
+    set (CXX_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E time ${CXX_COMPILER_LAUNCHER})
+    set (C_COMPILER_LAUNCHER ${CMAKE_COMMAND} -E time ${C_COMPILER_LAUNCHER})
 endif ()