Update Tensorflow to 2.16.1 (#805)

* Update TensorFlow to v2.16.1

* Update Bazel and bazelrc file

* Update WORKSPACE with TF changes

Uses hermetic Python

* Update MLIR build target paths

* Rename `--remote_http_cache` to `--remote_cache`

Disable read-only cache on PR

* Add explicit Python package requirements

* Remove `pip install` because of Hermetic Python

* Update filecheck test patterns

* Replace `llvm::Optional` by `std::optional`

* TF_USE_LEGACY_KERAS in bazel

* Update Android NDK configuration in `configure.py`

* Update `build_lce_aar.sh`

* Update MLIR passes

* Fix release macOS runners

* Fix artifact v4 actions

* Simplify upload-wheels job

* Run `pip-compile` in Windows release

* Add `disable_forced_mkl.patch`

* Use Github actions temp disk as bazel root

* Use an even shorter bazel root dir on Windows

* Shorten workspace name on Windows

* Add `addBenefit` for quantization pattern

---------

Co-authored-by: Tom Bannink <tombannink@gmail.com>

Author: CNugteren

.bazelversion

@@ -1 +1 @@
-5.3.0
+6.5.0

.github/tools/release_linux.sh

@@ -3,13 +3,17 @@ set -e -x
 
 python configure.py
 
-# Build
-bazel build :build_pip_pkg \
+# Inside the docker container on github actions there is not
+# enough space for the bazel cache, but a larger disk is mounted at /github_disk
+# so we tell bazel to store everything there
+
+# `release_cpu_linux` will activate absolute paths to files that only exist in the tensorflow/build:2.16-pythonXX docker container
+bazel --output_user_root=/github_disk/bazel_root \
+  build :build_pip_pkg \
+  -c opt \
+  --config=release_cpu_linux \
   --copt=-fvisibility=hidden \
-  --copt=-mavx \
-  --distinct_host_configuration=false \
-  --verbose_failures \
-  --crosstool_top=@ubuntu20.04-gcc9_manylinux2014-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain
+  --verbose_failures
 
 # Package Whl
 bazel-bin/build_pip_pkg artifacts

.github/workflows/release.yml

@@ -36,7 +36,7 @@ jobs:
         if: steps.cache.outputs.cache-hit != 'true'
         run: ./third_party/install_android.sh
       - name: Configure Bazel
-        run: LCE_SET_ANDROID_WORKSPACE=1 ANDROID_SDK_HOME="/tmp/lce_android" ANDROID_NDK_HOME="/tmp/lce_android/ndk/21.4.7075529" ./configure.py
+        run: LCE_SET_ANDROID_WORKSPACE=1 ANDROID_SDK_HOME="/tmp/lce_android" ANDROID_API_LEVEL=30 ANDROID_NDK_HOME="/tmp/lce_android/ndk/25.2.9519653" ANDROID_NDK_API_LEVEL=30 ANDROID_BUILD_TOOLS_VERSION=31.0.0 ./configure.py
         shell: bash
       - run: mkdir benchmark-binaries
       - name: Build Benchmark utility for AArch64
@@ -108,7 +108,7 @@ jobs:
         if: steps.cache.outputs.cache-hit != 'true'
         run: ./third_party/install_android.sh
       - name: Configure Bazel
-        run: LCE_SET_ANDROID_WORKSPACE=1 ANDROID_SDK_HOME="/tmp/lce_android" ANDROID_NDK_HOME="/tmp/lce_android/ndk/21.4.7075529" ./configure.py
+        run: LCE_SET_ANDROID_WORKSPACE=1 ANDROID_SDK_HOME="/tmp/lce_android" ANDROID_API_LEVEL=30 ANDROID_NDK_HOME="/tmp/lce_android/ndk/25.2.9519653" ANDROID_NDK_API_LEVEL=30 ANDROID_BUILD_TOOLS_VERSION=31.0.0 ./configure.py
         shell: bash
       - name: Build LCE AAR
         run: BUILDER=bazelisk ./larq_compute_engine/tflite/java/build_lce_aar.sh
@@ -134,10 +134,10 @@ jobs:
 
   macos-release-wheel:
     name: Build release wheels for macOS
-    runs-on: macos-latest
+    runs-on: macos-13
     strategy:
       matrix:
-        python-version: [3.9, "3.10", 3.11]
+        python-version: ["3.10", 3.11]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -154,33 +154,35 @@ jobs:
           python -m pip install delocate wheel setuptools numpy six --no-cache-dir
 
           ./configure.py
-          export MACOSX_DEPLOYMENT_TARGET=10.14
+          # This matches `release_macox_x86` in .tensorflow.bazelrc
+          export MACOSX_DEPLOYMENT_TARGET=10.15
 
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
-            echo -e 'build --remote_http_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-macos-python${{ matrix.python-version }}' >> .bazelrc.user
+            echo -e 'build --remote_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-macos-python${{ matrix.python-version }}' >> .bazelrc.user
             echo -e 'build --google_default_credentials' >> .bazelrc.user
           fi
 
-          bazelisk build :build_pip_pkg --copt=-fvisibility=hidden --copt=-mavx --linkopt=-dead_strip --distinct_host_configuration=false
-          bazel-bin/build_pip_pkg artifacts --plat-name macosx_10_14_x86_64
+          bazelisk build :build_pip_pkg --config=release_macos_x86 --config=release_cpu_macos --copt=-fvisibility=hidden --linkopt=-dead_strip
+          bazel-bin/build_pip_pkg artifacts --plat-name macosx_10_15_x86_64
 
           for f in artifacts/*.whl; do
             delocate-wheel -w wheelhouse $f
           done
         env:
           LCE_RELEASE_VERSION: ${{ github.event.inputs.version }}
+          TF_PYTHON_VERSION: ${{ matrix.python-version }}
         shell: bash
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ runner.os }}-wheels
+          name: ${{ runner.os }}-wheels-${{ matrix.python-version }}
           path: wheelhouse
 
   macos-arm-release-wheel:
     name: Build release arm wheels for macOS
-    runs-on: macos-11
+    runs-on: macos-14
     strategy:
       matrix:
-        python-version: [3.9, "3.10", 3.11]
+        python-version: ["3.10", 3.11]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -197,33 +199,35 @@ jobs:
           python -m pip install delocate wheel setuptools numpy six --no-cache-dir
 
           ./configure.py
-          export MACOSX_DEPLOYMENT_TARGET=11.0
+          # This matches `release_macox_arm64` in .tensorflow.bazelrc
+          export MACOSX_DEPLOYMENT_TARGET=12.0
 
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
-            echo -e 'build --remote_http_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-macos-arm-python${{ matrix.python-version }}' >> .bazelrc.user
+            echo -e 'build --remote_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-macos-arm-python${{ matrix.python-version }}' >> .bazelrc.user
             echo -e 'build --google_default_credentials' >> .bazelrc.user
           fi
 
-          bazelisk build :build_pip_pkg --copt=-fvisibility=hidden --linkopt=-dead_strip --config=macos_arm64
-          bazel-bin/build_pip_pkg artifacts --plat-name macosx_11_0_arm64
+          bazelisk build :build_pip_pkg --config=release_macos_arm64 --copt=-fvisibility=hidden --linkopt=-dead_strip
+          bazel-bin/build_pip_pkg artifacts --plat-name macosx_12_0_arm64
 
           for f in artifacts/*.whl; do
             delocate-wheel -w wheelhouse $f
           done
         env:
           LCE_RELEASE_VERSION: ${{ github.event.inputs.version }}
+          TF_PYTHON_VERSION: ${{ matrix.python-version }}
         shell: bash
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ runner.os }}-arm-wheels
+          name: ${{ runner.os }}-arm-wheels-${{ matrix.python-version }}
           path: wheelhouse
 
   manylinux-release-wheel:
     name: Build release wheels for manylinux2014
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9, "3.10", 3.11]
+        python-version: ["3.10", 3.11]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -237,15 +241,17 @@ jobs:
       - name: Build manylinux2014 wheels
         run: |
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
-            echo -e 'build --remote_http_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-manylinux-python${{ matrix.python-version }}' >> .bazelrc.user
+            echo -e 'build --remote_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-manylinux-python${{ matrix.python-version }}' >> .bazelrc.user
             echo -e 'build --google_default_credentials' >> .bazelrc.user
           fi
 
           docker run -e LCE_RELEASE_VERSION=${{ github.event.inputs.version }} \
             -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-credentials.json \
+            -e TF_PYTHON_VERSION=${{ matrix.python-version }} \
             -v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/gcloud-credentials.json:ro \
             -v ${PWD}:/compute-engine -w /compute-engine \
-            tensorflow/build:2.13-python${{ matrix.python-version }} \
+            -v /mnt:/github_disk \
+            tensorflow/build:2.16-python${{ matrix.python-version }} \
             .github/tools/release_linux.sh
 
           sudo apt-get -y -qq install patchelf --no-install-recommends
@@ -258,15 +264,15 @@ jobs:
           ls -al wheelhouse/
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ runner.os }}-wheels
+          name: ${{ runner.os }}-wheels-${{ matrix.python-version }}
           path: wheelhouse
 
   windows-release-wheel:
     name: Build release wheels for Windows
     runs-on: windows-2019
     strategy:
       matrix:
-        python-version: [3.9, "3.10", 3.11]
+        python-version: ["3.10", 3.11]
       fail-fast: false
     steps:
       - name: Configure Pagefile
@@ -294,18 +300,24 @@ jobs:
           $Env:CC_OPT_FLAGS = "/O2"
 
           python --version
-          python -m pip install wheel setuptools numpy six --no-cache-dir
+          python -m pip install wheel setuptools numpy six pip-tools --no-cache-dir
+          # This is needed because the requirements on windows are different than on other systems
+          pip-compile --strip-extras --no-emit-index-url --allow-unsafe larq_compute_engine/requirements.in
+
+          # Fix for path length limit: replace workspace name by 'lce'
+          (Get-Content WORKSPACE).Replace('workspace(name = "larq_compute_engine")', 'workspace(name = "lce")') | Set-Content WORKSPACE
 
           "" | python configure.py
 
-          bazelisk --output_base=C:\build_output build :build_pip_pkg --enable_runfiles --local_ram_resources=4096 --remote_http_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-windows-python${{ matrix.python-version }} --google_default_credentials
+          bazelisk --output_base=C:\bzl build :build_pip_pkg --enable_runfiles --local_ram_resources=4096 --remote_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-release-windows-python${{ matrix.python-version }} --google_default_credentials
           bazel-bin/build_pip_pkg wheelhouse
         env:
           LCE_RELEASE_VERSION: ${{ github.event.inputs.version }}
+          TF_PYTHON_VERSION: ${{ matrix.python-version }}
         shell: pwsh
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ runner.os }}-wheels
+          name: ${{ runner.os }}-wheels-${{ matrix.python-version }}
           path: wheelhouse
 
   upload-wheels:
@@ -322,31 +334,11 @@ jobs:
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: Linux-wheels
-          path: Linux-wheels
-        if: ${{ needs.manylinux-release-wheel.result == 'success' }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: macOS-wheels
-          path: macOS-wheels
-        if: ${{ needs.macos-release-wheel.result == 'success' }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: macOS-arm-wheels
-          path: macOS-arm-wheels
-        if: ${{ needs.macos-arm-release-wheel.result == 'success' }}
-      - uses: actions/download-artifact@v4
-        with:
-          name: Windows-wheels
-          path: Windows-wheels
-        if: ${{ needs.windows-release-wheel.result == 'success' }}
+          pattern: "*wheels*"
+          path: dist
+          merge-multiple: true
       - run: |
           set -x
-          mkdir -p dist
-          cp Linux-wheels/*.whl dist/ || true
-          cp macOS-wheels/*.whl dist/ || true
-          cp macOS-arm-wheels/*.whl dist/ || true
-          cp Windows-wheels/*.whl dist/ || true
           ls -la dist/
           sha256sum dist/*.whl
       - uses: pypa/gh-action-pypi-publish@master

.github/workflows/unittests.yml

@@ -10,6 +10,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  TF_PYTHON_VERSION: "3.11"
+
 jobs:
   TFLite:
     runs-on: ubuntu-latest
@@ -19,16 +22,8 @@ jobs:
         with:
           submodules: true
           fetch-depth: 0
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-      - name: Configure Bazel
-        run: ./configure.py
-        shell: bash
-      - name: Install pip dependencies
-        run: pip install numpy --no-cache-dir
       - name: Run C++ Unit Tests
-        run: bazelisk test larq_compute_engine/tests:cc_tests --distinct_host_configuration=false --test_output=all
+        run: bazelisk test larq_compute_engine/tests:cc_tests --test_output=all
       - name: Build TF Lite Static Library with CMake
         run: |
           mkdir build
@@ -45,14 +40,6 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends qemu-user
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-      - name: Configure Bazel
-        run: ./configure.py
-        shell: bash
-      - name: Install pip dependencies
-        run: pip install numpy six --no-cache-dir
       - name: "TF Lite Arm32: Cross-compile and run unit tests in qemu"
         run: bazelisk test larq_compute_engine/tests:arm32_tests --config=rpi3 --test_output=all --test_filter="-*BigTest*" --copt=-O1
       - name: "TF Lite Aarch64: Cross-compile and run unit tests in qemu"
@@ -69,32 +56,21 @@ jobs:
         continue-on-error: true
         with:
           credentials_json: ${{ secrets.gcs_bazel_cache }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
       - name: Configure Bazel
         run: |
-          ./configure.py
-          echo -e 'build --distinct_host_configuration=false' >> .bazelrc.user
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
-            echo -e 'build --remote_http_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-ubuntu' >> .bazelrc.user
+            echo -e 'build --remote_cache=https://storage.googleapis.com/plumerai-bazel-cache/lce-ubuntu' >> .bazelrc.user
             echo -e 'build --google_default_credentials' >> .bazelrc.user
           fi
         shell: bash
-      - name: Set bazel cache
-        run: echo -e 'build --remote_upload_local_results=false' >> .bazelrc.user
-        if: github.ref != 'refs/heads/main'
-        shell: bash
-      - name: Install pip dependencies
-        run: pip install tensorflow-cpu~=2.13.0 larq~=0.13 pytest tensorflow_datasets~=4.9 flatbuffers==23.1.21 tqdm --no-cache-dir
       - name: Run Interpreter test
         run: bazelisk test larq_compute_engine/tflite/tests:interpreter_test --test_output=all
       - name: Run FileCheck tests
         run: bazelisk test larq_compute_engine/mlir/tests:all --test_output=all
       - name: Run End2End tests
-        run: bazelisk test larq_compute_engine/tests:end2end_test --test_output=all
+        run: bazelisk test larq_compute_engine/tests:end2end_test --test_output=all --test_env=TF_USE_LEGACY_KERAS=1
       - name: Run Strip dequantize op tests
-        run: bazelisk test larq_compute_engine/tests:strip_lcedequantize_test --test_output=all
+        run: bazelisk test larq_compute_engine/tests:strip_lcedequantize_test --test_output=all --test_env=TF_USE_LEGACY_KERAS=1
 
   ConverterPython:
     runs-on: ubuntu-latest
@@ -109,6 +85,10 @@ jobs:
             python-version: 3.11
             flatbuffers-version: 23.1.21
             protobuf-version: 4.23.4
+          - tf-version: 2.16.1
+            python-version: 3.11
+            flatbuffers-version: 24.3.25
+            protobuf-version: 4.25.3
     if: "!contains(github.event.head_commit.message, 'ci-skip')"
     steps:
       - uses: actions/checkout@v4
@@ -117,14 +97,17 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install TensorFlow
         run: pip install tensorflow==${{matrix.tf-version}} --no-cache-dir
+      - name: Install legacy tf-keras
+        if: matrix.tf-version == '2.16.1'
+        run: pip install tf-keras==2.16.0
       - name: Install flatbuffers
         run: pip install flatbuffers==${{matrix.flatbuffers-version}} --no-cache-dir
       - name: Install protobuf
         run: pip install protobuf==${{matrix.protobuf-version}} --no-cache-dir
       - name: Install other dependencies
         run: pip install larq~=0.13.3 packaging tqdm --no-cache-dir
       - name: Run Converter test
-        run: PYTHONPATH=./ python larq_compute_engine/mlir/python/converter_test.py
+        run: TF_USE_LEGACY_KERAS=1 PYTHONPATH=./ python larq_compute_engine/mlir/python/converter_test.py
 
   Android_AAR:
     runs-on: ubuntu-latest
@@ -142,15 +125,13 @@ jobs:
         with:
           path: /tmp/lce_android
           key: ${{ runner.os }}-${{ hashFiles('**/third_party/install_android.sh') }}
-      - name: Install pip dependencies
-        run: pip install numpy six --no-cache-dir
       - name: Set Java version
         run: echo "JAVA_HOME=${JAVA_HOME_8_X64}" >> $GITHUB_ENV
       - name: Download and install Android NDK/SDK
         if: steps.cache.outputs.cache-hit != 'true'
         run: ./third_party/install_android.sh
       - name: Configure Bazel
-        run: LCE_SET_ANDROID_WORKSPACE=1 ANDROID_SDK_HOME="/tmp/lce_android" ANDROID_NDK_HOME="/tmp/lce_android/ndk/21.4.7075529" ./configure.py
+        run: LCE_SET_ANDROID_WORKSPACE=1 ANDROID_SDK_HOME="/tmp/lce_android" ANDROID_API_LEVEL=30 ANDROID_NDK_HOME="/tmp/lce_android/ndk/25.2.9519653" ANDROID_NDK_API_LEVEL=30 ANDROID_BUILD_TOOLS_VERSION=31.0.0 ./configure.py
         shell: bash
       - name: Build LCE AAR
         run: BUILDER=bazelisk ./larq_compute_engine/tflite/java/build_lce_aar.sh