From 5dd0905822fd01462ba16128389ebd7dd21e8766 Mon Sep 17 00:00:00 2001 From: evtrouble Date: Thu, 7 Aug 2025 20:50:46 +0800 Subject: [PATCH 1/2] reduce ci performance time --- .github/workflows/build-test.yml | 391 +++++++++++++++++++++++++++++++ .github/workflows/build.yml | 78 ------ .github/workflows/test.yml | 170 -------------- 3 files changed, 391 insertions(+), 248 deletions(-) create mode 100644 .github/workflows/build-test.yml delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 000000000..a3079e2f8 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,391 @@ +name: build + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) + BUILD_TYPE: Release + +jobs: + init-ubuntu: + runs-on: ubuntu-latest + name: init-ubuntu + outputs: + cache-key: ${{ steps.build-cache.outputs.cache-key }} + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Setup build cache + id: build-cache + run: | + echo "cache-key=build-${{ github.sha }}-${{ runner.os }}-${{ github.run_id }}" >> $GITHUB_OUTPUT + + - name: Cache build dependencies + uses: actions/cache@v4 + with: + path: | + deps/3rd/usr/local + build_debug + build_release + key: ${{ steps.build-cache.outputs.cache-key }} + restore-keys: | + build-${{ github.sha }}-${{ runner.os }}- + build-${{ github.sha }}- + build- + + - name: Init + shell: bash + run: sudo bash build.sh init + + build-and-test-on-ubuntu: + runs-on: ubuntu-latest + name: build-and-test-on-ubuntu + needs: init-ubuntu + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Cache build dependencies + uses: actions/cache@v4 + with: + path: | + deps/3rd/usr/local + build_debug + key: ${{ needs.init-ubuntu.outputs.cache-key }} + restore-keys: | + build-${{ github.sha }}-${{ runner.os }}- + build-${{ github.sha }}- + build- + + - name: BuildDebug + shell: bash + run: bash build.sh debug -DCONCURRENCY=ON -DENABLE_COVERAGE=ON -DWITH_BENCHMARK=ON -DWITH_MEMTRACER=ON -DWITH_UNIT_TESTS=ON --make -j4 + + - name: Test + shell: bash + run: | + cd build_debug + ctest -E memtracer_test --verbose + + - name: lcov + shell: bash + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get update && sudo apt-get install -y lcov + cd build_debug + rm -rf unittest + lcov -c -d ./ -o coverage.info --ignore-errors source + + - uses: codecov/codecov-action@v3 + with: + file: build_debug/coverage.info + token: ${{ secrets.CODECOV_TOKEN }} + + build-release: + runs-on: ubuntu-latest + name: build-release-ubuntu + needs: init-ubuntu + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Cache build dependencies + uses: actions/cache@v4 + with: + path: | + deps/3rd/usr/local + build_release + key: ${{ needs.init-ubuntu.outputs.cache-key }} + restore-keys: | + build-${{ github.sha }}-${{ runner.os }}- + build-${{ github.sha }}- + build- + + - name: BuildRelease + shell: bash + run: bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=ON -DWITH_BENCHMARK=ON -DENABLE_ASAN=OFF -DWITH_MEMTRACER=ON --make -j4 + + - name: Upload release artifacts + uses: actions/upload-artifact@v4 + with: + name: build-release-artifacts + path: | + build_release/ + deps/3rd/usr/local/ + retention-days: 7 + + basic-test: + runs-on: ubuntu-latest + needs: build-release + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-release-artifacts + path: ./ + + - name: Fix permissions + shell: bash + run: | + chmod +x build_release/bin/* || true + chmod +x build_release/benchmark/* || true + chmod +x build_release/unittest/* || true + find build_release -name "*.so" -exec chmod +x {} \; || true + find build_release -name "*.a" -exec chmod +x {} \; || true + + - name: run basic test + shell: bash + run: | + echo "begin test..." + python3 test/case/miniob_test.py --test-cases=basic | tail -1 | grep "basic is success" + + sysbench-test: + needs: [build-release] + strategy: + matrix: + include: + # 基础sysbench测试 + - thread_model: 'one-thread-per-connection' + test_case: 'miniob_insert' + build_type: 'release' + memtracer: '' + - thread_model: 'one-thread-per-connection' + test_case: 'miniob_delete' + build_type: 'release' + memtracer: '' + - thread_model: 'one-thread-per-connection' + test_case: 'miniob_select' + build_type: 'release' + memtracer: '' + - thread_model: 'java-thread-pool' + test_case: 'miniob_insert' + build_type: 'release' + memtracer: '' + - thread_model: 'java-thread-pool' + test_case: 'miniob_delete' + build_type: 'release' + memtracer: '' + - thread_model: 'java-thread-pool' + test_case: 'miniob_select' + build_type: 'release' + memtracer: '' + # memtracer sysbench测试 + - thread_model: 'one-thread-per-connection' + test_case: 'miniob_insert' + build_type: 'release' + memtracer: 'LD_PRELOAD=./lib/libmemtracer.so' + + runs-on: ubuntu-latest + name: sysbench-test-${{ matrix.build_type }}-${{ matrix.thread_model }}-${{ matrix.test_case }}-${{ matrix.memtracer == '' && 'normal' || 'memtracer' }} + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.build_type == 'debug' && 'build-debug-artifacts' || 'build-release-artifacts' }} + path: ./ + + - name: Fix permissions + shell: bash + run: | + chmod +x build_${{ matrix.build_type }}/bin/* || true + chmod +x build_${{ matrix.build_type }}/benchmark/* || true + chmod +x build_${{ matrix.build_type }}/unittest/* || true + find build_${{ matrix.build_type }} -name "*.so" -exec chmod +x {} \; || true + find build_${{ matrix.build_type }} -name "*.a" -exec chmod +x {} \; || true + + - name: install sysbench and mariadb-client + shell: bash + run: | + curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh + sudo bash script.deb.sh + sudo apt -y install sysbench mariadb-client + + - name: start server + shell: bash + run: | + nohup sh -c '${{ matrix.memtracer }} ./build_${{ matrix.build_type }}/bin/observer -T ${{ matrix.thread_model }} -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk' & + sleep 10 && echo "wake up" + mysql --version + mysql -S /tmp/miniob.sock -e "show tables" + + - name: sysbench test + shell: bash + run: | + cd test/sysbench + sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} prepare + sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} run || { cat ../../nohup.txt*; exit 1; } + + - name: stop server + shell: bash + run: | + mysql -S /tmp/miniob.sock -e "create table t(id int)" + mysql -S /tmp/miniob.sock -e "show tables" + killall observer + + - name: restart server + shell: bash + run: | + nohup sh -c '${{ matrix.memtracer }} ./build_${{ matrix.build_type }}/bin/observer -T ${{ matrix.thread_model }} -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk' & + sleep 10 && echo "wake up" + mysql -S /tmp/miniob.sock -e "show tables" + + - name: sysbench test again + shell: bash + run: | + cd test/sysbench + sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} run || { cat ../../nohup.txt*; exit 1; } + + benchmark-test: + runs-on: ubuntu-latest + needs: build-release + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-release-artifacts + path: ./ + + - name: Fix permissions + shell: bash + run: | + chmod +x build_release/bin/* || true + chmod +x build_release/benchmark/* || true + chmod +x build_release/unittest/* || true + find build_release -name "*.so" -exec chmod +x {} \; || true + find build_release -name "*.a" -exec chmod +x {} \; || true + + - name: concurrency-test + shell: bash + run: | + cd build_release/bin/ + for file in `find ./ -name "*_concurrency_test" -executable`; do $file; if [ $? -ne 0 ]; then exit 1; fi; done + + memtracer-test: + needs: build-release + strategy: + matrix: + memtracer: ['LD_PRELOAD=./lib/libmemtracer.so', ''] + runs-on: ubuntu-latest + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-release-artifacts + path: ./ + + - name: Fix permissions + shell: bash + run: | + chmod +x build_release/bin/* || true + chmod +x build_release/benchmark/* || true + chmod +x build_release/unittest/* || true + find build_release -name "*.so" -exec chmod +x {} \; || true + find build_release -name "*.a" -exec chmod +x {} \; || true + + - name: memtracer-performance-test + shell: bash + run: | + cd build_release + ${{matrix.memtracer}} ./benchmark/memtracer_performance_test + - name: memtracer-unittest + shell: bash + run: | + cd build_release + LD_PRELOAD=./lib/libmemtracer.so ./unittest/memtracer_test + + integration-test: + needs: build-release + runs-on: ubuntu-latest + steps: + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: build-release-artifacts + path: ./ + + - name: Fix permissions + shell: bash + run: | + chmod +x build_release/bin/* || true + chmod +x build_release/benchmark/* || true + chmod +x build_release/unittest/* || true + find build_release -name "*.so" -exec chmod +x {} \; || true + find build_release -name "*.a" -exec chmod +x {} \; || true + + - name: init mysql + uses: shogo82148/actions-setup-mysql@v1 + with: + mysql-version: "8.0" + - name: init + shell: bash + run: | + sudo apt -y install pip python3-pymysql python3-psutil + + - name: integration test + shell: bash + run: | + cd test/integration_test/ + bash ./miniob_test_docker_entry.sh + python3 ./libminiob_test.py -c conf.ini + + build-on-mac: + runs-on: macos-latest + name: build-macos + + steps: + - name: Init environment + shell: bash + run: | + brew install bison + + - name: Checkout repository and submodules + uses: actions/checkout@v4 + + - name: Setup build cache + id: build-cache + run: | + echo "cache-key=build-${{ github.sha }}-${{ runner.os }}-${{ github.run_id }}" >> $GITHUB_OUTPUT + + - name: Cache build dependencies + uses: actions/cache@v4 + with: + path: | + deps/3rd/usr/local + build_release + build_debug + key: ${{ steps.build-cache.outputs.cache-key }} + restore-keys: | + build-${{ github.sha }}-${{ runner.os }}- + build-${{ github.sha }}- + build- + + - name: Build + shell: bash + run: | + export ASAN_OPTIONS=detect_container_overflow=0 + export PATH="/opt/homebrew/opt/bison/bin:$PATH" + sudo bash build.sh init + bash build.sh debug -DWITH_MEMTRACER=ON --make -j4 + bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF -DWITH_BENCHMARK=ON -DWITH_MEMTRACER=ON --make -j4 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 5f00c4b24..000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: build - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - -jobs: - build-and-test-on-ubuntu: - # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac. - # You can convert this to a matrix build if you need cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: ubuntu-latest - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: Init - shell: bash - run: sudo bash build.sh init - - - name: BuildRelease - shell: bash - run: bash build.sh release --make -j4 - - - name: BuildDebug - shell: bash - run: bash build.sh debug -DCONCURRENCY=ON -DENABLE_COVERAGE=ON --make -j4 - - # `memtracer_test` unittest runs in `memtracer-test` action. - - name: Test - shell: bash - run: | - cd build_debug - ctest -E memtracer_test --verbose - - - name: lcov - shell: bash - run: | - export DEBIAN_FRONTEND=noninteractive - sudo apt-get update && sudo apt-get install -y lcov - cd build_debug - rm -rf unittest - lcov -c -d ./ -o coverage.info --ignore-errors source - - - uses: codecov/codecov-action@v3 - with: - file: build_debug/coverage.info - token: ${{ secrets.CODECOV_TOKEN }} - - build-on-mac: - # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac. - # You can convert this to a matrix build if you need cross-platform coverage. - # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix - runs-on: macos-latest - - steps: - - name: Init environment - shell: bash - run: | - brew install bison - - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: Build - shell: bash - run: | - export ASAN_OPTIONS=detect_container_overflow=0 - export PATH="/opt/homebrew/opt/bison/bin:$PATH" - sudo bash build.sh init - bash build.sh release --make -j4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 01beccb47..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,170 +0,0 @@ -name: test - -on: - pull_request: - branches: [ "main" ] - -env: - # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - -jobs: - basic-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: run basic test - shell: bash - run: | - sudo bash build.sh init - echo "begin test..." - python3 test/case/miniob_test.py --test-cases=basic | tail -1 | grep "basic is success" - - # I found that sysbench would send more request before receiving last response - sysbench-test: - strategy: - matrix: - thread_model: ['one-thread-per-connection', 'java-thread-pool'] - test_case: ['miniob_insert', 'miniob_delete', 'miniob_select'] - - runs-on: ubuntu-latest - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: install sysbench and mariadb-client - shell: bash - run: | - curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh - sudo bash script.deb.sh - sudo apt -y install sysbench mariadb-client - - - name: build miniob - shell: bash - run: | - sudo bash build.sh init - bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF - - - name: start server - shell: bash - run: | - nohup ./build_release/bin/observer -T ${{ matrix.thread_model }} -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk 2>&1 & - sleep 10 && echo "wake up" - mysql --version - mysql -S /tmp/miniob.sock -e "show tables" - - # error number 41 is LOCKED_CONCURRENCY_CONFLICT - # we should change the error number if we update the code - - name: sysbench test - shell: bash - run: | - cd test/sysbench - sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} prepare - sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} run || { cat ../../nohup.txt*; exit 1; } - - - name: stop server - shell: bash - run: | - mysql -S /tmp/miniob.sock -e "create table t(id int)" - mysql -S /tmp/miniob.sock -e "show tables" - killall observer - - - name: restart server - shell: bash - run: | - nohup ./build_release/bin/observer -T ${{ matrix.thread_model }} -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk 2>&1 & - sleep 10 && echo "wake up" - mysql -S /tmp/miniob.sock -e "show tables" - - - name: sysbench test again - shell: bash - run: | - cd test/sysbench - sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} run || { cat ../../nohup.txt*; exit 1; } - - benchmark-test: - runs-on: ubuntu-latest - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: build observer and benchmark - shell: bash - run: | - sudo bash build.sh init - bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF -DWITH_BENCHMARK=ON - - - name: concurrency-test - shell: bash - run: | - cd build_release/bin/ - for file in `find ./ -name "*_concurrency_test" -executable`; do $file; if [ $? -ne 0 ]; then exit 1; fi; done - memtracer-test: - strategy: - matrix: - memtracer: ['LD_PRELOAD=./lib/libmemtracer.so', ''] - runs-on: ubuntu-latest - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: build - shell: bash - run: | - sudo bash build.sh init - bash build.sh release -DWITH_BENCHMARK=ON -DENABLE_ASAN=OFF -DCONCURRENCY=ON -DWITH_MEMTRACER=ON - - - name: memtracer-performance-test - shell: bash - run: | - cd build_release - ${{matrix.memtracer}} ./benchmark/memtracer_performance_test - - name: memtracer-unittest - shell: bash - run: | - cd build_release - LD_PRELOAD=./lib/libmemtracer.so ./unittest/memtracer_test - - name: memtracer-sysbench - shell: bash - run: | - curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh - sudo bash script.deb.sh - sudo apt -y install sysbench mariadb-client - nohup sh -c '${{matrix.memtracer}} ./build_release/bin/observer -T one-thread-per-connection -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk' & - sleep 10 && echo "wake up" - mysql --version - mysql -S /tmp/miniob.sock -e "show tables" - cd test/sysbench - sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 miniob_insert prepare - sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 miniob_insert run - killall observer - cd ../.. - nohup ./build_release/bin/observer -T one-thread-per-connection -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk & - sleep 10 && echo "wake up" - mysql -S /tmp/miniob.sock -e "show tables" - integration-test: - runs-on: ubuntu-latest - steps: - - name: Checkout repository and submodules - uses: actions/checkout@v4 - - - name: init mysql - uses: shogo82148/actions-setup-mysql@v1 - with: - mysql-version: "8.0" - - name: init - shell: bash - run: | - sudo bash build.sh init - sudo apt -y install pip python3-pymysql python3-psutil - - - name: integration test - shell: bash - run: | - cd test/integration_test/ - bash ./miniob_test_docker_entry.sh - python3 ./libminiob_test.py -c conf.ini - From 0b0a027769028c678ae4903697ff64d4ba808c00 Mon Sep 17 00:00:00 2001 From: evtrouble Date: Fri, 8 Aug 2025 14:21:02 +0800 Subject: [PATCH 2/2] Fix the issue where subsequent jobs are unable to obtain pre jobs and upload them to the cache --- .github/workflows/build-test.yml | 70 ++++++++++++++++---------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index a3079e2f8..afcb122ea 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -10,6 +10,10 @@ env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) BUILD_TYPE: Release +permissions: + contents: read + actions: write + jobs: init-ubuntu: runs-on: ubuntu-latest @@ -23,8 +27,7 @@ jobs: - name: Setup build cache id: build-cache run: | - echo "cache-key=build-${{ github.sha }}-${{ runner.os }}-${{ github.run_id }}" >> $GITHUB_OUTPUT - + echo "cache-key=build-${{ runner.os }}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**', 'deps/3rd/**', 'build.sh') }}" >> $GITHUB_OUTPUT - name: Cache build dependencies uses: actions/cache@v4 with: @@ -34,13 +37,17 @@ jobs: build_release key: ${{ steps.build-cache.outputs.cache-key }} restore-keys: | - build-${{ github.sha }}-${{ runner.os }}- - build-${{ github.sha }}- + build-${{ runner.os }}- build- - - name: Init shell: bash run: sudo bash build.sh init + - name: Upload deps artifact (fallback for fork PR) + uses: actions/upload-artifact@v4 + with: + name: deps-${{ runner.os }} + path: deps/3rd/usr/local + retention-days: 7 build-and-test-on-ubuntu: runs-on: ubuntu-latest @@ -52,6 +59,7 @@ jobs: uses: actions/checkout@v4 - name: Cache build dependencies + id: cache uses: actions/cache@v4 with: path: | @@ -59,10 +67,14 @@ jobs: build_debug key: ${{ needs.init-ubuntu.outputs.cache-key }} restore-keys: | - build-${{ github.sha }}-${{ runner.os }}- - build-${{ github.sha }}- + build-${{ runner.os }}- build- - + - name: Download deps artifact if cache missed + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/download-artifact@v4 + with: + name: deps-${{ runner.os }} + path: deps/3rd/usr/local - name: BuildDebug shell: bash run: bash build.sh debug -DCONCURRENCY=ON -DENABLE_COVERAGE=ON -DWITH_BENCHMARK=ON -DWITH_MEMTRACER=ON -DWITH_UNIT_TESTS=ON --make -j4 @@ -72,7 +84,6 @@ jobs: run: | cd build_debug ctest -E memtracer_test --verbose - - name: lcov shell: bash run: | @@ -81,7 +92,6 @@ jobs: cd build_debug rm -rf unittest lcov -c -d ./ -o coverage.info --ignore-errors source - - uses: codecov/codecov-action@v3 with: file: build_debug/coverage.info @@ -97,6 +107,7 @@ jobs: uses: actions/checkout@v4 - name: Cache build dependencies + id: cache uses: actions/cache@v4 with: path: | @@ -104,10 +115,14 @@ jobs: build_release key: ${{ needs.init-ubuntu.outputs.cache-key }} restore-keys: | - build-${{ github.sha }}-${{ runner.os }}- - build-${{ github.sha }}- + build-${{ runner.os }}- build- - + - name: Download deps artifact if cache missed + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/download-artifact@v4 + with: + name: deps-${{ runner.os }} + path: deps/3rd/usr/local - name: BuildRelease shell: bash run: bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=ON -DWITH_BENCHMARK=ON -DENABLE_ASAN=OFF -DWITH_MEMTRACER=ON --make -j4 @@ -142,13 +157,11 @@ jobs: chmod +x build_release/unittest/* || true find build_release -name "*.so" -exec chmod +x {} \; || true find build_release -name "*.a" -exec chmod +x {} \; || true - - name: run basic test shell: bash run: | echo "begin test..." python3 test/case/miniob_test.py --test-cases=basic | tail -1 | grep "basic is success" - sysbench-test: needs: [build-release] strategy: @@ -205,14 +218,12 @@ jobs: chmod +x build_${{ matrix.build_type }}/unittest/* || true find build_${{ matrix.build_type }} -name "*.so" -exec chmod +x {} \; || true find build_${{ matrix.build_type }} -name "*.a" -exec chmod +x {} \; || true - - name: install sysbench and mariadb-client shell: bash run: | curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh sudo bash script.deb.sh sudo apt -y install sysbench mariadb-client - - name: start server shell: bash run: | @@ -220,41 +231,36 @@ jobs: sleep 10 && echo "wake up" mysql --version mysql -S /tmp/miniob.sock -e "show tables" - - name: sysbench test shell: bash run: | cd test/sysbench sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} prepare sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} run || { cat ../../nohup.txt*; exit 1; } - - name: stop server shell: bash run: | mysql -S /tmp/miniob.sock -e "create table t(id int)" mysql -S /tmp/miniob.sock -e "show tables" killall observer - - name: restart server shell: bash run: | nohup sh -c '${{ matrix.memtracer }} ./build_${{ matrix.build_type }}/bin/observer -T ${{ matrix.thread_model }} -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc -d disk' & sleep 10 && echo "wake up" mysql -S /tmp/miniob.sock -e "show tables" - - name: sysbench test again shell: bash run: | cd test/sysbench sysbench --mysql-socket=/tmp/miniob.sock --mysql-ignore-errors=41 --threads=10 ${{ matrix.test_case }} run || { cat ../../nohup.txt*; exit 1; } - benchmark-test: runs-on: ubuntu-latest needs: build-release steps: - name: Checkout repository and submodules uses: actions/checkout@v4 - + - name: Download build artifacts uses: actions/download-artifact@v4 with: @@ -269,13 +275,11 @@ jobs: chmod +x build_release/unittest/* || true find build_release -name "*.so" -exec chmod +x {} \; || true find build_release -name "*.a" -exec chmod +x {} \; || true - - name: concurrency-test shell: bash run: | cd build_release/bin/ for file in `find ./ -name "*_concurrency_test" -executable`; do $file; if [ $? -ne 0 ]; then exit 1; fi; done - memtracer-test: needs: build-release strategy: @@ -300,7 +304,6 @@ jobs: chmod +x build_release/unittest/* || true find build_release -name "*.so" -exec chmod +x {} \; || true find build_release -name "*.a" -exec chmod +x {} \; || true - - name: memtracer-performance-test shell: bash run: | @@ -311,7 +314,6 @@ jobs: run: | cd build_release LD_PRELOAD=./lib/libmemtracer.so ./unittest/memtracer_test - integration-test: needs: build-release runs-on: ubuntu-latest @@ -342,14 +344,12 @@ jobs: shell: bash run: | sudo apt -y install pip python3-pymysql python3-psutil - - name: integration test shell: bash run: | cd test/integration_test/ bash ./miniob_test_docker_entry.sh python3 ./libminiob_test.py -c conf.ini - build-on-mac: runs-on: macos-latest name: build-macos @@ -359,16 +359,15 @@ jobs: shell: bash run: | brew install bison - - name: Checkout repository and submodules uses: actions/checkout@v4 - name: Setup build cache id: build-cache run: | - echo "cache-key=build-${{ github.sha }}-${{ runner.os }}-${{ github.run_id }}" >> $GITHUB_OUTPUT - + echo "cache-key=build-${{ runner.os }}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**', 'deps/3rd/**', 'build.sh') }}" >> $GITHUB_OUTPUT - name: Cache build dependencies + id: cache uses: actions/cache@v4 with: path: | @@ -377,10 +376,9 @@ jobs: build_debug key: ${{ steps.build-cache.outputs.cache-key }} restore-keys: | - build-${{ github.sha }}-${{ runner.os }}- - build-${{ github.sha }}- + build-${{ runner.os }}- build- - + - name: Build shell: bash run: | @@ -388,4 +386,4 @@ jobs: export PATH="/opt/homebrew/opt/bison/bin:$PATH" sudo bash build.sh init bash build.sh debug -DWITH_MEMTRACER=ON --make -j4 - bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF -DWITH_BENCHMARK=ON -DWITH_MEMTRACER=ON --make -j4 + bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF -DWITH_BENCHMARK=ON -DWITH_MEMTRACER=ON --make -j4 \ No newline at end of file