diff --git a/.asf.yaml b/.asf.yaml index 16e358f62f..23fccc71e8 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -24,7 +24,9 @@ github: squash: true merge: false rebase: false - autolink_jira: RATIS + autolink_jira: + - HDDS + - RATIS notifications: commits: commits@ratis.apache.org diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml new file mode 100644 index 0000000000..1655d60148 --- /dev/null +++ b/.github/workflows/check.yaml @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This reusable workflow executes a single check from `dev-support/checks/`. +# Before and after the check, it performs various steps based on workflow inputs. + +name: ci-check + +on: + workflow_call: + inputs: + # REQUIRED + script: + type: string + description: "Test script to run from dev-support/checks, without .sh extension" + required: true + + # OPTIONAL (ordered alphabetically) + java-version: + type: string + description: "Java version to set up (default: 8)" + default: '8' + required: false + + needs-binary-tarball: + type: boolean + description: "Whether to download Ratis binary tarball created by build (default: no)" + default: false + required: false + + needs-maven-repo: + type: boolean + description: "Whether to download Ratis jars created by build (default: no)" + default: false + required: false + + needs-source-tarball: + type: boolean + description: "Whether to download Ratis source tarball created by build (default: no)" + default: false + required: false + + runner: + type: string + description: "GitHub Actions runner to use" + default: 'ubuntu-24.04' + required: false + + script-args: + type: string + description: "Arguments for the test script" + default: '' + required: false + + split: + type: string + description: "Name of split for matrix jobs, only used in display name" + default: '' + required: false + + timeout-minutes: + type: number + description: "Job timeout in minutes (default: 30)" + default: 30 + required: false + +env: + MAVEN_ARGS: --batch-mode --show-version + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + WITH_COVERAGE: ${{ github.event_name == 'push' }} + +jobs: + check: + name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - name: Checkout project + if: ${{ !inputs.needs-source-tarball }} + uses: actions/checkout@v4 + + - name: Download source tarball + if: ${{ inputs.needs-source-tarball }} + uses: actions/download-artifact@v4 + with: + name: ratis-src + + - name: Extract source tarball + if: ${{ inputs.needs-source-tarball }} + run: | + tar --strip-components 1 -xzvf ratis*-src.tar.gz + + - name: Create cache for Maven dependencies + if: ${{ inputs.script == 'build' }} + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ratis + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Restore cache for Maven dependencies + if: ${{ inputs.script != 'build' }} + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ratis + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Download Maven repo + id: download-maven-repo + if: ${{ inputs.needs-maven-repo }} + uses: actions/download-artifact@v4 + with: + name: maven-repo + path: | + ~/.m2/repository/org/apache/ratis + + - name: Download binary tarball + if: ${{ inputs.needs-binary-tarball }} + uses: actions/download-artifact@v4 + with: + name: ratis-bin + + - name: Extract binary tarball + if: ${{ inputs.needs-binary-tarball }} + run: | + mkdir -p ratis-assembly/target + tar xzvf ratis-*-bin.tar.gz -C ratis-assembly/target + + - name: Setup java ${{ inputs.java-version }} + if: ${{ inputs.java-version }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + + - name: Execute tests + run: | + dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }} + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + + - name: Summary of failures + if: ${{ failure() }} + run: | + if [[ -s "target/${{ inputs.script }}/summary.txt" ]]; then + cat target/${{ inputs.script }}/summary.txt + fi + + - name: Archive build results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }} + path: target/${{ inputs.script }} + continue-on-error: true + + # The following steps are hard-coded to be run only for 'build' check, + # to avoid the need for 3 more inputs. + - name: Store binaries for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ratis-bin + path: | + ratis-assembly/target/ratis-assembly-*-bin.tar.gz + retention-days: 1 + + - name: Store source tarball for compilation + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ratis-src + path: | + ratis-assembly/target/ratis-assembly-*-src.tar.gz + retention-days: 1 + + - name: Store Maven repo for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: maven-repo + path: | + ~/.m2/repository/org/apache/ratis + retention-days: 1 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..20704f5ea9 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,150 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CI + +on: + workflow_call: + inputs: + ref: + type: string + description: Ratis git ref (branch, tag or commit hash) + default: '' + required: false + +jobs: + build: + uses: ./.github/workflows/check.yaml + with: + script: build + script-args: -Prelease + timeout-minutes: 30 + secrets: inherit + + compile: + needs: + - build + strategy: + matrix: + java: [ 11, 17, 21 ] + fail-fast: false + uses: ./.github/workflows/check.yaml + with: + java-version: ${{ matrix.java }} + needs-source-tarball: true + script: compile + script-args: -Djavac.version=${{ matrix.java }} + split: ${{ matrix.java }} + timeout-minutes: 30 + secrets: inherit + + release: + uses: ./.github/workflows/check.yaml + with: + script: release + timeout-minutes: 30 + secrets: inherit + + repro: + needs: + - build + uses: ./.github/workflows/check.yaml + with: + needs-maven-repo: true + script: repro + script-args: -Prelease + timeout-minutes: 30 + secrets: inherit + + basic: + strategy: + matrix: + check: + - author + - checkstyle + - findbugs + - rat + fail-fast: false + uses: ./.github/workflows/check.yaml + with: + script: ${{ matrix.check }} + timeout-minutes: 30 + secrets: inherit + + unit: + strategy: + matrix: + profile: + - grpc + - server + - misc + - flaky + fail-fast: false + uses: ./.github/workflows/check.yaml + with: + script: unit + script-args: -P${{ matrix.profile }}-tests + split: ${{ matrix.profile }} + timeout-minutes: 60 + secrets: inherit + + coverage: + needs: + - build + - unit + runs-on: ubuntu-24.04 + timeout-minutes: 30 + if: github.event_name != 'pull_request' + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository + !~/.m2/repository/org/apache/ratis + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + - name: Setup java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: target/artifacts + - name: Untar binaries + run: | + mkdir -p ratis-assembly/target + tar xzvf target/artifacts/ratis-bin/ratis-assembly-*.tar.gz -C ratis-assembly/target + - name: Calculate combined coverage + run: ./dev-support/checks/coverage.sh + - name: Upload coverage to Sonar + if: github.repository == 'apache/ratis' + run: ./dev-support/checks/sonar.sh + env: + SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Archive build results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ github.job }} + path: target/${{ github.job }} diff --git a/.github/workflows/post-commit.yaml b/.github/workflows/post-commit.yaml new file mode 100644 index 0000000000..583eb7bcf5 --- /dev/null +++ b/.github/workflows/post-commit.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: build-branch + +on: + push: + branches-ignore: + - 'dependabot/**' + tags: + - '**' + pull_request: + +concurrency: + group: ci-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + CI: + uses: ./.github/workflows/ci.yaml + secrets: inherit diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml deleted file mode 100644 index 9978caacf8..0000000000 --- a/.github/workflows/post-commit.yml +++ /dev/null @@ -1,265 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: build-branch -on: - - push - - pull_request -env: - WITH_COVERAGE: true -jobs: - build: - runs-on: ubuntu-20.04 - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 8 - - name: Run a full build - run: ./dev-support/checks/build.sh -Prelease assembly:single - - name: Store binaries for tests - uses: actions/upload-artifact@v3 - with: - name: ratis-bin - path: | - ratis-assembly/target/apache-ratis-*.tar.gz - !ratis-assembly/target/apache-ratis-*-src.tar.gz - retention-days: 1 - - name: Store source tarball for compilation - uses: actions/upload-artifact@v3 - with: - name: ratis-src - path: ratis-assembly/target/apache-ratis-*-src.tar.gz - retention-days: 1 - compile: - needs: - - build - runs-on: ubuntu-20.04 - strategy: - matrix: - java: [ 11 ] - fail-fast: false - steps: - - name: Download source tarball - uses: actions/download-artifact@v3 - with: - name: ratis-src - - name: Untar sources - run: | - tar --strip-components 1 -xzvf apache-ratis-*-src.tar.gz - - name: Cache for maven dependencies - uses: actions/cache/restore@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Run a full build - run: ./dev-support/checks/build.sh - rat: - name: rat - runs-on: ubuntu-20.04 - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache/restore@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Run tests - run: ./dev-support/checks/rat.sh - - name: Upload results - uses: actions/upload-artifact@v3 - if: always() - with: - name: rat - path: target/rat - author: - name: author - runs-on: ubuntu-20.04 - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Run tests - run: ./dev-support/checks/author.sh - - name: Upload results - uses: actions/upload-artifact@v3 - if: always() - with: - name: author - path: target/author - unit: - name: unit - runs-on: ubuntu-20.04 - strategy: - matrix: - profile: - - grpc - - server - - misc - fail-fast: false - steps: - # TEMPORARY WHILE GITHUB FIXES https://github.com/actions/virtual-environments/issues/3185 - - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file - run: | - echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - # REMOVE CODE ABOVE WHEN ISSUE IS ADDRESSED! - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache/restore@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 8 - - name: Run tests - run: ./dev-support/checks/unit.sh -P${{ matrix.profile }}-tests - - name: Summary of failures - run: cat target/${{ github.job }}/summary.txt - if: ${{ !cancelled() }} - - name: Upload results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: unit-${{ matrix.profile }} - path: target/unit - checkstyle: - name: checkstyle - runs-on: ubuntu-20.04 - steps: - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache/restore@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Run tests - run: ./dev-support/checks/checkstyle.sh - - name: Upload results - uses: actions/upload-artifact@v3 - if: always() - with: - name: checkstyle - path: target/checkstyle - findbugs: - name: findbugs - runs-on: ubuntu-20.04 - steps: - - name: Setup java - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 8 - - name: Checkout project - uses: actions/checkout@v3 - - name: Cache for maven dependencies - uses: actions/cache/restore@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Run tests - run: ./dev-support/checks/findbugs.sh - - name: Upload results - uses: actions/upload-artifact@v3 - if: always() - with: - name: findbugs - path: target/findbugs - coverage: - needs: - - build - - unit - runs-on: ubuntu-20.04 - if: (github.repository == 'apache/ratis' || github.repository == 'apache/incubator-ratis') && github.event_name != 'pull_request' - steps: - - name: Checkout project - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Cache for maven dependencies - uses: actions/cache/restore@v3 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java 17 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: 17 - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - path: target/artifacts - - name: Untar binaries - run: | - mkdir -p ratis-assembly/target - tar xzvf target/artifacts/ratis-bin/apache-ratis*.tar.gz -C ratis-assembly/target - - name: Calculate combined coverage - run: ./dev-support/checks/coverage.sh - - name: Upload coverage to Sonar - run: ./dev-support/checks/sonar.sh - env: - SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Archive build results - uses: actions/upload-artifact@v3 - if: always() - with: - name: ${{ github.job }} - path: target/${{ github.job }} diff --git a/.github/workflows/repeat-test.yml b/.github/workflows/repeat-test.yaml similarity index 91% rename from .github/workflows/repeat-test.yml rename to .github/workflows/repeat-test.yaml index 86150d2598..66fec90f16 100644 --- a/.github/workflows/repeat-test.yml +++ b/.github/workflows/repeat-test.yaml @@ -50,7 +50,7 @@ env: run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}#{1}[{2}]-{3}x{4}', inputs.test-class, inputs.test-method, inputs.ref, inputs.splits, inputs.iterations) || '' }} jobs: prepare: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.generate.outputs.matrix }} test-spec: ${{ steps.test-spec.outputs.test-spec }} @@ -79,7 +79,7 @@ jobs: if: ${{ always() }} needs: - prepare - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: TEST_SPEC: ${{ needs.prepare.outputs.test-spec }} strategy: @@ -87,11 +87,11 @@ jobs: split: ${{ fromJson(needs.prepare.outputs.matrix) }} fail-fast: ${{ fromJson(github.event.inputs.fail-fast) }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.ref }} - name: Cache for maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.m2/repository @@ -100,7 +100,7 @@ jobs: restore-keys: | maven-repo- - name: Setup java - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 8 @@ -113,18 +113,18 @@ jobs: run: dev-support/checks/_summary.sh target/unit/summary.txt if: ${{ !cancelled() }} - name: Archive build results - uses: actions/upload-artifact@v3 - if: always() + uses: actions/upload-artifact@v4 + if: ${{ failure() }} with: - name: result-${{ env.TEST_CLASS }}-split-${{ matrix.split }} + name: result-${{ github.run_number }}-${{ github.run_id }}-split-${{ matrix.split }} path: target/unit count-failures: - if: ${{ always() }} + if: ${{ failure() }} needs: test - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Download build results - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Count failures run: | failures=$(find . -name 'summary.txt' | grep -v 'iteration' | xargs grep -v 'exit code: 0' | wc -l) diff --git a/.gitignore b/.gitignore index 9379453102..ecaf1f2df9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ .hugo_build.lock .idea .classpath +.mvn/.develocity/ .project .settings target diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml new file mode 100644 index 0000000000..3bef395946 --- /dev/null +++ b/.mvn/develocity.xml @@ -0,0 +1,53 @@ + + + + ratis + + https://develocity.apache.org + false + + + + true + true + true + false + + #{isFalse(env['GITHUB_ACTIONS'])} + + + + + #{{'0.0.0.0'}} + + + + + #{isFalse(env['GITHUB_ACTIONS'])} + + + false + + + diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml new file mode 100644 index 0000000000..8ceede33b9 --- /dev/null +++ b/.mvn/extensions.xml @@ -0,0 +1,34 @@ + + + + + com.gradle + develocity-maven-extension + 1.22.2 + + + com.gradle + common-custom-user-data-maven-extension + 2.0.1 + + diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 08ea486aa5..d58dfb70ba 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.0/apache-maven-3.9.0-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/dev-support/checks/build.sh b/dev-support/checks/build.sh index 6add1ae605..ee8d3f3d7e 100755 --- a/dev-support/checks/build.sh +++ b/dev-support/checks/build.sh @@ -20,7 +20,7 @@ source "${DIR}/../find_maven.sh" : ${WITH_COVERAGE:="false"} -MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests --no-transfer-progress' +MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests' if [[ "${WITH_COVERAGE}" != "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" diff --git a/dev-support/checks/checkstyle.sh b/dev-support/checks/checkstyle.sh index a2ee427380..cb06fdaacd 100755 --- a/dev-support/checks/checkstyle.sh +++ b/dev-support/checks/checkstyle.sh @@ -23,7 +23,7 @@ REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../target/checkstyle"} mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -MAVEN_OPTIONS='-B -fae --no-transfer-progress -Dcheckstyle.failOnViolation=false' +MAVEN_OPTIONS='-B -fae -Dcheckstyle.failOnViolation=false' declare -i rc ${MVN} ${MAVEN_OPTIONS} checkstyle:check | tee "${REPORT_DIR}/output.log" diff --git a/dev-support/checks/shellcheck.sh b/dev-support/checks/compile.sh similarity index 65% rename from dev-support/checks/shellcheck.sh rename to dev-support/checks/compile.sh index feb307a2db..fba4394dec 100755 --- a/dev-support/checks/shellcheck.sh +++ b/dev-support/checks/compile.sh @@ -16,22 +16,16 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../.." || exit 1 -REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../target/shellcheck"} -mkdir -p "$REPORT_DIR" -REPORT_FILE="$REPORT_DIR/summary.txt" +source "${DIR}/../find_maven.sh" -echo "" > "$OUTPUT_FILE" -if [[ "$(uname -s)" = "Darwin" ]]; then - find . -type f -perm '-500' -else - find . -type f -executable -fi \ - | grep -v -e target/ -e node_modules/ -e '\.\(ico\|py\|yml\)$' \ - | xargs -n1 shellcheck \ - | tee "$REPORT_FILE" +: ${WITH_COVERAGE:="false"} -wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures" +MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests' -if [[ -s "${REPORT_FILE}" ]]; then - exit 1 +if [[ "${WITH_COVERAGE}" != "true" ]]; then + MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" fi + +export MAVEN_OPTS="-Xmx4096m" +${MVN} ${MAVEN_OPTIONS} clean verify "$@" +exit $? diff --git a/dev-support/checks/coverage.sh b/dev-support/checks/coverage.sh index a2fab9b32a..ff0aef1a48 100755 --- a/dev-support/checks/coverage.sh +++ b/dev-support/checks/coverage.sh @@ -29,7 +29,7 @@ mkdir -p "$REPORT_DIR" JACOCO_VERSION=$(${MVN} help:evaluate -Dexpression=jacoco.version -q -DforceStdout) #Install jacoco cli -${MVN} --non-recursive --no-transfer-progress \ +${MVN} --non-recursive \ org.apache.maven.plugins:maven-dependency-plugin:3.6.1:copy \ -Dartifact=org.jacoco:org.jacoco.cli:${JACOCO_VERSION}:jar:nodeps diff --git a/dev-support/checks/findbugs.sh b/dev-support/checks/findbugs.sh index 17c669b8d5..3a063b3fa1 100755 --- a/dev-support/checks/findbugs.sh +++ b/dev-support/checks/findbugs.sh @@ -20,7 +20,7 @@ source "${DIR}/../find_maven.sh" : ${WITH_COVERAGE:="false"} -MAVEN_OPTIONS='-B -fae --no-transfer-progress' +MAVEN_OPTIONS='-B -fae' if ! type unionBugs >/dev/null 2>&1 || ! type convertXmlToText >/dev/null 2>&1; then #shellcheck disable=SC2086 diff --git a/dev-support/checks/rat.sh b/dev-support/checks/rat.sh index 34d8a25854..9b55878eff 100755 --- a/dev-support/checks/rat.sh +++ b/dev-support/checks/rat.sh @@ -23,7 +23,7 @@ mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -${MVN} -B -fn --no-transfer-progress org.apache.rat:apache-rat-plugin:0.13:check +${MVN} -B -fn org.apache.rat:apache-rat-plugin:0.13:check cd "$DIR/../.." || exit 1 diff --git a/dev-support/checks/release.sh b/dev-support/checks/release.sh new file mode 100755 index 0000000000..1297b36b0c --- /dev/null +++ b/dev-support/checks/release.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -u -o pipefail + +# This script tests the local part of the release process. It does not publish anything. + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../.." || exit 1 + +: "${RATISVERSION:="0.0.1"}" +: "${RC:="-ci-test"}" +: "${STAGING_REPO_DIR:="/tmp/ratis.staging-repo"}" +: "${SVNDISTDIR:="/tmp/ratis.svn"}" +: "${USERID:="ratis-ci-not-for-release"}" + +MVN_REPO_DIR="${HOME}/.m2/repository" + +mkdir -p "${SVNDISTDIR}" + +if [[ -z "${CODESIGNINGKEY:-}" ]]; then + gpg --batch --passphrase '' --pinentry-mode loopback --quick-generate-key "${USERID}" rsa4096 default 1d + CODESIGNINGKEY=$(gpg --list-keys --with-colons "${USERID}" | grep '^pub:' | cut -f5 -d:) +fi + +git config user.email || git config user.email 'test@example.com' +git config user.name || git config user.name 'Test User' + +export CODESIGNINGKEY MVN_REPO_DIR RATISVERSION RC SVNDISTDIR + +export MAVEN_ARGS="--batch-mode" + +dev-support/make_rc.sh 1-prepare-src +dev-support/make_rc.sh 2-verify-bin +dev-support/make_rc.sh 3-publish-mvn -DaltDeploymentRepository="local::default::file://${STAGING_REPO_DIR}" +dev-support/make_rc.sh 4-assembly diff --git a/dev-support/checks/repro.sh b/dev-support/checks/repro.sh new file mode 100755 index 0000000000..88941bc286 --- /dev/null +++ b/dev-support/checks/repro.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../.." || exit 1 + +source "${DIR}/../find_maven.sh" + +: ${WITH_COVERAGE:="false"} + +MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests' + +if [[ "${WITH_COVERAGE}" != "true" ]]; then + MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" +fi + +export MAVEN_OPTS="-Xmx4096m" +${MVN} ${MAVEN_OPTIONS} clean verify artifact:compare "$@" +exit $? diff --git a/dev-support/checks/sonar.sh b/dev-support/checks/sonar.sh index 55a46cfec7..55edbdaaea 100755 --- a/dev-support/checks/sonar.sh +++ b/dev-support/checks/sonar.sh @@ -23,7 +23,7 @@ if [ ! "$SONAR_TOKEN" ]; then exit 1 fi -${MVN} -B verify -DskipShade -DskipTests --no-transfer-progress \ - org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.0.1398:sonar \ +${MVN} -B verify -DskipShade -DskipTests \ + sonar:sonar \ -Dsonar.coverage.jacoco.xmlReportPaths="$(pwd)/target/coverage/all.xml" \ -Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache-ratis diff --git a/dev-support/checks/unit.sh b/dev-support/checks/unit.sh index f7a4f3017e..c0369898e7 100755 --- a/dev-support/checks/unit.sh +++ b/dev-support/checks/unit.sh @@ -34,7 +34,11 @@ REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../target/unit"} mkdir -p "$REPORT_DIR" export MAVEN_OPTS="-Xmx4096m" -MAVEN_OPTIONS='-V -B --no-transfer-progress' +MAVEN_OPTIONS='-V -B' + +if [[ "$@" =~ "-Pflaky-tests" ]]; then + MAVEN_OPTIONS="${MAVEN_OPTIONS} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.timeout=1200" +fi if [[ "${FAIL_FAST}" == "true" ]]; then MAVEN_OPTIONS="${MAVEN_OPTIONS} --fail-fast -Dsurefire.skipAfterFailureCount=1" @@ -65,6 +69,12 @@ for i in $(seq 1 ${ITERATIONS}); do fi if [[ ${ITERATIONS} -gt 1 ]]; then + if ! grep -q "Running .*Test" "${REPORT_DIR}/output.log"; then + echo "No tests were run" >> "${REPORT_DIR}/summary.txt" + irc=1 + FAIL_FAST=true + fi + if [[ ${irc} == 0 ]]; then rm -fr "${REPORT_DIR}" fi diff --git a/dev-support/make_rc.sh b/dev-support/make_rc.sh index b5bec51dde..c6a93a6d9d 100755 --- a/dev-support/make_rc.sh +++ b/dev-support/make_rc.sh @@ -91,12 +91,12 @@ mvnFun() { MAVEN_OPTS="${mvnopts}" ${MVN} -Dmaven.repo.local="${repodir}" "$@" } -prepare-src() { +1-prepare-src() { cd "$projectdir" git reset --hard git clean -fdx mvnFun versions:set -DnewVersion="$RATISVERSION" - git commit -a -m "Change version for the version $RATISVERSION $RC" + git commit --allow-empty -a -m "Change version for the version $RATISVERSION $RC" git config user.signingkey "${CODESIGNINGKEY}" git tag -s -m "Release $RATISVERSION $RC" ratis-"${RATISVERSION}${RC}" @@ -106,69 +106,72 @@ prepare-src() { #grep -r SNAPSHOT --include=pom.xml - mvnFun clean install assembly:single -DskipTests=true -Prelease -Papache-release -Dgpg.keyname="${CODESIGNINGKEY}" + mvnFun clean install -DskipTests=true -Prelease -Papache-release -Dgpg.keyname="${CODESIGNINGKEY}" } -prepare-bin() { +2-verify-bin() { echo "Cleaning up workingdir $WORKINGDIR" rm -rf "$WORKINGDIR" mkdir -p "$WORKINGDIR" cd "$WORKINGDIR" - tar zvxf "$projectdir/ratis-assembly/target/apache-ratis-${RATISVERSION}-src.tar.gz" + tar zvxf "$projectdir/ratis-assembly/target/ratis-assembly-${RATISVERSION}-src.tar.gz" mv "apache-ratis-${RATISVERSION}-src" "apache-ratis-${RATISVERSION}" cd "apache-ratis-${RATISVERSION}" - mvnFun clean install assembly:single -DskipTests=true -Prelease -Papache-release -Dgpg.keyname="${CODESIGNINGKEY}" + mvnFun clean verify -DskipTests=true -Prelease -Papache-release -Dgpg.keyname="${CODESIGNINGKEY}" "$@" } -assembly() { +3-publish-mvn() { + cd "$projectdir" + mvnFun verify artifact:compare deploy:deploy -DdeployAtEnd=true -DskipTests=true -Prelease -Papache-release -Dgpg.keyname="${CODESIGNINGKEY}" "$@" +} + +4-assembly() { cd "$SVNDISTDIR" RCDIR="$SVNDISTDIR/${RATISVERSION}/${RC#-}" mkdir -p "$RCDIR" cd "$RCDIR" - cp "$WORKINGDIR/apache-ratis-${RATISVERSION}/ratis-assembly/target/apache-ratis-${RATISVERSION}-bin.tar.gz" "apache-ratis-${RATISVERSION}-bin.tar.gz" - cp "$projectdir/ratis-assembly/target/apache-ratis-${RATISVERSION}-src.tar.gz" "apache-ratis-${RATISVERSION}-src.tar.gz" + cp "$projectdir/ratis-assembly/target/ratis-assembly-${RATISVERSION}-bin.tar.gz" "apache-ratis-${RATISVERSION}-bin.tar.gz" + cp "$projectdir/ratis-assembly/target/ratis-assembly-${RATISVERSION}-src.tar.gz" "apache-ratis-${RATISVERSION}-src.tar.gz" for i in *.tar.gz; do gpg -u "${CODESIGNINGKEY}" --armor --output "${i}.asc" --detach-sig "${i}"; done for i in *.tar.gz; do gpg --print-md SHA512 "${i}" > "${i}.sha512"; done for i in *.tar.gz; do gpg --print-mds "${i}" > "${i}.mds"; done cd "$SVNDISTDIR" - svn add "${RATISVERSION}" || svn add "${RATISVERSION}/${RC#-}" + # skip svn add in CI + if [[ -z "${CI:-}" ]]; then + svn add "${RATISVERSION}" || svn add "${RATISVERSION}/${RC#-}" + fi } -publish-git(){ +5-publish-git(){ cd "$projectdir" git push apache "ratis-${RATISVERSION}${RC}" } -publish-svn() { +6-publish-svn() { cd "${SVNDISTDIR}" svn commit -m "Publish proposed version of the next Ratis release ${RATISVERSION}${RC}" } -publish-mvn(){ - cd "$projectdir" - mvnFun -X clean deploy assembly:single -DskipTests=true -Prelease -Papache-release -Dgpg.keyname="${CODESIGNINGKEY}" -} - -if [ "$#" -ne 1 ]; then +if [ "$#" -lt 1 ]; then cat << EOF -Please choose from available phases (eg. make_rc.sh prepare-src): +Please choose from available phases (eg. make_rc.sh 1-prepare-src): - 1. prepare-src: This is the first step. It modifies the mvn version, creates the git tag and + 1-prepare-src: This is the first step. It modifies the mvn version, creates the git tag and builds the project to create the source artifacts. IT INCLUDES A GIT RESET + CLEAN. ALL THE LOCAL CHANGES WILL BE LOST! - 2. prepare-bin: The source artifact is copied to the $WORKINGDIR and the binary artifact is created from the source. + 2-verify-bin: The source artifact is copied to the $WORKINGDIR and the binary artifact is created from the source. This is an additional check as the the released source artifact should be enough to build the whole project. - 3. assembly : This step copies all the required artifacts to the svn directory and ($SVNDISTDIR) creates the signatures/checksum files. + 3-publish-mvn: Performs the final build, and uploads the artifacts to the maven staging repository - 4. publish-git: The first remote step, only do it if everything is fine. It pushes the rc tag to the repository. + 4-assembly: This step copies all the required artifacts to the svn directory and ($SVNDISTDIR) creates the signatures/checksum files. - 5. publish-svn: Uploads the artifacts to the apache dev staging area to start the vote. + 5-publish-git: Only do it if everything is fine. It pushes the rc tag to the repository. - 6. publish-mvn: Uploads the artifacts to the maven staging repository + 6-publish-svn: Uploads the artifacts to the apache dev staging area to start the vote. The next steps of the release process are not scripted: @@ -189,5 +192,7 @@ The next steps of the release process are not scripted: EOF else set -x - eval "$1" + func="$1" + shift + eval "$func" "$@" fi diff --git a/mvnw b/mvnw index 8d937f4c14..19529ddf8c 100755 --- a/mvnw +++ b/mvnw @@ -19,290 +19,241 @@ # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- -# Apache Maven Wrapper startup batch script, version 3.2.0 -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir +# Apache Maven Wrapper startup batch script, version 3.3.2 # # Optional ENV vars # ----------------- -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output # ---------------------------------------------------------------------------- -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /usr/local/etc/mavenrc ] ; then - . /usr/local/etc/mavenrc - fi - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false +# OS specific support. +native_path() { printf %s\\n "$1"; } case "$(uname)" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME - else - JAVA_HOME="/Library/Java/Home"; export JAVA_HOME - fi - fi - ;; +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; esac -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=$(java-config --jre-home) - fi -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$JAVA_HOME" ] && - JAVA_HOME=$(cygpath --unix "$JAVA_HOME") - [ -n "$CLASSPATH" ] && - CLASSPATH=$(cygpath --path --unix "$CLASSPATH") -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && - JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="$(which javac)" - if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=$(which readlink) - if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then - if $darwin ; then - javaHome="$(dirname "\"$javaExecutable\"")" - javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" - else - javaExecutable="$(readlink -f "\"$javaExecutable\"")" - fi - javaHome="$(dirname "\"$javaExecutable\"")" - javaHome=$(expr "$javaHome" : '\(.*\)/bin') - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" else JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi fi else - JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi fi +} - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=$(cd "$wdir/.." || exit 1; pwd) - fi - # end of workaround +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" done - printf '%s' "$(cd "$basedir" || exit 1; pwd)" + printf %x\\n $h } -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - # Remove \r in case we run on Windows within Git Bash - # and check out the repository with auto CRLF management - # enabled. Otherwise, we may read lines that are delimited with - # \r\n and produce $'-Xarg\r' rather than -Xarg due to word - # splitting rules. - tr -s '\r\n' ' ' < "$1" - fi +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 } -log() { - if [ "$MVNW_VERBOSE" = true ]; then - printf '%s\n' "$1" - fi +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" } -BASE_DIR=$(find_maven_basedir "$(dirname "$0")") -if [ -z "$BASE_DIR" ]; then - exit 1; +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" fi -MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR -log "$MAVEN_PROJECTBASEDIR" +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" -if [ -r "$wrapperJarPath" ]; then - log "Found $wrapperJarPath" +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT else - log "Couldn't find $wrapperJarPath, downloading it ..." + die "cannot create temp dir" +fi - if [ -n "$MVNW_REPOURL" ]; then - wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - else - wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - fi - while IFS="=" read -r key value; do - # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) - safeValue=$(echo "$value" | tr -d '\r') - case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; - esac - done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" - log "Downloading from: $wrapperUrl" +mkdir -p -- "${MAVEN_HOME%/*}" - if $cygwin; then - wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") - fi +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" - if command -v wget > /dev/null; then - log "Found wget ... using wget" - [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" - else - wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" - fi - elif command -v curl > /dev/null; then - log "Found curl ... using curl" - [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" - else - curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" - fi - else - log "Falling back to using Java to download" - javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" - javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" - # For Cygwin, switch paths to Windows format before running javac - if $cygwin; then - javaSource=$(cygpath --path --windows "$javaSource") - javaClass=$(cygpath --path --windows "$javaClass") - fi - if [ -e "$javaSource" ]; then - if [ ! -e "$javaClass" ]; then - log " - Compiling MavenWrapperDownloader.java ..." - ("$JAVA_HOME/bin/javac" "$javaSource") - fi - if [ -e "$javaClass" ]; then - log " - Running MavenWrapperDownloader.java ..." - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" - fi - fi - fi +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" fi -########################################################################################## -# End of extension -########################################################################################## -# If specified, validate the SHA-256 sum of the Maven wrapper jar file -wrapperSha256Sum="" -while IFS="=" read -r key value; do - case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; - esac -done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" -if [ -n "$wrapperSha256Sum" ]; then - wrapperSha256Result=false - if command -v sha256sum > /dev/null; then - if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then - wrapperSha256Result=true +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true fi - elif command -v shasum > /dev/null; then - if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then - wrapperSha256Result=true + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true fi else - echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." - echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 exit 1 fi - if [ $wrapperSha256Result = false ]; then - echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 - echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 - echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 exit 1 fi fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$JAVA_HOME" ] && - JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") - [ -n "$CLASSPATH" ] && - CLASSPATH=$(cygpath --path --windows "$CLASSPATH") - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" -# Provide a "standardized" way to retrieve the CLI args that will -# work with both Windows and non-Windows executions. -MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" -export MAVEN_CMD_LINE_ARGS - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -# shellcheck disable=SC2086 # safe args -exec "$JAVACMD" \ - $MAVEN_OPTS \ - $MAVEN_DEBUG_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd index f80fbad3e7..b150b91ed5 100644 --- a/mvnw.cmd +++ b/mvnw.cmd @@ -1,3 +1,4 @@ +<# : batch portion @REM ---------------------------------------------------------------------------- @REM Licensed to the Apache Software Foundation (ASF) under one @REM or more contributor license agreements. See the NOTICE file @@ -18,188 +19,131 @@ @REM ---------------------------------------------------------------------------- @REM ---------------------------------------------------------------------------- -@REM Apache Maven Wrapper startup batch script, version 3.2.0 -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir +@REM Apache Maven Wrapper startup batch script, version 3.3.2 @REM @REM Optional ENV vars -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output @REM ---------------------------------------------------------------------------- -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* -if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - -FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - if "%MVNW_VERBOSE%" == "true" ( - echo Found %WRAPPER_JAR% - ) -) else ( - if not "%MVNW_REPOURL%" == "" ( - SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - ) - if "%MVNW_VERBOSE%" == "true" ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %WRAPPER_URL% - ) - - powershell -Command "&{"^ - "$webclient = new-object System.Net.WebClient;"^ - "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ - "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ - "}"^ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ - "}" - if "%MVNW_VERBOSE%" == "true" ( - echo Finished downloading %WRAPPER_JAR% - ) -) -@REM End of extension - -@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file -SET WRAPPER_SHA_256_SUM="" -FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) ) -IF NOT %WRAPPER_SHA_256_SUM%=="" ( - powershell -Command "&{"^ - "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ - "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ - " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ - " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ - " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ - " exit 1;"^ - "}"^ - "}" - if ERRORLEVEL 1 goto error -) - -@REM Provide a "standardized" way to retrieve the CLI args that will -@REM work with both Windows and non-Windows executions. -set MAVEN_CMD_LINE_ARGS=%* - -%MAVEN_JAVA_EXE% ^ - %JVM_CONFIG_MAVEN_PROPS% ^ - %MAVEN_OPTS% ^ - %MAVEN_DEBUG_OPTS% ^ - -classpath %WRAPPER_JAR% ^ - "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ - %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" -if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%"=="on" pause - -if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% - -cmd /C exit /B %ERROR_CODE% +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml index 68488077d4..af14e426f2 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT Apache Ratis pom @@ -51,6 +51,9 @@ ${distMgmtSnapshotsId} ${distMgmtSnapshotsName} ${distMgmtSnapshotsUrl} + + false + repository.jboss.org @@ -91,13 +94,6 @@ ratis-assembly - - - apache.snapshots - https://repository.apache.org/snapshots/ - - - scm:git:git://git.apache.org/ratis.git scm:git:https://git-wip-us.apache.org/repos/asf/ratis.git @@ -176,8 +172,12 @@ 3.3.0 4.0.6 1.6.1 - 3.0.0 + 3.3.0 + 3.6.0 + 3.0.0-M4 3.5.3 + 3.4.0 + 5.0.0.4389 @@ -188,7 +188,7 @@ 2.2.0 0.6.1 - 1.0 + 2.8.0 4.2.1 4.2.0 @@ -203,16 +203,16 @@ bash - 1.8 + 8 ${javac.version} 3.3.9 - 1.0.5 + 1.0.9 - 3.24.4 - 1.58.0 + 3.25.5 + 1.71.0 true @@ -221,8 +221,9 @@ 4 2.0.7 - 5.10.1 - 0.8.11 + 5.11.2 + 0.8.12 + flaky | org.apache.ratis.test.tag.FlakyTest @@ -425,7 +426,7 @@ org.junit junit-bom - ${junit.jupiter.version} + ${junit-bom.version} pom import @@ -440,14 +441,6 @@ 6.0.53 provided - - - com.github.spotbugs - spotbugs-annotations - ${spotbugs.version} - provided - true - @@ -462,6 +455,11 @@ + + org.apache.hadoop + hadoop-maven-plugins + ${hadoop-maven-plugins.version} + org.xolstice.maven.plugins protobuf-maven-plugin @@ -477,11 +475,6 @@ license-maven-plugin ${license-maven-plugin.version} - - com.coderplus.maven.plugins - copy-rename-maven-plugin - ${copy-rename-maven-plugin.version} - org.apache.maven.plugins @@ -499,6 +492,11 @@ + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + com.github.spotbugs spotbugs-maven-plugin @@ -529,6 +527,7 @@ org.apache.maven.plugins maven-enforcer-plugin + 3.2.1 @@ -581,8 +580,6 @@ org.apache.maven.plugins maven-compiler-plugin - ${javac.version} - ${javac.version} true 512m 2048m @@ -642,9 +639,10 @@ false false false + all 600 - -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError @{argLine} + -Xmx2g -XX:+HeapDumpOnOutOfMemoryError @{argLine} ${project.build.directory}/log ${project.build.directory}/tmp @@ -658,12 +656,6 @@ **/Test*$*.java ${test.exclude.pattern} - - - listener - org.apache.ratis.JUnitRunListener - - @@ -704,6 +696,11 @@ org.apache.maven.plugins maven-antrun-plugin + + org.apache.maven.plugins + maven-remote-resources-plugin + ${maven-remote-resources-plugin.version} + org.apache.maven.plugins maven-site-plugin @@ -731,6 +728,16 @@ jacoco-maven-plugin ${jacoco.version} + + org.cyclonedx + cyclonedx-maven-plugin + ${cyclonedx.version} + + + org.sonarsource.scanner.maven + sonar-maven-plugin + ${sonar-maven-plugin.version} + @@ -788,11 +795,31 @@ true true + + org.apache.hadoop + hadoop-maven-plugins + + + version-info + generate-resources + + version-info + + + + ${project.basedir} + + + + + org.apache.maven.plugins maven-checkstyle-plugin ${maven-checkstyle-plugin.version} + + dev-support/checkstyle.xml true false @@ -807,24 +834,6 @@ false - - org.codehaus.mojo - buildnumber-maven-plugin - 1.4 - - - generate-resources - - create-metadata - - - target/classes - ratis-version.properties - Unknown - - - - org.codehaus.mojo build-helper-maven-plugin @@ -870,9 +879,41 @@ + + + ${project.basedir}/src/main/resources + false + + + ${project.basedir}/../src/main/resources + + ratis-version.properties + + true + + + + java8 + + [,8] + + + ${javac.version} + ${javac.version} + + + + java9-or-later + + [9,] + + + ${javac.version} + + experiments-build @@ -939,9 +980,6 @@ jar - - ${project.build.directory} - @@ -1060,6 +1098,18 @@ + + org.cyclonedx + cyclonedx-maven-plugin + + + package + + makeBom + + + + @@ -1075,6 +1125,7 @@ org.apache.ratis.grpc.** + ${flaky-test-groups} @@ -1092,6 +1143,7 @@ org.apache.ratis.datastream.** org.apache.ratis.server.** + ${flaky-test-groups} @@ -1110,6 +1162,21 @@ org.apache.ratis.grpc.** org.apache.ratis.server.** + ${flaky-test-groups} + + + + + + + flaky-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${flaky-test-groups} diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml index 98826e142a..2b735fcb1e 100644 --- a/ratis-assembly/pom.xml +++ b/ratis-assembly/pom.xml @@ -17,7 +17,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-assembly @@ -33,6 +33,8 @@ ${project.build.directory}/test-classes ${project.build.directory}/test-classes true + + true @@ -120,8 +122,6 @@ org.apache.maven.plugins maven-assembly-plugin - - apache-ratis-${project.version} false true gnu @@ -137,25 +137,18 @@ src/main/assembly/src.xml - apache-ratis-${project.version}-src - false - default-cli + bin package single - src/main/assembly/examples-bin.xml - src/main/assembly/shell-bin.xml src/main/assembly/bin.xml - src/main/assembly/bin-pkg.xml - apache-ratis-${project.version}-bin - false diff --git a/ratis-assembly/src/main/assembly/bin-pkg.xml b/ratis-assembly/src/main/assembly/bin-pkg.xml deleted file mode 100644 index 4d89869578..0000000000 --- a/ratis-assembly/src/main/assembly/bin-pkg.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - bin-pkg - - tar.gz - - - - ${project.basedir}/target/apache-ratis-${project.version}-bin/apache-ratis-${project.version}-bin - .. - - bin/ratis - libexec/*.sh - examples/bin/*.sh - - - - ${project.basedir}/target/apache-ratis-${project.version}-bin/apache-ratis-${project.version}-bin - .. - - bin/ratis - libexec/*.sh - examples/bin/*.sh - - 0755 - - - diff --git a/ratis-assembly/src/main/assembly/bin.xml b/ratis-assembly/src/main/assembly/bin.xml index 7be3d01d61..7fd3cc0df5 100644 --- a/ratis-assembly/src/main/assembly/bin.xml +++ b/ratis-assembly/src/main/assembly/bin.xml @@ -22,9 +22,18 @@ */ --> bin + apache-ratis-${project.version}-bin - dir + tar.gz + + + + org.apache.ratis:ratis-examples + + examples/lib + + true @@ -42,6 +51,7 @@ org.apache.ratis:ratis-metrics-api org.apache.ratis:ratis-metrics-default org.apache.ratis:ratis-metrics-dropwizard3 + org.apache.ratis:ratis-shell org.apache.ratis:ratis-tools org.apache.ratis:ratis-resource-bundle @@ -63,6 +73,16 @@ 0644 + + + ${project.basedir}/../target + . + + bom.json + bom.xml + + 0644 + ${project.basedir}/../ratis-docs/target/classes/docs @@ -70,5 +90,50 @@ 0644 0755 + + ${project.basedir}/../ratis-shell/src/main/bin + bin + 0755 + + + ${project.basedir}/../ratis-shell/src/main/libexec + libexec + 0755 + 0755 + + + ${project.basedir}/../ratis-shell/src/main/conf + conf + 644 + + + ${project.basedir}/../ratis-shell/target/lib/ + jars + + + ${project.basedir}/../ratis-examples + examples + + README.md + + 0644 + + + ${project.basedir}/../ratis-examples/src/main/bin + examples/bin + + *.* + + 0755 + + + ${project.basedir}/../ratis-examples/src/main/resources + examples/conf + + conf.properties + log4j.properties + + 644 + diff --git a/ratis-assembly/src/main/assembly/examples-bin.xml b/ratis-assembly/src/main/assembly/examples-bin.xml deleted file mode 100644 index 21cc7eced2..0000000000 --- a/ratis-assembly/src/main/assembly/examples-bin.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - examples-bin - - dir - - - - - org.apache.ratis:ratis-examples - - examples/lib - - - - - ${project.basedir}/src/main/resources - . - - README.md - LICENSE - NOTICE - - 0644 - - - ${project.basedir}/../ratis-examples - examples - - README.md - - 0644 - - - ${project.basedir}/../ratis-examples/src/main/bin - examples/bin - - *.* - - 0755 - - - ${project.basedir}/../ratis-examples/src/main/resources - examples/conf - - conf.properties - log4j.properties - - 644 - - - diff --git a/ratis-assembly/src/main/assembly/shell-bin.xml b/ratis-assembly/src/main/assembly/shell-bin.xml deleted file mode 100644 index 470870f41c..0000000000 --- a/ratis-assembly/src/main/assembly/shell-bin.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - shell - - dir - - - - ${project.basedir}/../ratis-shell/target/ - jars - - ratis-shell-${project.version}.jar - - - - ${project.basedir}/../ratis-shell/target/lib/ - jars - - - ${project.basedir}/src/main/resources - . - - README.md - LICENSE - NOTICE - - 0644 - - - ${project.basedir}/../ratis-shell/src/main/bin - bin - 0755 - - - ${project.basedir}/../ratis-shell/src/main/libexec - libexec - 0755 - 0755 - - - ${project.basedir}/../ratis-shell/src/main/conf - conf - 644 - - - diff --git a/ratis-assembly/src/main/assembly/src.xml b/ratis-assembly/src/main/assembly/src.xml index 98e06c5739..6aac4ec062 100644 --- a/ratis-assembly/src/main/assembly/src.xml +++ b/ratis-assembly/src/main/assembly/src.xml @@ -22,6 +22,7 @@ */ --> src + apache-ratis-${project.version}-src tar.gz @@ -102,6 +103,7 @@ README.md mvnw.cmd pom.xml + src/** start-build-env.sh 0644 diff --git a/ratis-assembly/src/main/resources/NOTICE b/ratis-assembly/src/main/resources/NOTICE index 9bc9242c6a..0e3c94434b 100644 --- a/ratis-assembly/src/main/resources/NOTICE +++ b/ratis-assembly/src/main/resources/NOTICE @@ -292,27 +292,5 @@ networking library, which can be obtained at: * HOMEPAGE: * https://netty.io * LOCATION_IN_GRPC: -* netty/third_party/netty ------------------------------------------------------------------------ -The JSR-305 reference implementation (jsr305.jar) is distributed under the terms of the New BSD: - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - + * netty/third_party/netty ----------------------------------------------------------------------- diff --git a/ratis-client/pom.xml b/ratis-client/pom.xml index d1b08d5a6a..1456e682ae 100644 --- a/ratis-client/pom.xml +++ b/ratis-client/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-client Apache Ratis Client + + + true + + org.apache.ratis diff --git a/ratis-client/src/main/java/org/apache/ratis/client/RaftClientConfigKeys.java b/ratis-client/src/main/java/org/apache/ratis/client/RaftClientConfigKeys.java index 7360a9cadb..925324c21c 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/RaftClientConfigKeys.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/RaftClientConfigKeys.java @@ -42,7 +42,7 @@ interface Rpc { String PREFIX = RaftClientConfigKeys.PREFIX + ".rpc"; String REQUEST_TIMEOUT_KEY = PREFIX + ".request.timeout"; - TimeDuration REQUEST_TIMEOUT_DEFAULT = TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS); + TimeDuration REQUEST_TIMEOUT_DEFAULT = TimeDuration.valueOf(3, TimeUnit.SECONDS); static TimeDuration requestTimeout(RaftProperties properties) { return getTimeDuration(properties.getTimeDuration(REQUEST_TIMEOUT_DEFAULT.getUnit()), REQUEST_TIMEOUT_KEY, REQUEST_TIMEOUT_DEFAULT, getDefaultLog()); @@ -52,8 +52,7 @@ static void setRequestTimeout(RaftProperties properties, TimeDuration timeoutDur } String WATCH_REQUEST_TIMEOUT_KEY = PREFIX + ".watch.request.timeout"; - TimeDuration WATCH_REQUEST_TIMEOUT_DEFAULT = - TimeDuration.valueOf(10000, TimeUnit.MILLISECONDS); + TimeDuration WATCH_REQUEST_TIMEOUT_DEFAULT = TimeDuration.valueOf(10, TimeUnit.SECONDS); static TimeDuration watchRequestTimeout(RaftProperties properties) { return getTimeDuration(properties.getTimeDuration(WATCH_REQUEST_TIMEOUT_DEFAULT.getUnit()), WATCH_REQUEST_TIMEOUT_KEY, WATCH_REQUEST_TIMEOUT_DEFAULT, getDefaultLog()); @@ -125,7 +124,7 @@ static void setFlushRequestBytesMin(RaftProperties properties, SizeInBytes flush } String REQUEST_TIMEOUT_KEY = PREFIX + ".request.timeout"; - TimeDuration REQUEST_TIMEOUT_DEFAULT = TimeDuration.valueOf(10000, TimeUnit.MILLISECONDS); + TimeDuration REQUEST_TIMEOUT_DEFAULT = TimeDuration.valueOf(10, TimeUnit.SECONDS); static TimeDuration requestTimeout(RaftProperties properties) { return getTimeDuration(properties.getTimeDuration(REQUEST_TIMEOUT_DEFAULT.getUnit()), REQUEST_TIMEOUT_KEY, REQUEST_TIMEOUT_DEFAULT, getDefaultLog()); diff --git a/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java b/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java index edd0475442..f83d976040 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/api/SnapshotManagementApi.java @@ -27,6 +27,24 @@ */ public interface SnapshotManagementApi { - /** trigger create snapshot file. */ - RaftClientReply create(long timeoutMs) throws IOException; + /** The same as create(0, timeoutMs). */ + default RaftClientReply create(long timeoutMs) throws IOException { + return create(0, timeoutMs); + } + + /** The same as create(force? 1 : 0, timeoutMs). */ + default RaftClientReply create(boolean force, long timeoutMs) throws IOException { + return create(force? 1 : 0, timeoutMs); + } + + /** + * Trigger to create a snapshot. + * + * @param creationGap When (creationGap > 0) and (astAppliedIndex - lastSnapshotIndex < creationGap), + * return lastSnapshotIndex; otherwise, take a new snapshot and then return its index. + * When creationGap == 0, use the server configured value as the creationGap. + * @return a reply. When {@link RaftClientReply#isSuccess()} is true, + * {@link RaftClientReply#getLogIndex()} is the snapshot index fulfilling the operation. + */ + RaftClientReply create(long creationGap, long timeoutMs) throws IOException; } diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java index 4be9fa3275..76987801ba 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/BlockingImpl.java @@ -119,16 +119,18 @@ public RaftClientRequest newRequestImpl() { ioe = e; } - pending.incrementExceptionCount(ioe); - ClientRetryEvent event = new ClientRetryEvent(request, ioe, pending); + if (client.isClosed()) { + throw new AlreadyClosedException(this + " is closed."); + } + + final ClientRetryEvent event = pending.newClientRetryEvent(request, ioe); final RetryPolicy retryPolicy = client.getRetryPolicy(); final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event); - TimeDuration sleepTime = client.getEffectiveSleepTime(ioe, action.getSleepTime()); - if (!action.shouldRetry()) { - throw (IOException)client.noMoreRetries(event); + throw client.noMoreRetries(event); } + final TimeDuration sleepTime = client.getEffectiveSleepTime(ioe, action.getSleepTime()); try { sleepTime.sleep(); } catch (InterruptedException e) { diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java index db19831955..36c0b3937f 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/ClientProtoUtils.java @@ -179,11 +179,11 @@ static RaftClientRequest toRaftClientRequest(RaftClientRequestProto p) { final RaftClientRequest.Builder b = RaftClientRequest.newBuilder(); - final RaftPeerId perrId = RaftPeerId.valueOf(request.getReplyId()); + final RaftPeerId peerId = RaftPeerId.valueOf(request.getReplyId()); if (request.getToLeader()) { - b.setLeaderId(perrId); + b.setLeaderId(peerId); } else { - b.setServerId(perrId); + b.setServerId(peerId); } if (request.hasSlidingWindowEntry()) { b.setSlidingWindowEntry(request.getSlidingWindowEntry()); @@ -204,9 +204,13 @@ static ByteBuffer toRaftClientRequestProtoByteBuffer(RaftClientRequest request) } static RaftClientRequestProto toRaftClientRequestProto(RaftClientRequest request) { + return toRaftClientRequestProto(request, true); + } + + static RaftClientRequestProto toRaftClientRequestProto(RaftClientRequest request, boolean withMsg) { final RaftClientRequestProto.Builder b = RaftClientRequestProto.newBuilder() .setRpcRequest(toRaftRpcRequestProtoBuilder(request)); - if (request.getMessage() != null) { + if (withMsg && request.getMessage() != null) { b.setMessage(toClientMessageEntryProtoBuilder(request.getMessage())); } @@ -364,6 +368,7 @@ static GroupInfoReplyProto toGroupInfoReplyProto(GroupInfoReply reply) { b.setIsRaftStorageHealthy(reply.isRaftStorageHealthy()); b.setRole(reply.getRoleInfoProto()); b.addAllCommitInfos(reply.getCommitInfos()); + b.setLogInfo(reply.getLogInfoProto()); } } return b.build(); @@ -397,7 +402,8 @@ static RaftClientReply toRaftClientReply(RaftClientReplyProto replyProto) { e = new NotLeaderException(serverMemberId, suggestedLeader, peers); } else if (replyProto.getExceptionDetailsCase() == NOTREPLICATEDEXCEPTION) { final NotReplicatedExceptionProto nre = replyProto.getNotReplicatedException(); - e = new NotReplicatedException(nre.getCallId(), nre.getReplication(), nre.getLogIndex()); + e = new NotReplicatedException(nre.getCallId(), nre.getReplication(), nre.getLogIndex(), + replyProto.getCommitInfosList()); } else if (replyProto.getExceptionDetailsCase().equals(STATEMACHINEEXCEPTION)) { e = toStateMachineException(serverMemberId, replyProto.getStateMachineException()); } else if (replyProto.getExceptionDetailsCase().equals(DATASTREAMEXCEPTION)) { @@ -506,7 +512,8 @@ static GroupInfoReply toGroupInfoReply(GroupInfoReplyProto replyProto) { ProtoUtils.toRaftGroup(replyProto.getGroup()), replyProto.getRole(), replyProto.getIsRaftStorageHealthy(), - replyProto.hasConf()? replyProto.getConf(): null); + replyProto.hasConf()? replyProto.getConf(): null, + replyProto.getLogInfo()); } static Message toMessage(final ClientMessageEntryProto p) { @@ -657,7 +664,8 @@ static SnapshotManagementRequest toSnapshotManagementRequest(SnapshotManagementR switch(p.getOpCase()) { case CREATE: return SnapshotManagementRequest.newCreate(clientId, serverId, - ProtoUtils.toRaftGroupId(m.getRaftGroupId()), m.getCallId(), m.getTimeoutMs()); + ProtoUtils.toRaftGroupId(m.getRaftGroupId()), m.getCallId(), m.getTimeoutMs(), + p.getCreate().getCreationGap()); default: throw new IllegalArgumentException("Unexpected op " + p.getOpCase() + " in " + p); } @@ -669,7 +677,7 @@ static SnapshotManagementRequestProto toSnapshotManagementRequestProto( .setRpcRequest(toRaftRpcRequestProtoBuilder(request)); final SnapshotManagementRequest.Create create = request.getCreate(); if (create != null) { - b.setCreate(SnapshotCreateRequestProto.newBuilder().build()); + b.setCreate(SnapshotCreateRequestProto.newBuilder().setCreationGap(create.getCreationGap()).build()); } return b.build(); } diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/DataStreamClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/DataStreamClientImpl.java index 26d01c356f..313131cbda 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/DataStreamClientImpl.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/DataStreamClientImpl.java @@ -40,6 +40,7 @@ import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.exceptions.AlreadyClosedException; import org.apache.ratis.rpc.CallId; +import org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf; import org.apache.ratis.util.IOUtils; import org.apache.ratis.protocol.*; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; @@ -152,6 +153,9 @@ private CompletableFuture combineHeader(CompletableFuture writeAsyncImpl(Object data, long length, Iterable options) { if (isClosed()) { + if (data instanceof ByteBuf) { + ((ByteBuf) data).release(); + } return JavaUtils.completeExceptionally(new AlreadyClosedException( clientId + ": stream already closed, request=" + header)); } @@ -169,6 +173,10 @@ private CompletableFuture writeAsyncImpl(Object data, long leng return f; } + public CompletableFuture writeAsync(ByteBuf src, Iterable options) { + return writeAsyncImpl(src, src.readableBytes(), options); + } + @Override public CompletableFuture writeAsync(ByteBuffer src, Iterable options) { return writeAsyncImpl(src, src.remaining(), options); @@ -235,7 +243,7 @@ public DataStreamClientRpc getClientRpc() { } @Override - public DataStreamOutputRpc stream(RaftClientRequest request) { + public DataStreamOutputImpl stream(RaftClientRequest request) { return new DataStreamOutputImpl(request); } diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java index a1aa58681c..1e21b171b3 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedAsync.java @@ -32,6 +32,7 @@ import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.retry.RetryPolicy; import org.apache.ratis.rpc.CallId; +import org.apache.ratis.util.BatchLogger; import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; @@ -50,6 +51,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.LongFunction; @@ -57,6 +59,10 @@ public final class OrderedAsync { public static final Logger LOG = LoggerFactory.getLogger(OrderedAsync.class); + private enum BatchLogKey implements BatchLogger.Key { + SEND_REQUEST_EXCEPTION + } + static class PendingOrderedRequest extends PendingClientRequest implements SlidingWindow.ClientSideRequest { private final long callId; @@ -149,10 +155,6 @@ private void failAllAsyncRequests(RaftClientRequest request, Throwable t) { getSlidingWindow(request).fail(request.getSlidingWindowEntry().getSeqNum(), t); } - private void handleAsyncRetryFailure(ClientRetryEvent event) { - failAllAsyncRequests(event.getRequest(), client.noMoreRetries(event)); - } - CompletableFuture send(RaftClientRequest.Type type, Message message, RaftPeerId server) { if (!type.is(TypeCase.WATCH) && !type.is(TypeCase.MESSAGESTREAM)) { Objects.requireNonNull(message, "message == null"); @@ -187,85 +189,71 @@ private void sendRequestWithRetry(PendingOrderedRequest pending) { if (pending == null) { return; } - - final CompletableFuture f = pending.getReplyFuture(); - if (f.isDone()) { + if (pending.getReplyFuture().isDone()) { return; } - final RaftClientRequest request = pending.newRequestImpl(); + final RaftClientRequest request = pending.newRequest(); if (request == null) { // already done - LOG.debug("{} newRequestImpl returns null", pending); + LOG.debug("{} newRequest returns null", pending); return; } - final RetryPolicy retryPolicy = client.getRetryPolicy(); - sendRequest(pending).exceptionally(e -> { - if (e instanceof CompletionException) { - e = JavaUtils.unwrapCompletionException(e); - scheduleWithTimeout(pending, request, retryPolicy, e); - return null; - } - f.completeExceptionally(e); - return null; - }); - } - - private void scheduleWithTimeout(PendingOrderedRequest pending, - RaftClientRequest request, RetryPolicy retryPolicy, Throwable e) { - final int attempt = pending.getAttemptCount(); - final ClientRetryEvent event = new ClientRetryEvent(request, e, pending); - final TimeDuration sleepTime = client.getEffectiveSleepTime(e, - retryPolicy.handleAttemptFailure(event).getSleepTime()); - LOG.debug("schedule* attempt #{} with sleep {} and policy {} for {}", attempt, sleepTime, retryPolicy, request); - scheduleWithTimeout(pending, sleepTime, getSlidingWindow(request)); - } - - private void scheduleWithTimeout(PendingOrderedRequest pending, TimeDuration sleepTime, - SlidingWindow.Client slidingWindow) { - client.getScheduler().onTimeout(sleepTime, - () -> slidingWindow.retry(pending, this::sendRequestWithRetry), - LOG, () -> "Failed* to retry " + pending); - } - - private CompletableFuture sendRequest(PendingOrderedRequest pending) { - final RetryPolicy retryPolicy = client.getRetryPolicy(); - final RaftClientRequest request; if (getSlidingWindow((RaftPeerId) null).isFirst(pending.getSeqNum())) { pending.setFirstRequest(); } - request = pending.newRequest(); LOG.debug("{}: send* {}", client.getId(), request); - return client.getClientRpc().sendRequestAsync(request).thenApply(reply -> { + client.getClientRpc().sendRequestAsync(request).thenAccept(reply -> { LOG.debug("{}: receive* {}", client.getId(), reply); Objects.requireNonNull(reply, "reply == null"); client.handleReply(request, reply); getSlidingWindow(request).receiveReply( request.getSlidingWindowEntry().getSeqNum(), reply, this::sendRequestWithRetry); - return reply; }).exceptionally(e -> { - LOG.error(client.getId() + ": Failed* " + request, e); - e = JavaUtils.unwrapCompletionException(e); - if (e instanceof IOException && !(e instanceof GroupMismatchException)) { - pending.incrementExceptionCount(e); - final ClientRetryEvent event = new ClientRetryEvent(request, e, pending); - if (!retryPolicy.handleAttemptFailure(event).shouldRetry()) { - handleAsyncRetryFailure(event); - } else { - if (e instanceof NotLeaderException) { - NotLeaderException nle = (NotLeaderException)e; - client.handleNotLeaderException(request, nle, this::resetSlidingWindow); - } else { - client.handleIOException(request, (IOException) e, null, this::resetSlidingWindow); - } - } - throw new CompletionException(e); - } - failAllAsyncRequests(request, e); + final Throwable exception = e; + final String key = client.getId() + "-" + request.getCallId() + "-" + exception; + final Consumer op = suffix -> LOG.error("{} {}: Failed* {}", suffix, client.getId(), request, exception); + BatchLogger.print(BatchLogKey.SEND_REQUEST_EXCEPTION, key, op); + handleException(pending, request, e); return null; }); } + private void handleException(PendingOrderedRequest pending, RaftClientRequest request, Throwable e) { + final RetryPolicy retryPolicy = client.getRetryPolicy(); + if (client.isClosed()) { + failAllAsyncRequests(request, new AlreadyClosedException(client + " is closed.")); + return; + } + + e = JavaUtils.unwrapCompletionException(e); + if (!(e instanceof IOException) || e instanceof GroupMismatchException) { + // non-retryable exceptions + failAllAsyncRequests(request, e); + return; + } + + final ClientRetryEvent event = pending.newClientRetryEvent(request, e); + final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event); + if (!action.shouldRetry()) { + failAllAsyncRequests(request, client.noMoreRetries(event)); + return; + } + + if (e instanceof NotLeaderException) { + client.handleNotLeaderException(request, (NotLeaderException) e, this::resetSlidingWindow); + } else { + client.handleIOException(request, (IOException) e, null, this::resetSlidingWindow); + } + final TimeDuration sleepTime = client.getEffectiveSleepTime(e, action.getSleepTime()); + LOG.debug("schedule* retry with sleep {} for attempt #{} of {}, {}", + sleepTime, event.getAttemptCount(), request, retryPolicy); + final SlidingWindow.Client slidingWindow = getSlidingWindow(request); + client.getScheduler().onTimeout(sleepTime, + () -> slidingWindow.retry(pending, this::sendRequestWithRetry), + LOG, () -> "Failed* to retry " + pending); + } + void assertRequestSemaphore(int expectedAvailablePermits, int expectedQueueLength) { Preconditions.assertSame(expectedAvailablePermits, requestSemaphore.availablePermits(), "availablePermits"); Preconditions.assertSame(expectedQueueLength, requestSemaphore.getQueueLength(), "queueLength"); diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedStreamAsync.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedStreamAsync.java index 989c00cbbc..275755514f 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedStreamAsync.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/OrderedStreamAsync.java @@ -21,12 +21,14 @@ import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.datastream.impl.DataStreamPacketByteBuffer; +import org.apache.ratis.datastream.impl.DataStreamRequestByteBuf; import org.apache.ratis.datastream.impl.DataStreamRequestByteBuffer; import org.apache.ratis.datastream.impl.DataStreamRequestFilePositionCount; import org.apache.ratis.io.FilePositionCount; import org.apache.ratis.protocol.DataStreamReply; import org.apache.ratis.protocol.DataStreamRequest; import org.apache.ratis.protocol.DataStreamRequestHeader; +import org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf; import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.SlidingWindow; @@ -56,6 +58,8 @@ static class DataStreamWindowRequest implements SlidingWindow.ClientSideRequest< DataStreamRequest getDataStreamRequest() { if (header.getDataLength() == 0) { return new DataStreamRequestByteBuffer(header, DataStreamPacketByteBuffer.EMPTY_BYTE_BUFFER); + } else if (data instanceof ByteBuf) { + return new DataStreamRequestByteBuf(header, (ByteBuf)data); } else if (data instanceof ByteBuffer) { return new DataStreamRequestByteBuffer(header, (ByteBuffer)data); } else if (data instanceof FilePositionCount) { diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java index ec16763c2c..db789aef2f 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java @@ -44,11 +44,13 @@ import org.apache.ratis.thirdparty.com.google.common.cache.Cache; import org.apache.ratis.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.ratis.util.CollectionUtils; +import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.TimeoutExecutor; +import org.apache.ratis.util.Timestamp; import java.io.IOException; import java.util.ArrayList; @@ -65,6 +67,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -79,16 +82,18 @@ public final class RaftClientImpl implements RaftClient { .build(); public abstract static class PendingClientRequest { - private final long creationTimeInMs = System.currentTimeMillis(); + private final Timestamp creationTime = Timestamp.currentTime(); private final CompletableFuture replyFuture = new CompletableFuture<>(); private final AtomicInteger attemptCount = new AtomicInteger(); - private final Map, Integer> exceptionCount = new ConcurrentHashMap<>(); + private final Map, Integer> exceptionCounts = new ConcurrentHashMap<>(); public abstract RaftClientRequest newRequestImpl(); final RaftClientRequest newRequest() { - attemptCount.incrementAndGet(); - return newRequestImpl(); + final int attempt = attemptCount.incrementAndGet(); + final RaftClientRequest request = newRequestImpl(); + LOG.debug("attempt #{}, newRequest {}", attempt, request); + return request; } CompletableFuture getReplyFuture() { @@ -99,19 +104,10 @@ public int getAttemptCount() { return attemptCount.get(); } - int incrementExceptionCount(Throwable t) { - return t != null ? exceptionCount.compute(t.getClass(), (k, v) -> v != null ? v + 1 : 1) : 0; - } - - public int getExceptionCount(Throwable t) { - return t != null ? Optional.ofNullable(exceptionCount.get(t.getClass())).orElse(0) : 0; - } - - public boolean isRequestTimeout(TimeDuration timeout) { - if (timeout == null) { - return false; - } - return System.currentTimeMillis() - creationTimeInMs > timeout.toLong(TimeUnit.MILLISECONDS); + public ClientRetryEvent newClientRetryEvent(RaftClientRequest request, Throwable throwable) { + final int exceptionCount = throwable == null? 0 + : exceptionCounts.compute(throwable.getClass(), (k, v) -> v == null? 1: v+1); + return new ClientRetryEvent(getAttemptCount(), request, exceptionCount, throwable, creationTime); } } @@ -176,6 +172,7 @@ private synchronized Set getAndReset() { private final RaftGroupId groupId; private final RetryPolicy retryPolicy; + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile RaftPeerId leaderId; /** The callIds of the replied requests. */ private final RepliedCallIds repliedCallIds; @@ -194,6 +191,8 @@ private synchronized Set getAndReset() { private final ConcurrentMap leaderElectionManagement = new ConcurrentHashMap<>(); + private final AtomicBoolean closed = new AtomicBoolean(); + @SuppressWarnings("checkstyle:ParameterNumber") RaftClientImpl(ClientId clientId, RaftGroup group, RaftPeerId leaderId, RaftPeer primaryDataStreamServer, RaftClientRpc clientRpc, RetryPolicy retryPolicy, RaftProperties properties, Parameters parameters) { @@ -344,11 +343,11 @@ public DataStreamApi getDataStreamApi() { return dataStreamApi.get(); } - Throwable noMoreRetries(ClientRetryEvent event) { + IOException noMoreRetries(ClientRetryEvent event) { final int attemptCount = event.getAttemptCount(); final Throwable throwable = event.getCause(); if (attemptCount == 1 && throwable != null) { - return throwable; + return IOUtils.asIOException(throwable); } return new RaftRetryFailureException(event.getRequest(), attemptCount, retryPolicy, throwable); } @@ -416,8 +415,7 @@ void handleIOException(RaftClientRequest request, IOException ioe) { void handleIOException(RaftClientRequest request, IOException ioe, RaftPeerId newLeader, Consumer handler) { - LOG.debug("{}: suggested new leader: {}. Failed {} with {}", - clientId, newLeader, request, ioe); + LOG.debug("{}: suggested new leader: {}. Failed {}", clientId, newLeader, request, ioe); if (LOG.isTraceEnabled()) { LOG.trace("Stack trace", new Throwable("TRACE")); } @@ -454,8 +452,17 @@ public RaftClientRpc getClientRpc() { return clientRpc; } + boolean isClosed() { + return closed.get(); + } + @Override public void close() throws IOException { + if (!closed.compareAndSet(false, true)) { + return; + } + + LOG.debug("close {}", getId()); clientRpc.close(); if (dataStreamApi.isInitialized()) { dataStreamApi.get().close(); diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java index 1762dc0e49..65c54d0f21 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/SnapshotManagementImpl.java @@ -37,9 +37,10 @@ class SnapshotManagementImpl implements SnapshotManagementApi { } @Override - public RaftClientReply create(long timeoutMs) throws IOException { + public RaftClientReply create(long creationGap, long timeoutMs) throws IOException { final long callId = CallId.getAndIncrement(); return client.io().sendRequestWithRetry(() -> SnapshotManagementRequest.newCreate(client.getId(), - Optional.ofNullable(server).orElseGet(client::getLeaderId), client.getGroupId(), callId, timeoutMs)); + Optional.ofNullable(server).orElseGet(client::getLeaderId), + client.getGroupId(), callId, timeoutMs, creationGap)); } } diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java index 84b817b581..eccda4dbdd 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/UnorderedAsync.java @@ -22,6 +22,7 @@ import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.exceptions.AlreadyClosedException; import org.apache.ratis.protocol.exceptions.GroupMismatchException; import org.apache.ratis.protocol.exceptions.NotLeaderException; import org.apache.ratis.protocol.RaftClientReply; @@ -89,11 +90,14 @@ static void sendRequestWithRetry(PendingClientRequest pending, RaftClientImpl cl } final Throwable cause = replyException != null ? replyException : e; - pending.incrementExceptionCount(cause); - final ClientRetryEvent event = new ClientRetryEvent(request, cause, pending); + if (client.isClosed()) { + f.completeExceptionally(new AlreadyClosedException(client + " is closed")); + return; + } + + final ClientRetryEvent event = pending.newClientRetryEvent(request, cause); RetryPolicy retryPolicy = client.getRetryPolicy(); final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event); - TimeDuration sleepTime = client.getEffectiveSleepTime(cause, action.getSleepTime()); if (!action.shouldRetry()) { f.completeExceptionally(client.noMoreRetries(event)); return; @@ -124,7 +128,9 @@ static void sendRequestWithRetry(PendingClientRequest pending, RaftClientImpl cl } } - LOG.debug("schedule retry for attempt #{}, policy={}, request={}", attemptCount, retryPolicy, request); + final TimeDuration sleepTime = client.getEffectiveSleepTime(cause, action.getSleepTime()); + LOG.debug("schedule~ attempt #{} with sleep {} and policy {} for {}", + attemptCount, sleepTime, retryPolicy, request); client.getScheduler().onTimeout(sleepTime, () -> sendRequestWithRetry(pending, client), LOG, () -> clientId + ": Failed~ to retry " + request); } catch (Exception ex) { diff --git a/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java b/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java index f0c38efb96..c6a8beb06f 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/retry/ClientRetryEvent.java @@ -17,12 +17,11 @@ */ package org.apache.ratis.client.retry; -import org.apache.ratis.client.impl.RaftClientImpl.PendingClientRequest; import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.retry.RetryPolicy; -import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; +import org.apache.ratis.util.Timestamp; /** An {@link RetryPolicy.Event} specific to client request failure. */ public class ClientRetryEvent implements RetryPolicy.Event { @@ -30,23 +29,15 @@ public class ClientRetryEvent implements RetryPolicy.Event { private final int causeCount; private final RaftClientRequest request; private final Throwable cause; - private PendingClientRequest pending; + private final Timestamp pendingRequestCreationTime; - @VisibleForTesting - public ClientRetryEvent(int attemptCount, RaftClientRequest request, Throwable cause) { - this(attemptCount, request, attemptCount, cause); - } - - public ClientRetryEvent(RaftClientRequest request, Throwable t, PendingClientRequest pending) { - this(pending.getAttemptCount(), request, pending.getExceptionCount(t), t); - this.pending = pending; - } - - private ClientRetryEvent(int attemptCount, RaftClientRequest request, int causeCount, Throwable cause) { + public ClientRetryEvent(int attemptCount, RaftClientRequest request, int causeCount, Throwable cause, + Timestamp pendingRequestCreationTime) { this.attemptCount = attemptCount; this.causeCount = causeCount; this.request = request; this.cause = cause; + this.pendingRequestCreationTime = pendingRequestCreationTime; } @Override @@ -69,7 +60,7 @@ public Throwable getCause() { } boolean isRequestTimeout(TimeDuration timeout) { - return pending != null && pending.isRequestTimeout(timeout); + return timeout != null && pendingRequestCreationTime.elapsedTime().compareTo(timeout) >= 0; } @Override @@ -77,6 +68,7 @@ public String toString() { return JavaUtils.getClassSimpleName(getClass()) + ":attempt=" + attemptCount + ",request=" + request - + ",cause=" + cause; + + ",cause=" + cause + + ",causeCount=" + causeCount; } } diff --git a/ratis-common/pom.xml b/ratis-common/pom.xml index 9bb36bcce5..b5676f2c9c 100644 --- a/ratis-common/pom.xml +++ b/ratis-common/pom.xml @@ -17,7 +17,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-common @@ -69,12 +69,5 @@ junit-jupiter-params test - - - com.github.spotbugs - spotbugs-annotations - provided - true - diff --git a/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java b/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java index c1fb9268c9..43706faabc 100644 --- a/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/conf/ConfUtils.java @@ -18,7 +18,6 @@ package org.apache.ratis.conf; import org.apache.ratis.security.TlsConf; -import org.apache.ratis.thirdparty.com.google.common.base.Objects; import org.apache.ratis.util.NetUtils; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; @@ -33,6 +32,9 @@ import java.net.InetSocketAddress; import java.util.Arrays; import java.util.List; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -41,10 +43,24 @@ public interface ConfUtils { Logger LOG = LoggerFactory.getLogger(ConfUtils.class); + class Utils { + private static final ConcurrentMap CACHE = new ConcurrentHashMap<>(); + + private static boolean isNew(String key, T value) { + if (value == null) { + final Object previous = CACHE.remove(key); + return previous != null; + } else { + final Object previous = CACHE.put(key, value); + return !value.equals(previous); + } + } + } + static void logGet(String key, T value, T defaultValue, Consumer logger) { - if (logger != null) { + if (logger != null && Utils.isNew(key, value)) { logger.accept(String.format("%s = %s (%s)", key, value, - Objects.equal(value, defaultValue)? "default": "custom")); + Objects.equals(value, defaultValue)? "default": "custom")); } } diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamRequestByteBuf.java b/ratis-common/src/main/java/org/apache/ratis/datastream/impl/DataStreamRequestByteBuf.java similarity index 96% rename from ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamRequestByteBuf.java rename to ratis-common/src/main/java/org/apache/ratis/datastream/impl/DataStreamRequestByteBuf.java index 2542b1ec6f..1873bec9b4 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamRequestByteBuf.java +++ b/ratis-common/src/main/java/org/apache/ratis/datastream/impl/DataStreamRequestByteBuf.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package org.apache.ratis.netty.server; +package org.apache.ratis.datastream.impl; -import org.apache.ratis.datastream.impl.DataStreamPacketImpl; import org.apache.ratis.io.WriteOption; import org.apache.ratis.proto.RaftProtos.DataStreamPacketHeaderProto.Type; import org.apache.ratis.protocol.ClientId; diff --git a/ratis-common/src/main/java/org/apache/ratis/io/MD5Hash.java b/ratis-common/src/main/java/org/apache/ratis/io/MD5Hash.java index e60bef9652..0d19feb938 100644 --- a/ratis-common/src/main/java/org/apache/ratis/io/MD5Hash.java +++ b/ratis-common/src/main/java/org/apache/ratis/io/MD5Hash.java @@ -30,13 +30,15 @@ public class MD5Hash { public static final int MD5_LEN = 16; private static final ThreadLocal DIGESTER_FACTORY = - ThreadLocal.withInitial(() -> { + ThreadLocal.withInitial(MD5Hash::newDigester); + + public static MessageDigest newDigester() { try { return MessageDigest.getInstance("MD5"); } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to create MessageDigest for MD5", e); } - }); + } private byte[] digest; diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/ClientId.java b/ratis-common/src/main/java/org/apache/ratis/protocol/ClientId.java index 4de615730c..09b77e6e81 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/ClientId.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/ClientId.java @@ -18,6 +18,7 @@ package org.apache.ratis.protocol; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.util.WeakValueCache; import java.util.UUID; @@ -26,13 +27,17 @@ * to correctly identify retry requests from the same client. */ public final class ClientId extends RaftId { - private static final Factory FACTORY = new Factory() { + private static final Factory FACTORY = new Factory(ClientId.class) { @Override ClientId newInstance(UUID uuid) { return new ClientId(uuid); } }; + static WeakValueCache getCache() { + return FACTORY.getCache(); + } + public static ClientId emptyClientId() { return FACTORY.emptyId(); } diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/GroupInfoReply.java b/ratis-common/src/main/java/org/apache/ratis/protocol/GroupInfoReply.java index 632fa65293..bfac81a2b0 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/GroupInfoReply.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/GroupInfoReply.java @@ -19,6 +19,7 @@ import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto; import org.apache.ratis.proto.RaftProtos.CommitInfoProto; +import org.apache.ratis.proto.RaftProtos.LogInfoProto; import org.apache.ratis.proto.RaftProtos.RoleInfoProto; import java.util.Collection; @@ -33,25 +34,27 @@ public class GroupInfoReply extends RaftClientReply { private final RoleInfoProto roleInfoProto; private final boolean isRaftStorageHealthy; private final RaftConfigurationProto conf; + private final LogInfoProto logInfoProto; public GroupInfoReply(RaftClientRequest request, Collection commitInfos, RaftGroup group, RoleInfoProto roleInfoProto, boolean isRaftStorageHealthy, - RaftConfigurationProto conf) { + RaftConfigurationProto conf, LogInfoProto logInfoProto) { this(request.getClientId(), request.getServerId(), request.getRaftGroupId(), request.getCallId(), commitInfos, - group, roleInfoProto, isRaftStorageHealthy, conf); + group, roleInfoProto, isRaftStorageHealthy, conf, logInfoProto); } @SuppressWarnings("parameternumber") public GroupInfoReply(ClientId clientId, RaftPeerId serverId, RaftGroupId groupId, long callId, Collection commitInfos, RaftGroup group, RoleInfoProto roleInfoProto, boolean isRaftStorageHealthy, - RaftConfigurationProto conf) { + RaftConfigurationProto conf, LogInfoProto logInfoProto) { super(clientId, serverId, groupId, callId, true, null, null, 0L, commitInfos); this.group = group; this.roleInfoProto = roleInfoProto; this.isRaftStorageHealthy = isRaftStorageHealthy; this.conf = conf; + this.logInfoProto = logInfoProto; } public RaftGroup getGroup() { @@ -69,4 +72,8 @@ public boolean isRaftStorageHealthy() { public Optional getConf() { return Optional.ofNullable(conf); } + + public LogInfoProto getLogInfoProto() { + return logInfoProto; + } } diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientAsynchronousProtocol.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientAsynchronousProtocol.java index 1985bbe667..222ccff057 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientAsynchronousProtocol.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientAsynchronousProtocol.java @@ -46,11 +46,11 @@ default CompletableFuture submitClientRequestAsync( ReferenceCountedObject requestRef) { try { // for backward compatibility - return submitClientRequestAsync(requestRef.retain()) - .whenComplete((r, e) -> requestRef.release()); + return submitClientRequestAsync(requestRef.retain()); } catch (Exception e) { - requestRef.release(); return JavaUtils.completeExceptionally(e); + } finally { + requestRef.release(); } } } \ No newline at end of file diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientMessage.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientMessage.java index 8d3104a73d..92ae77ce21 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientMessage.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientMessage.java @@ -18,7 +18,8 @@ package org.apache.ratis.protocol; import org.apache.ratis.util.JavaUtils; -import org.apache.ratis.util.Preconditions; + +import java.util.Objects; public abstract class RaftClientMessage implements RaftRpcMessage { private final ClientId clientId; @@ -27,9 +28,9 @@ public abstract class RaftClientMessage implements RaftRpcMessage { private final long callId; RaftClientMessage(ClientId clientId, RaftPeerId serverId, RaftGroupId groupId, long callId) { - this.clientId = Preconditions.assertNotNull(clientId, "clientId"); - this.serverId = Preconditions.assertNotNull(serverId, "serverId"); - this.groupId = Preconditions.assertNotNull(groupId, "groupId"); + this.clientId = Objects.requireNonNull(clientId, "clientId == null"); + this.serverId = Objects.requireNonNull(serverId, "serverId == null"); + this.groupId = Objects.requireNonNull(groupId, "groupId == null"); this.callId = callId; } diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientReply.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientReply.java index 64d667955d..8d9980a532 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientReply.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientReply.java @@ -187,11 +187,6 @@ public Collection getCommitInfos() { return commitInfos; } - @Override - public final boolean isRequest() { - return false; - } - public long getLogIndex() { return logIndex; } diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientRequest.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientRequest.java index ed41f1ea2c..18c157130b 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientRequest.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftClientRequest.java @@ -474,7 +474,13 @@ public long getTimeoutMs() { @Override public String toString() { - return super.toString() + ", seq=" + ProtoUtils.toString(slidingWindowEntry) + ", " - + type + ", " + getMessage(); + return toStringShort() + ", " + getMessage(); + } + + /** + * @return a short string which does not include {@link #message}. + */ + public String toStringShort() { + return super.toString() + ", seq=" + ProtoUtils.toString(slidingWindowEntry) + ", " + type; } } diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroup.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroup.java index 0612a16f9d..5cf970afc3 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroup.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroup.java @@ -19,7 +19,13 @@ import org.apache.ratis.util.Preconditions; -import java.util.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + /** * Description of a raft group, which has a unique {@link RaftGroupId} and a collection of {@link RaftPeer}. diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroupId.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroupId.java index 9caedf7574..af40746918 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroupId.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftGroupId.java @@ -18,6 +18,7 @@ package org.apache.ratis.protocol; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.util.WeakValueCache; import java.util.UUID; @@ -27,13 +28,17 @@ * This is a value-based class. */ public final class RaftGroupId extends RaftId { - private static final Factory FACTORY = new Factory() { + private static final Factory FACTORY = new Factory(RaftGroupId.class) { @Override RaftGroupId newInstance(UUID uuid) { return new RaftGroupId(uuid); } }; + static WeakValueCache getCache() { + return FACTORY.getCache(); + } + public static RaftGroupId emptyGroupId() { return FACTORY.emptyId(); } diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftId.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftId.java index 9c2a83ffa3..d089c7d3cb 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftId.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftId.java @@ -17,17 +17,15 @@ */ package org.apache.ratis.protocol; -import org.apache.ratis.thirdparty.com.google.common.cache.Cache; -import org.apache.ratis.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.WeakValueCache; import java.nio.ByteBuffer; import java.util.Objects; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.function.Supplier; /** Unique identifier implemented using {@link UUID}. */ @@ -53,18 +51,20 @@ static ByteString toByteString(UUID uuid) { } abstract static class Factory { - private final Cache cache = CacheBuilder.newBuilder() - .weakValues() - .build(); + private final WeakValueCache cache; + + Factory(Class clazz) { + this.cache = new WeakValueCache<>(clazz.getSimpleName() + "_UUID", this::newInstance); + } abstract ID newInstance(UUID uuid); + WeakValueCache getCache() { + return cache; + } + final ID valueOf(UUID uuid) { - try { - return cache.get(uuid, () -> newInstance(uuid)); - } catch (ExecutionException e) { - throw new IllegalStateException("Failed to valueOf(" + uuid + ")", e); - } + return cache.getOrCreate(uuid); } final ID valueOf(ByteString bytes) { @@ -85,7 +85,7 @@ ID randomId() { private final Supplier uuidString; RaftId(UUID uuid) { - this.uuid = Preconditions.assertNotNull(uuid, "uuid"); + this.uuid = Objects.requireNonNull(uuid, "uuid == null"); this.uuidBytes = JavaUtils.memoize(() -> toByteString(uuid)); this.uuidString = JavaUtils.memoize(() -> createUuidString(uuid)); Preconditions.assertTrue(ZERO_UUID == uuid || !uuid.equals(ZERO_UUID), diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftPeerId.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftPeerId.java index 8098039d1b..8db842d734 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftPeerId.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftPeerId.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.protocol; -import javax.annotation.concurrent.Immutable; import org.apache.ratis.proto.RaftProtos.RaftPeerIdProto; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.util.JavaUtils; @@ -34,7 +33,6 @@ *

* This is a value-based class. */ -@Immutable public final class RaftPeerId { private static final Map BYTE_STRING_MAP = new ConcurrentHashMap<>(); private static final Map STRING_MAP = new ConcurrentHashMap<>(); diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftRpcMessage.java b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftRpcMessage.java index 70727af2e2..7c497129d7 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/RaftRpcMessage.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/RaftRpcMessage.java @@ -19,7 +19,9 @@ public interface RaftRpcMessage { - boolean isRequest(); + default boolean isRequest() { + return false; + } String getRequestorId(); diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java b/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java index 2ea2059b51..269fdfc591 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/SnapshotManagementRequest.java @@ -24,7 +24,16 @@ public final class SnapshotManagementRequest extends RaftClientRequest { public abstract static class Op { } - public static class Create extends Op { + + public static final class Create extends Op { + private final long creationGap; + private Create(long creationGap) { + this.creationGap = creationGap; + } + + public long getCreationGap() { + return creationGap; + } @Override public String toString() { @@ -35,8 +44,13 @@ public String toString() { public static SnapshotManagementRequest newCreate(ClientId clientId, RaftPeerId serverId, RaftGroupId groupId, long callId, long timeoutMs) { + return newCreate(clientId, serverId, groupId, callId, timeoutMs, 0); + } + + public static SnapshotManagementRequest newCreate(ClientId clientId, + RaftPeerId serverId, RaftGroupId groupId, long callId, long timeoutMs, long creationGap) { return new SnapshotManagementRequest(clientId, - serverId, groupId, callId, timeoutMs,new SnapshotManagementRequest.Create()); + serverId, groupId, callId, timeoutMs, new SnapshotManagementRequest.Create(creationGap)); } private final Op op; diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotLeaderException.java b/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotLeaderException.java index 8d5c2cb4e9..c7dc6a3961 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotLeaderException.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotLeaderException.java @@ -30,7 +30,8 @@ public class NotLeaderException extends RaftException { private final Collection peers; public NotLeaderException(RaftGroupMemberId memberId, RaftPeer suggestedLeader, Collection peers) { - super("Server " + memberId + " is not the leader" + (suggestedLeader != null? " " + suggestedLeader: "")); + super("Server " + memberId + " is not the leader" + + (suggestedLeader != null ? ", suggested leader is: " + suggestedLeader : "")); this.suggestedLeader = suggestedLeader; this.peers = peers != null? Collections.unmodifiableCollection(peers): Collections.emptyList(); Preconditions.assertUnique(this.peers); diff --git a/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotReplicatedException.java b/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotReplicatedException.java index 5f48654eec..37ff816245 100644 --- a/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotReplicatedException.java +++ b/ratis-common/src/main/java/org/apache/ratis/protocol/exceptions/NotReplicatedException.java @@ -17,12 +17,17 @@ */ package org.apache.ratis.protocol.exceptions; +import org.apache.ratis.proto.RaftProtos.CommitInfoProto; import org.apache.ratis.proto.RaftProtos.ReplicationLevel; +import java.util.Collection; + public class NotReplicatedException extends RaftException { private final long callId; private final ReplicationLevel requiredReplication; private final long logIndex; + /** This is only populated on client-side since RaftClientReply already has commitInfos */ + private Collection commitInfos; public NotReplicatedException(long callId, ReplicationLevel requiredReplication, long logIndex) { super("Request with call Id " + callId + " and log index " + logIndex @@ -32,6 +37,12 @@ public NotReplicatedException(long callId, ReplicationLevel requiredReplication, this.logIndex = logIndex; } + public NotReplicatedException(long callId, ReplicationLevel requiredReplication, long logIndex, + Collection commitInfos) { + this(callId, requiredReplication, logIndex); + this.commitInfos = commitInfos; + } + public long getCallId() { return callId; } @@ -43,4 +54,8 @@ public ReplicationLevel getRequiredReplication() { public long getLogIndex() { return logIndex; } + + public Collection getCommitInfos() { + return commitInfos; + } } diff --git a/ratis-common/src/main/java/org/apache/ratis/retry/ExponentialBackoffRetry.java b/ratis-common/src/main/java/org/apache/ratis/retry/ExponentialBackoffRetry.java index bb2f50e43a..d506c85c80 100644 --- a/ratis-common/src/main/java/org/apache/ratis/retry/ExponentialBackoffRetry.java +++ b/ratis-common/src/main/java/org/apache/ratis/retry/ExponentialBackoffRetry.java @@ -17,9 +17,9 @@ */ package org.apache.ratis.retry; -import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.TimeDuration; +import java.util.Objects; import java.util.concurrent.ThreadLocalRandom; /** @@ -56,7 +56,7 @@ public Builder setMaxSleepTime(TimeDuration maxSleepTime) { } public ExponentialBackoffRetry build() { - Preconditions.assertNotNull(baseSleepTime, "baseSleepTime"); + Objects.requireNonNull(baseSleepTime, "baseSleepTime == null"); return new ExponentialBackoffRetry(baseSleepTime, maxSleepTime, maxAttempts); } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/AtomicFileOutputStream.java b/ratis-common/src/main/java/org/apache/ratis/util/AtomicFileOutputStream.java index 530eb383c7..b54427066f 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/AtomicFileOutputStream.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/AtomicFileOutputStream.java @@ -17,6 +17,10 @@ */ package org.apache.ratis.util; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,7 +28,6 @@ import java.io.FilterOutputStream; import java.io.IOException; import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -60,7 +63,7 @@ public AtomicFileOutputStream(File outFile) throws IOException { } public AtomicFileOutputStream(File outFile, File tmpFile) throws IOException { - super(FileUtils.newOutputStreamForceAtClose(tmpFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE)); + super(FileUtils.newOutputStreamForceAtClose(tmpFile, CREATE, TRUNCATE_EXISTING, WRITE)); this.outFile = outFile.getAbsoluteFile(); this.tmpFile = tmpFile.getAbsoluteFile(); } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/AutoCloseableLock.java b/ratis-common/src/main/java/org/apache/ratis/util/AutoCloseableLock.java index 8a5409bafe..9581e925a5 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/AutoCloseableLock.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/AutoCloseableLock.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.util; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; @@ -45,6 +46,13 @@ public static AutoCloseableLock acquire(final Lock lock, Runnable preUnlock) { return new AutoCloseableLock(lock, preUnlock); } + public static AutoCloseableLock tryAcquire(final Lock lock, Runnable preUnlock, TimeDuration timeout) + throws InterruptedException { + Objects.requireNonNull(timeout, "timeout == null"); + final boolean locked = lock.tryLock(timeout.getDuration(), timeout.getUnit()); + return locked? new AutoCloseableLock(lock, preUnlock): null; + } + private final Lock underlying; private final AtomicBoolean closed = new AtomicBoolean(false); private final Runnable preUnlock; diff --git a/ratis-common/src/main/java/org/apache/ratis/util/BatchLogger.java b/ratis-common/src/main/java/org/apache/ratis/util/BatchLogger.java index 9ccd66ad71..b57bed704c 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/BatchLogger.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/BatchLogger.java @@ -35,13 +35,19 @@ public final class BatchLogger { private BatchLogger() { } - public interface Key {} + public interface Key { + TimeDuration DEFAULT_DURATION = TimeDuration.valueOf(5, TimeUnit.SECONDS); + + default TimeDuration getBatchDuration() { + return DEFAULT_DURATION; + } + } private static final class UniqueId { private final Key key; - private final String name; + private final Object name; - private UniqueId(Key key, String name) { + private UniqueId(Key key, Object name) { this.key = Objects.requireNonNull(key, "key == null"); this.name = name; } @@ -93,11 +99,15 @@ private synchronized boolean tryStartBatch(Consumer op) { private static final TimeoutExecutor SCHEDULER = TimeoutExecutor.getInstance(); private static final ConcurrentMap LOG_CACHE = new ConcurrentHashMap<>(); - public static void warn(Key key, String name, Consumer op, TimeDuration batchDuration) { - warn(key, name, op, batchDuration, true); + public static void print(Key key, Object name, Consumer op) { + print(key, name, op, key.getBatchDuration(), true); + } + + public static void print(Key key, Object name, Consumer op, TimeDuration batchDuration) { + print(key, name, op, batchDuration, true); } - public static void warn(Key key, String name, Consumer op, TimeDuration batchDuration, boolean shouldBatch) { + public static void print(Key key, Object name, Consumer op, TimeDuration batchDuration, boolean shouldBatch) { if (!shouldBatch || batchDuration.isNonPositive()) { op.accept(""); return; diff --git a/ratis-common/src/main/java/org/apache/ratis/util/BiWeakValueCache.java b/ratis-common/src/main/java/org/apache/ratis/util/BiWeakValueCache.java new file mode 100644 index 0000000000..d7eaf5744a --- /dev/null +++ b/ratis-common/src/main/java/org/apache/ratis/util/BiWeakValueCache.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.util; + +import org.apache.ratis.thirdparty.com.google.common.collect.MapMaker; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +/** + * Weak Value Cache: ({@link OUTER}, {@link INNER}) -> {@link T}. + *

+ * Note that the cached values are weakly referenced. + * A cached value could be garage-collected (i.e. evicted from the cache) + * when there are no external (strong) references. + *

+ * For key types with a component, use {@link WeakValueCache}. + * + * @param the type of the outer keys. + * @param the type of the inner keys. + * @param the type to be cached. + */ +public final class BiWeakValueCache { + static ConcurrentMap newMap() { + return new MapMaker().weakValues().makeMap(); + } + + private final String outerName; + private final String innerName; + private final String name; + + /** For constructing {@link T} values from ({@link OUTER}, {@link INNER}) keys. */ + private final BiFunction constructor; + /** Count the number of {@link T} values constructed. */ + private final AtomicInteger valueCount = new AtomicInteger(0); + + /** + * Actual map {@link OUTER} -> ({@link INNER} -> {@link T}) + * for the logical view ({@link OUTER}, {@link INNER}) -> {@link T}. + */ + private final ConcurrentMap> map = new ConcurrentHashMap<>(); + + /** + * Create a cache for mapping ({@link OUTER}, {@link INNER}) keys to {@link T} values. + * + * @param outerName the name of the outer keys. + * @param innerName the name of the inner keys. + * @param constructor for constructing {@link T} values. + */ + public BiWeakValueCache(String outerName, String innerName, BiFunction constructor) { + this.outerName = outerName; + this.innerName = innerName; + this.name = "(" + outerName + ", " + innerName + ")-cache"; + this.constructor = constructor; + } + + private T construct(OUTER outer, INNER inner) { + final T constructed = constructor.apply(outer, inner); + Objects.requireNonNull(constructed, "constructed == null"); + valueCount.incrementAndGet(); + return constructed; + } + + /** + * If the key ({@link OUTER}, {@link INNER}) is in the cache, return the cached values. + * Otherwise, create a new value and then return it. + */ + public T getOrCreate(OUTER outer, INNER inner) { + Objects.requireNonNull(outer, () -> outerName + " (outer) == null"); + Objects.requireNonNull(inner, () -> innerName + " (inner) == null"); + final ConcurrentMap innerMap = map.computeIfAbsent(outer, k -> newMap()); + final T computed = innerMap.computeIfAbsent(inner, i -> construct(outer, i)); + if ((valueCount.get() & 0xFFF) == 0) { + cleanupEmptyInnerMaps(); // cleanup empty maps once in a while + } + return computed; + } + + /** @return the value count for the given outer key. */ + int count(OUTER outer) { + final ConcurrentMap innerMap = map.get(outer); + if (innerMap == null) { + return 0; + } + + // size() may return incorrect result; see Guava MapMaker javadoc + int n = 0; + for (INNER ignored : innerMap.keySet()) { + n++; + } + return n; + } + + void cleanupEmptyInnerMaps() { + // isEmpty() may return incorrect result; see Guava MapMaker javadoc + map.values().removeIf(e -> !e.entrySet().iterator().hasNext()); + } + + @Override + public String toString() { + return name; + } + + /** The cache content for debugging. */ + int dump(Consumer out) { + out.accept(name + ":\n"); + int emptyCount = 0; + for (Map.Entry> entry : map.entrySet()) { + final OUTER outer = entry.getKey(); + final ConcurrentMap innerMap = entry.getValue(); + final int count = count(outer); + if (count == 0) { + emptyCount++; + } + + out.accept(" " + outerName + ":" + outer); + out.accept(", " + innerName + ":" + innerMap.keySet()); + out.accept(", count=" + count); + out.accept(", size=" + innerMap.size()); + out.accept("\n"); + } + out.accept(" emptyCount=" + emptyCount); + out.accept("\n"); + return emptyCount; + } +} diff --git a/ratis-common/src/main/java/org/apache/ratis/util/CodeInjectionForTesting.java b/ratis-common/src/main/java/org/apache/ratis/util/CodeInjectionForTesting.java index a7d36ac0eb..112f6bd250 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/CodeInjectionForTesting.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/CodeInjectionForTesting.java @@ -68,4 +68,9 @@ public static boolean execute(String injectionPoint, Object localId, } return code.execute(localId, remoteId, args); } + + /** Remove an injection point. */ + public static void remove(String injectionPoint) { + INJECTION_POINTS.remove(injectionPoint); + } } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/CollectionUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/CollectionUtils.java index 11f484608a..2615c2659c 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/CollectionUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/CollectionUtils.java @@ -17,7 +17,14 @@ */ package org.apache.ratis.util; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadLocalRandom; import java.util.function.BiFunction; diff --git a/ratis-common/src/main/java/org/apache/ratis/util/DataBlockingQueue.java b/ratis-common/src/main/java/org/apache/ratis/util/DataBlockingQueue.java index 842b8f1549..fb0f0715c5 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/DataBlockingQueue.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/DataBlockingQueue.java @@ -29,6 +29,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.function.ToLongFunction; /** @@ -46,6 +47,8 @@ public class DataBlockingQueue extends DataQueue { private final Condition notFull = lock.newCondition(); private final Condition notEmpty = lock.newCondition(); + private boolean closed = false; + public DataBlockingQueue(Object name, SizeInBytes byteLimit, int elementLimit, ToLongFunction getNumBytes) { super(name, byteLimit, elementLimit, getNumBytes); } @@ -72,10 +75,34 @@ public void clear() { } } + /** Apply the given handler to each element and then {@link #clear()}. */ + public void clear(Consumer handler) { + try(AutoCloseableLock auto = AutoCloseableLock.acquire(lock)) { + for(E e : this) { + handler.accept(e); + } + super.clear(); + } + } + + /** + * Close this queue to stop accepting new elements, i.e. the offer(…) methods always return false. + * Note that closing the queue will not clear the existing elements. + * The existing elements can be peeked, polled or cleared after close. + */ + public void close() { + try(AutoCloseableLock ignored = AutoCloseableLock.acquire(lock)) { + closed = true; + } + } + @Override public boolean offer(E element) { Objects.requireNonNull(element, "element == null"); try(AutoCloseableLock auto = AutoCloseableLock.acquire(lock)) { + if (closed) { + return false; + } if (super.offer(element)) { notEmpty.signal(); return true; @@ -95,6 +122,9 @@ public boolean offer(E element, TimeDuration timeout) throws InterruptedExceptio long nanos = timeout.toLong(TimeUnit.NANOSECONDS); try(AutoCloseableLock auto = AutoCloseableLock.acquire(lock)) { for(;;) { + if (closed) { + return false; + } if (super.offer(element)) { notEmpty.signal(); return true; @@ -162,4 +192,11 @@ public List pollList(long timeoutM return results; } } + + @Override + public E peek() { + try(AutoCloseableLock auto = AutoCloseableLock.acquire(lock)) { + return super.peek(); + } + } } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/DataQueue.java b/ratis-common/src/main/java/org/apache/ratis/util/DataQueue.java index 3db06f56e6..38762caa17 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/DataQueue.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/DataQueue.java @@ -154,6 +154,11 @@ public E poll() { return polled; } + /** Peek the head element from this queue. */ + public E peek() { + return q.peek(); + } + /** The same as {@link java.util.Collection#remove(Object)}. */ public boolean remove(E e) { final boolean removed = q.remove(e); diff --git a/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java index d5141e9171..4b9d9e3b28 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/FileUtils.java @@ -27,7 +27,18 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.nio.file.*; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import java.util.List; @@ -201,8 +212,10 @@ static File move(File src, String suffix) throws IOException { } /** The same as passing f.toPath() to {@link #delete(Path)}. */ - static void deleteFile(File f) throws IOException { - delete(f.toPath()); + static Path deleteFile(File f) throws IOException { + final Path path = f.toPath(); + delete(path); + return path; } /** diff --git a/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java index 00725903a7..958e88cee5 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/JavaUtils.java @@ -148,7 +148,7 @@ static T doPrivileged(Supplier action, Supplier name) { * otherwise, return system property value. */ static String getSystemProperty(final String key) { - Preconditions.assertNotNull(key, "key"); + Objects.requireNonNull(key, "key == null"); Preconditions.assertTrue(!key.isEmpty(), "key is empty."); return doPrivileged(() -> System.getProperty(key), () -> "get system property " + key); } @@ -166,9 +166,9 @@ static String getEnv(String variable) { * When there is a {@link SecurityException}, this becomes a NOOP. */ static void setSystemProperty(String key, String value) { - Preconditions.assertNotNull(key, "key"); + Objects.requireNonNull(key, "key == null"); Preconditions.assertTrue(!key.isEmpty(), "key is empty."); - Preconditions.assertNotNull(value, "value"); + Objects.requireNonNull(value, "value == null"); doPrivileged(() -> System.setProperty(key, value), () -> "set system property " + key + " to " + value); } @@ -229,7 +229,7 @@ static RETURN attempt( } if (log != null && log.isWarnEnabled()) { log.warn("FAILED \"" + name.get() + "\", attempt #" + i + "/" + numAttempts - + ": " + t + ", sleep " + sleepTime + " and then retry.", t); + + ", sleep " + sleepTime + " and then retry: " + t); } } @@ -257,7 +257,6 @@ static void attemptUntilTrue( }, numAttempts, sleepTime, name, log); } - static Timer runRepeatedly(Runnable runnable, long delay, long period, TimeUnit unit) { final Timer timer = new Timer(true); timer.schedule(new TimerTask() { @@ -283,6 +282,10 @@ static CompletableFuture completeExceptionally(Throwable t) { return future; } + static boolean isCompletedNormally(CompletableFuture future) { + return future.isDone() && !future.isCancelled() && !future.isCompletedExceptionally(); + } + static Throwable unwrapCompletionException(Throwable t) { Objects.requireNonNull(t, "t == null"); return t instanceof CompletionException && t.getCause() != null? t.getCause(): t; diff --git a/ratis-common/src/main/java/org/apache/ratis/util/JmxRegister.java b/ratis-common/src/main/java/org/apache/ratis/util/JmxRegister.java index 54f7989245..4554410488 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/JmxRegister.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/JmxRegister.java @@ -46,15 +46,16 @@ static ObjectName tryRegister(String name, Object mBean) { } /** - * Try registering the mBean with the names one by one. + * Try registering the mxBean with the names one by one. * @return the registered name, or, if it fails, return null. */ - public synchronized String register(Object mBean, Iterable> names) { + public synchronized String register(Object mxBean, Iterable> names) { if (registeredName == null) { for (Supplier supplier : names) { final String name = supplier.get(); - registeredName = tryRegister(name, mBean); + registeredName = tryRegister(name, mxBean); if (registeredName != null) { + LOG.info("register mxBean {} as {}", mxBean.getClass(), name); return name; } } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/LeakDetector.java b/ratis-common/src/main/java/org/apache/ratis/util/LeakDetector.java new file mode 100644 index 0000000000..6f12205795 --- /dev/null +++ b/ratis-common/src/main/java/org/apache/ratis/util/LeakDetector.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * Simple general resource leak detector using {@link ReferenceQueue} and {@link java.lang.ref.WeakReference} to + * observe resource object life-cycle and assert proper resource closure before they are GCed. + * + *

+ * Example usage: + * + * 

 {@code
+ * class MyResource implements AutoClosable {
+ *   static final LeakDetector LEAK_DETECTOR = new LeakDetector("MyResource");
+ *
+ *   private final UncheckedAutoCloseable leakTracker = LEAK_DETECTOR.track(this, () -> {
+ *      // report leaks, don't refer to the original object (MyResource) here.
+ *      System.out.println("MyResource is not closed before being discarded.");
+ *   });
+ *
+ *   @Override
+ *   public void close() {
+ *     // proper resources cleanup...
+ *     // inform tracker that this object is closed properly.
+ *     leakTracker.close();
+ *   }
+ * }
+ *
+ * }
+ */ +public class LeakDetector { + private static final Logger LOG = LoggerFactory.getLogger(LeakDetector.class); + + private static class LeakTrackerSet { + private final Set set = Collections.newSetFromMap(new HashMap<>()); + + synchronized boolean remove(LeakTracker tracker) { + return set.remove(tracker); + } + + synchronized void removeExisting(LeakTracker tracker) { + final boolean removed = set.remove(tracker); + Preconditions.assertTrue(removed, () -> "Failed to remove existing " + tracker); + } + + synchronized LeakTracker add(Object referent, ReferenceQueue queue, Supplier leakReporter) { + final LeakTracker tracker = new LeakTracker(referent, queue, this::removeExisting, leakReporter); + final boolean added = set.add(tracker); + Preconditions.assertTrue(added, () -> "Failed to add " + tracker + " for " + referent); + return tracker; + } + + synchronized int getNumLeaks(boolean throwException) { + if (set.isEmpty()) { + return 0; + } + + int n = 0; + for (LeakTracker tracker : set) { + if (tracker.reportLeak() != null) { + n++; + } + } + if (throwException) { + assertNoLeaks(n); + } + return n; + } + + synchronized void assertNoLeaks(int leaks) { + Preconditions.assertTrue(leaks == 0, () -> { + final int size = set.size(); + return "#leaks = " + leaks + " > 0, #leaks " + (leaks == size? "==" : "!=") + " set.size = " + size; + }); + } + } + + private static final AtomicLong COUNTER = new AtomicLong(); + + private final ReferenceQueue queue = new ReferenceQueue<>(); + /** All the {@link LeakTracker}s. */ + private final LeakTrackerSet trackers = new LeakTrackerSet(); + /** When a leak is discovered, a message is printed and added to this list. */ + private final List leakMessages = Collections.synchronizedList(new ArrayList<>()); + private final String name; + + LeakDetector(String name) { + this.name = name + COUNTER.getAndIncrement(); + } + + LeakDetector start() { + Thread t = new Thread(this::run); + t.setName(LeakDetector.class.getSimpleName() + "-" + name); + t.setDaemon(true); + LOG.info("Starting leak detector thread {}.", name); + t.start(); + return this; + } + + private void run() { + while (true) { + try { + LeakTracker tracker = (LeakTracker) queue.remove(); + // Original resource already been GCed, if tracker is not closed yet, + // report a leak. + if (trackers.remove(tracker)) { + final String leak = tracker.reportLeak(); + if (leak != null) { + leakMessages.add(leak); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("Thread interrupted, exiting.", e); + break; + } + } + + LOG.warn("Exiting leak detector {}.", name); + } + + Runnable track(Object leakable, Supplier reportLeak) { + // TODO: A rate filter can be put here to only track a subset of all objects, e.g. 5%, 10%, + // if we have proofs that leak tracking impacts performance, or a single LeakDetector + // thread can't keep up with the pace of object allocation. + // For now, it looks effective enough and let keep it simple. + return trackers.add(leakable, queue, reportLeak)::remove; + } + + public int getLeakCount() { + return trackers.getNumLeaks(false); + } + + public void assertNoLeaks(int maxRetries, TimeDuration retrySleep) throws InterruptedException { + synchronized (leakMessages) { + // leakMessages are all the leaks discovered so far. + Preconditions.assertTrue(leakMessages.isEmpty(), + () -> "#leaks = " + leakMessages.size() + "\n" + leakMessages); + } + + for(int i = 0; i < maxRetries; i++) { + final int numLeaks = trackers.getNumLeaks(false); + if (numLeaks == 0) { + return; + } + LOG.warn("{}/{}) numLeaks == {} > 0, will wait and retry ...", i, maxRetries, numLeaks); + retrySleep.sleep(); + } + trackers.getNumLeaks(true); + } + + private static final class LeakTracker extends WeakReference { + private final Consumer removeMethod; + private final Supplier getLeakMessage; + + LeakTracker(Object referent, ReferenceQueue referenceQueue, + Consumer removeMethod, Supplier getLeakMessage) { + super(referent, referenceQueue); + this.removeMethod = removeMethod; + this.getLeakMessage = getLeakMessage; + } + + /** Called by the tracked resource when the object is completely released. */ + void remove() { + removeMethod.accept(this); + } + + /** @return the leak message if there is a leak; return null if there is no leak. */ + String reportLeak() { + return getLeakMessage.get(); + } + } +} diff --git a/ratis-common/src/main/java/org/apache/ratis/util/LogUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/LogUtils.java index d29f1e56ec..63bb08accf 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/LogUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/LogUtils.java @@ -118,7 +118,13 @@ static void warn(Logger log, Supplier message, Throwable t, Class... if (log.isWarnEnabled()) { if (ReflectionUtils.isInstance(t, exceptionClasses)) { // do not print stack trace for known exceptions. - log.warn(message.get() + ": " + t); + final StringBuilder b = new StringBuilder() + .append(message.get()) + .append(": ").append(t); + for(Throwable cause = t.getCause(); cause != null; cause = cause.getCause()) { + b.append("\n Caused by: ").append(cause); + } + log.warn(b.toString()); } else { log.warn(message.get(), t); } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java index cf2d060239..0e9ae44fa5 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedCheckedSupplier.java @@ -44,6 +44,8 @@ public static MemoizedCheckedSupplier initializer; + + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile RETURN value = null; /** diff --git a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java index f179d2dcd0..5c2754a307 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/MemoizedSupplier.java @@ -40,6 +40,7 @@ public static MemoizedSupplier valueOf(Supplier supplier) { } private final Supplier initializer; + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile T value = null; /** diff --git a/ratis-common/src/main/java/org/apache/ratis/util/NetUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/NetUtils.java index a6ce5af792..c5f22c2893 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/NetUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/NetUtils.java @@ -17,11 +17,19 @@ */ package org.apache.ratis.util; +import org.apache.ratis.thirdparty.com.google.common.net.InetAddresses; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.*; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -145,8 +153,12 @@ static String address2String(InetSocketAddress address) { if (address == null) { return null; } - final StringBuilder b = new StringBuilder(address.getHostName()); - if (address.getAddress() instanceof Inet6Address) { + String hostName = address.getHostName(); + final StringBuilder b = new StringBuilder(hostName); + // Surround with '[', ']' only if it is a IPv6 ip - not for a IPv6 host + if (address.getAddress() instanceof Inet6Address && + InetAddresses.isInetAddress(hostName) && + InetAddresses.forString(hostName).getAddress().length == 16) { b.insert(0, '[').append(']'); } return b.append(':').append(address.getPort()).toString(); diff --git a/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java b/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java index 105ecbfb43..eda41a0096 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/PeerProxyMap.java @@ -36,6 +36,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; /** A map from peer id to peer and its proxy. */ public class PeerProxyMap implements RaftPeer.Add, Closeable { @@ -44,6 +45,7 @@ public class PeerProxyMap implements RaftPeer.Add, Clos /** Peer and its proxy. */ private class PeerAndProxy { private final RaftPeer peer; + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile PROXY proxy = null; private final LifeCycle lifeCycle; @@ -65,7 +67,7 @@ PROXY getProxy() throws IOException { throw new AlreadyClosedException(name + " is already " + current); } lifeCycle.startAndTransition( - () -> proxy = createProxy.apply(peer), IOException.class); + () -> proxy = createProxyImpl(peer), IOException.class); } } } @@ -92,6 +94,7 @@ public String toString() { private final Object resetLock = new Object(); private final CheckedFunction createProxy; + private final AtomicBoolean closed = new AtomicBoolean(); public PeerProxyMap(String name, CheckedFunction createProxy) { this.name = name; @@ -102,6 +105,13 @@ public String getName() { return name; } + private PROXY createProxyImpl(RaftPeer peer) throws IOException { + if (closed.get()) { + throw new AlreadyClosedException(name + ": Failed to create proxy for " + peer); + } + return createProxy.apply(peer); + } + public PROXY getProxy(RaftPeerId id) throws IOException { Objects.requireNonNull(id, "id == null"); PeerAndProxy p = peers.get(id); @@ -161,6 +171,10 @@ public boolean handleException(RaftPeerId serverId, Throwable e, boolean reconne @Override public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + final List exceptions = Collections.synchronizedList(new ArrayList<>()); ConcurrentUtils.parallelForEachAsync(peers.values(), pp -> pp.setNullProxyAndClose().map(proxy -> closeProxy(proxy, pp)).ifPresent(exceptions::add), @@ -180,7 +194,7 @@ public void close() { private IOException closeProxy(PROXY proxy, PeerAndProxy pp) { try { - LOG.debug("{}: Closing proxy for peer {}", name, pp); + LOG.debug("{}: Closing proxy {} {} for peer {}", name, proxy.getClass().getSimpleName(), proxy, pp); proxy.close(); return null; } catch (IOException e) { diff --git a/ratis-common/src/main/java/org/apache/ratis/util/Preconditions.java b/ratis-common/src/main/java/org/apache/ratis/util/Preconditions.java index 36e647f0f0..f37b250f53 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/Preconditions.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/Preconditions.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; @@ -77,6 +78,11 @@ static void assertTrue(boolean value, Supplier message) { } } + static void assertSame(int expected, int computed, String name) { + assertTrue(expected == computed, + () -> name + ": expected == " + expected + " but computed == " + computed); + } + static void assertSame(long expected, long computed, String name) { assertTrue(expected == computed, () -> name + ": expected == " + expected + " but computed == " + computed); @@ -87,6 +93,11 @@ static void assertSame(Object expected, Object computed, String name) { () -> name + ": expected == " + expected + " but computed == " + computed); } + static void assertEquals(Object expected, Object computed, String name) { + assertTrue(Objects.equals(expected, computed), + () -> name + ": expected == " + expected + " but computed == " + computed); + } + static void assertNull(Object object, Supplier message) { assertTrue(object == null, message); } diff --git a/ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/RefCountingMap.java b/ratis-common/src/main/java/org/apache/ratis/util/RefCountingMap.java similarity index 88% rename from ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/RefCountingMap.java rename to ratis-common/src/main/java/org/apache/ratis/util/RefCountingMap.java index 6c3ad6f8cd..a6a9eb81ab 100644 --- a/ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/RefCountingMap.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/RefCountingMap.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ratis.metrics.dropwizard3; +package org.apache.ratis.util; import java.util.Collection; import java.util.Set; @@ -31,7 +31,7 @@ * call will increment the ref count, and each remove() will decrement it. The values are removed * from the map iff ref count == 0. */ -class RefCountingMap { +public final class RefCountingMap { private static class Payload { private final V value; private final AtomicInteger refCount = new AtomicInteger(); @@ -55,15 +55,15 @@ Payload decrement() { private final ConcurrentMap> map = new ConcurrentHashMap<>(); - V put(K k, Supplier supplier) { + public V put(K k, Supplier supplier) { return map.compute(k, (k1, old) -> old != null? old: new Payload<>(supplier.get())).increment(); } - static V get(Payload p) { + public static V get(Payload p) { return p == null ? null : p.get(); } - V get(K k) { + public V get(K k) { return get(map.get(k)); } @@ -72,23 +72,23 @@ V get(K k) { * @param k the key to remove * @return the value associated with the specified key or null if key is removed from map. */ - V remove(K k) { + public V remove(K k) { return get(map.computeIfPresent(k, (k1, v) -> v.decrement())); } - void clear() { + public void clear() { map.clear(); } - Set keySet() { + public Set keySet() { return map.keySet(); } - Collection values() { + public Collection values() { return map.values().stream().map(Payload::get).collect(Collectors.toList()); } - int size() { + public int size() { return map.size(); } } diff --git a/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedLeakDetector.java b/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedLeakDetector.java new file mode 100644 index 0000000000..271058e549 --- /dev/null +++ b/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedLeakDetector.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * A utility to detect leaks from @{@link ReferenceCountedObject}. + */ +public final class ReferenceCountedLeakDetector { + private static final Logger LOG = LoggerFactory.getLogger(ReferenceCountedLeakDetector.class); + // Leak detection is turned off by default. + + private static final AtomicReference FACTORY = new AtomicReference<>(Mode.NONE); + private static final Supplier SUPPLIER + = MemoizedSupplier.valueOf(() -> new LeakDetector(FACTORY.get().name()).start()); + + static Factory getFactory() { + return FACTORY.get(); + } + + public static LeakDetector getLeakDetector() { + return SUPPLIER.get(); + } + + private ReferenceCountedLeakDetector() { + } + + public static synchronized void enable(boolean advanced) { + FACTORY.set(advanced ? Mode.ADVANCED : Mode.SIMPLE); + } + + interface Factory { + ReferenceCountedObject create(V value, Runnable retainMethod, Consumer releaseMethod); + } + + private enum Mode implements Factory { + /** Leak detector is not enable in production to avoid performance impacts. */ + NONE { + @Override + public ReferenceCountedObject create(V value, Runnable retainMethod, Consumer releaseMethod) { + return new Impl<>(value, retainMethod, releaseMethod); + } + }, + /** Leak detector is enabled to detect leaks. This is intended to use in every tests. */ + SIMPLE { + @Override + public ReferenceCountedObject create(V value, Runnable retainMethod, Consumer releaseMethod) { + return new SimpleTracing<>(value, retainMethod, releaseMethod, getLeakDetector()); + } + }, + /** + * Leak detector is enabled to detect leaks and report object creation stacktrace as well as every retain and + * release stacktraces. This has severe impact in performance and only used to debug specific test cases. + */ + ADVANCED { + @Override + public ReferenceCountedObject create(V value, Runnable retainMethod, Consumer releaseMethod) { + return new AdvancedTracing<>(value, retainMethod, releaseMethod, getLeakDetector()); + } + } + } + + private static class Impl implements ReferenceCountedObject { + private final AtomicInteger count; + private final V value; + private final Runnable retainMethod; + private final Consumer releaseMethod; + + Impl(V value, Runnable retainMethod, Consumer releaseMethod) { + this.value = value; + this.retainMethod = retainMethod; + this.releaseMethod = releaseMethod; + count = new AtomicInteger(); + } + + @Override + public V get() { + final int previous = count.get(); + if (previous < 0) { + throw new IllegalStateException("Failed to get: object has already been completely released."); + } else if (previous == 0) { + throw new IllegalStateException("Failed to get: object has not yet been retained."); + } + return value; + } + + final int getCount() { + return count.get(); + } + + @Override + public V retain() { + // n < 0: exception + // n >= 0: n++ + if (count.getAndUpdate(n -> n < 0? n : n + 1) < 0) { + throw new IllegalStateException("Failed to retain: object has already been completely released."); + } + + retainMethod.run(); + return value; + } + + @Override + public boolean release() { + // n <= 0: exception + // n > 1: n-- + // n == 1: n = -1 + final int previous = count.getAndUpdate(n -> n <= 1? -1: n - 1); + if (previous < 0) { + throw new IllegalStateException("Failed to release: object has already been completely released."); + } else if (previous == 0) { + throw new IllegalStateException("Failed to release: object has not yet been retained."); + } + final boolean completedReleased = previous == 1; + releaseMethod.accept(completedReleased); + return completedReleased; + } + } + + private static class SimpleTracing extends Impl { + private final LeakDetector leakDetector; + private final Class valueClass; + private String valueString = null; + private Runnable removeMethod = null; + + SimpleTracing(T value, Runnable retainMethod, Consumer releaseMethod, LeakDetector leakDetector) { + super(value, retainMethod, releaseMethod); + this.valueClass = value.getClass(); + this.leakDetector = leakDetector; + } + + String getTraceString(int count) { + return "(" + valueClass + ", count=" + count + ", value=" + valueString + ")"; + } + + /** @return the leak message if there is a leak; return null if there is no leak. */ + String logLeakMessage() { + final int count = getCount(); + if (count == 0) { // never retain + return null; + } + final String message = "LEAK: " + getTraceString(count); + LOG.warn(message); + return message; + } + + @Override + public synchronized T get() { + try { + return super.get(); + } catch (Exception e) { + throw new IllegalStateException("Failed to get: " + getTraceString(getCount()), e); + } + } + + @Override + public synchronized T retain() { + final T value; + try { + value = super.retain(); + } catch (Exception e) { + throw new IllegalStateException("Failed to retain: " + getTraceString(getCount()), e); + } + if (getCount() == 1) { // this is the first retain + this.removeMethod = leakDetector.track(this, this::logLeakMessage); + this.valueString = value.toString(); + } + return value; + } + + @Override + public synchronized boolean release() { + final boolean released; + try { + released = super.release(); + } catch (Exception e) { + throw new IllegalStateException("Failed to release: " + getTraceString(getCount()), e); + } + + if (released) { + Objects.requireNonNull(removeMethod, () -> "Not yet retained (removeMethod == null): " + valueClass); + removeMethod.run(); + } + return released; + } + } + + private static class AdvancedTracing extends SimpleTracing { + enum Op {CREATION, RETAIN, RELEASE, CURRENT} + + static class Counts { + private final int refCount; + private final int retainCount; + private final int releaseCount; + + Counts() { + this.refCount = 0; + this.retainCount = 0; + this.releaseCount = 0; + } + + Counts(Op op, Counts previous) { + if (op == Op.RETAIN) { + this.refCount = previous.refCount + 1; + this.retainCount = previous.retainCount + 1; + this.releaseCount = previous.releaseCount; + } else if (op == Op.RELEASE) { + this.refCount = previous.refCount - 1; + this.retainCount = previous.retainCount; + this.releaseCount = previous.releaseCount + 1; + } else { + throw new IllegalStateException("Unexpected op: " + op); + } + } + + @Override + public String toString() { + return "refCount=" + refCount + + ", retainCount=" + retainCount + + ", releaseCount=" + releaseCount; + } + } + + static class TraceInfo { + private final int id; + private final Op op; + private final int previousRefCount; + private final Counts counts; + + private final String threadInfo; + private final StackTraceElement[] stackTraces; + private final int newTraceElementIndex; + + TraceInfo(int id, Op op, TraceInfo previous, int previousRefCount) { + this.id = id; + this.op = op; + this.previousRefCount = previousRefCount; + this.counts = previous == null? new Counts() + : op == Op.CURRENT ? previous.counts + : new Counts(op, previous.counts); + + final Thread thread = Thread.currentThread(); + this.threadInfo = "Thread_" + thread.getId() + ":" + thread.getName(); + this.stackTraces = thread.getStackTrace(); + this.newTraceElementIndex = previous == null? stackTraces.length - 1 + : findFirstUnequalFromTail(this.stackTraces, previous.stackTraces); + } + + static int findFirstUnequalFromTail(T[] current, T[] previous) { + int c = current.length == 0 ? 0 : current.length - 1; + for(int p = previous.length - 1; p >= 0; p--, c--) { + if (!previous[p].equals(current[c])) { + return c; + } + } + return -1; + } + + private StringBuilder appendTo(StringBuilder b) { + b.append(op).append("_").append(id) + .append(": previousRefCount=").append(previousRefCount) + .append(", ").append(counts) + .append(", ").append(threadInfo).append("\n"); + final int n = newTraceElementIndex + 1; + int line = 3; + for (; line <= n && line < stackTraces.length; line++) { + b.append(" ").append(stackTraces[line]).append("\n"); + } + if (line < stackTraces.length) { + b.append(" ...\n"); + } + return b; + } + + @Override + public String toString() { + return appendTo(new StringBuilder()).toString(); + } + } + + private final List traceInfos = new ArrayList<>(); + private TraceInfo previous; + + AdvancedTracing(T value, Runnable retainMethod, Consumer releaseMethod, LeakDetector leakDetector) { + super(value, retainMethod, releaseMethod, leakDetector); + addTraceInfo(Op.CREATION, -1); + } + + private synchronized TraceInfo addTraceInfo(Op op, int previousRefCount) { + final TraceInfo current = new TraceInfo(traceInfos.size(), op, previous, previousRefCount); + traceInfos.add(current); + previous = current; + return current; + } + + + @Override + public synchronized T retain() { + final int previousRefCount = getCount(); + final T retained = super.retain(); + final TraceInfo info = addTraceInfo(Op.RETAIN, previousRefCount); + Preconditions.assertSame(getCount(), info.counts.refCount, "refCount"); + return retained; + } + + @Override + public synchronized boolean release() { + final int previousRefCount = getCount(); + final boolean released = super.release(); + final TraceInfo info = addTraceInfo(Op.RELEASE, previousRefCount); + final int count = getCount(); + final int expected = count == -1? 0 : count; + Preconditions.assertSame(expected, info.counts.refCount, "refCount"); + return released; + } + + @Override + synchronized String getTraceString(int count) { + return super.getTraceString(count) + getTraceInfosString(); + } + + private String getTraceInfosString() { + final int n = traceInfos.size(); + final StringBuilder b = new StringBuilder(n << 10).append(" #TraceInfos=").append(n); + TraceInfo last = null; + for (TraceInfo info : traceInfos) { + info.appendTo(b.append("\n")); + last = info; + } + + // append current track info + final TraceInfo current = new TraceInfo(n, Op.CURRENT, last, getCount()); + current.appendTo(b.append("\n")); + + return b.toString(); + } + } +} diff --git a/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedObject.java b/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedObject.java index 3f72f5ffe8..1fc72c3445 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedObject.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/ReferenceCountedObject.java @@ -19,10 +19,11 @@ import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; +import java.util.Collection; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.Function; /** * A reference-counted object can be retained for later use @@ -44,6 +45,7 @@ * @param The object type. */ public interface ReferenceCountedObject { + /** @return the object. */ T get(); @@ -101,6 +103,32 @@ static ReferenceCountedObject wrap(V value) { return wrap(value, () -> {}, ignored -> {}); } + static ReferenceCountedObject delegateFrom(Collection> fromRefs, V value) { + return new ReferenceCountedObject() { + @Override + public V get() { + return value; + } + + @Override + public V retain() { + fromRefs.forEach(ReferenceCountedObject::retain); + return value; + } + + @Override + public boolean release() { + boolean allReleased = true; + for (ReferenceCountedObject ref : fromRefs) { + if (!ref.release()) { + allReleased = false; + } + } + return allReleased; + } + }; + } + /** * @return a {@link ReferenceCountedObject} of the given value by delegating to this object. */ @@ -125,6 +153,13 @@ public boolean release() { }; } + /** + * @return a {@link ReferenceCountedObject} by apply the given function to this object. + */ + default ReferenceCountedObject apply(Function function) { + return delegate(function.apply(get())); + } + /** * Wrap the given value as a {@link ReferenceCountedObject}. * @@ -140,48 +175,7 @@ static ReferenceCountedObject wrap(V value, Runnable retainMethod, Consum Objects.requireNonNull(retainMethod, "retainMethod == null"); Objects.requireNonNull(releaseMethod, "releaseMethod == null"); - return new ReferenceCountedObject() { - private final AtomicInteger count = new AtomicInteger(); - - @Override - public V get() { - final int previous = count.get(); - if (previous < 0) { - throw new IllegalStateException("Failed to get: object has already been completely released."); - } else if (previous == 0) { - throw new IllegalStateException("Failed to get: object has not yet been retained."); - } - return value; - } - - @Override - public V retain() { - // n < 0: exception - // n >= 0: n++ - if (count.getAndUpdate(n -> n < 0? n : n + 1) < 0) { - throw new IllegalStateException("Failed to retain: object has already been completely released."); - } - - retainMethod.run(); - return value; - } - - @Override - public boolean release() { - // n <= 0: exception - // n > 1: n-- - // n == 1: n = -1 - final int previous = count.getAndUpdate(n -> n <= 1? -1: n - 1); - if (previous < 0) { - throw new IllegalStateException("Failed to release: object has already been completely released."); - } else if (previous == 0) { - throw new IllegalStateException("Failed to release: object has not yet been retained."); - } - final boolean completedReleased = previous == 1; - releaseMethod.accept(completedReleased); - return completedReleased; - } - }; + return ReferenceCountedLeakDetector.getFactory().create(value, retainMethod, releaseMethod); } /** The same as wrap(value, retainMethod, ignored -> releaseMethod.run()). */ diff --git a/ratis-common/src/main/java/org/apache/ratis/util/ReflectionUtils.java b/ratis-common/src/main/java/org/apache/ratis/util/ReflectionUtils.java index 6f05c9c530..a8c217b6a1 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/ReflectionUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/ReflectionUtils.java @@ -20,11 +20,17 @@ package org.apache.ratis.util; - import java.lang.ref.WeakReference; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; /** diff --git a/ratis-common/src/main/java/org/apache/ratis/util/SizeInBytes.java b/ratis-common/src/main/java/org/apache/ratis/util/SizeInBytes.java index 683f0da628..0756f4feee 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/SizeInBytes.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/SizeInBytes.java @@ -33,14 +33,14 @@ public static SizeInBytes valueOf(long size) { } public static SizeInBytes valueOf(long n, TraditionalBinaryPrefix prefix) { - final long size = Objects.requireNonNull(prefix, "prefix = null").toLong(n); + final long size = Objects.requireNonNull(prefix, "prefix == null").toLong(n); final String input = n + " " + prefix.getSymbol(); final String description = input + " (=" + size + ")"; return new SizeInBytes(size, input, description); } public static SizeInBytes valueOf(String input) { - input = Objects.requireNonNull(input, "input = null").trim(); + input = Objects.requireNonNull(input, "input == null").trim(); final int last = input.length() - 1; final String s = "b".equalsIgnoreCase(input.substring(last))? diff --git a/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java b/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java index 316604db07..a41b4b136a 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/SlidingWindow.java @@ -28,7 +28,6 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.LongFunction; @@ -52,6 +51,9 @@ interface Request { boolean hasReply(); void fail(Throwable e); + + default void release() { + } } interface ClientSideRequest extends Request { @@ -70,9 +72,6 @@ class RequestMap, REPLY> implements Iterable iterator() { return requests.values().iterator(); } + /** @return true iff the request already exists. */ + boolean putIfAbsent(REQUEST request) { + final long seqNum = request.getSeqNum(); + final REQUEST previous = requests.putIfAbsent(seqNum, request); + return previous != null; + } + void putNewRequest(REQUEST request) { final long seqNum = request.getSeqNum(); CollectionUtils.putNew(seqNum, request, requests, () -> getName() + ":requests"); @@ -157,14 +163,19 @@ void endOfRequests(long nextToProcess, REQUEST end, Consumer replyMetho + " will NEVER be processed; request = " + r); r.fail(e); replyMethod.accept(r); + r.release(); } tail.clear(); putNewRequest(end); } - void clear() { + void clear(long nextToProcess) { LOG.debug("close {}", this); + final SortedMap tail = requests.tailMap(nextToProcess); + for (REQUEST r : tail.values()) { + r.release(); + } requests.clear(); } @@ -441,14 +452,26 @@ public synchronized String toString() { /** A request (or a retry) arrives (may be out-of-order except for the first request). */ public synchronized void receivedRequest(REQUEST request, Consumer processingMethod) { final long seqNum = request.getSeqNum(); + final boolean accepted; if (nextToProcess == -1 && (request.isFirstRequest() || seqNum == 0)) { nextToProcess = seqNum; - LOG.debug("{}: got seq={} (first request), set nextToProcess in {}", requests.getName(), seqNum, this); + requests.putNewRequest(request); + LOG.debug("Received seq={} (first request), {}", seqNum, this); + accepted = true; + } else if (request.getSeqNum() < nextToProcess) { + LOG.debug("Received seq={} < nextToProcess {}, {}", seqNum, nextToProcess, this); + accepted = false; + } else { + final boolean isRetry = requests.putIfAbsent(request); + LOG.debug("Received seq={}, isRetry? {}, {}", seqNum, isRetry, this); + accepted = !isRetry; + } + + if (accepted) { + processRequestsFromHead(processingMethod); } else { - LOG.debug("{}: got seq={} in {}", requests.getName(), seqNum, this); + request.release(); } - requests.putNewRequest(request); - processRequestsFromHead(processingMethod); } private void processRequestsFromHead(Consumer processingMethod) { @@ -457,6 +480,7 @@ private void processRequestsFromHead(Consumer processingMethod) { return; } else if (r.getSeqNum() == nextToProcess) { processingMethod.accept(r); + r.release(); nextToProcess++; } } @@ -502,7 +526,7 @@ public synchronized boolean endOfRequests(Consumer replyMethod) { @Override public void close() { - requests.clear(); + requests.clear(nextToProcess); } } } \ No newline at end of file diff --git a/ratis-common/src/main/java/org/apache/ratis/util/TimeDuration.java b/ratis-common/src/main/java/org/apache/ratis/util/TimeDuration.java index 2abdfdfaca..2a520083e0 100644 --- a/ratis-common/src/main/java/org/apache/ratis/util/TimeDuration.java +++ b/ratis-common/src/main/java/org/apache/ratis/util/TimeDuration.java @@ -144,7 +144,7 @@ public static long parse(String timeString, TimeUnit targetUnit) { * @return a {@link TimeDuration} in the target unit. */ public static TimeDuration valueOf(String timeString, TimeUnit defaultUnit) { - Objects.requireNonNull(timeString, "timeString = null"); + Objects.requireNonNull(timeString, "timeString == null"); final String lower = timeString.trim().replace("_", "").toLowerCase(); for(Abbreviation a : Abbreviation.values()) { for(String s : a.getSymbols()) { @@ -172,7 +172,7 @@ public static TimeDuration valueOf(long duration, TimeUnit unit) { private TimeDuration(long duration, TimeUnit unit) { this.duration = duration; - this.unit = Objects.requireNonNull(unit, "unit = null"); + this.unit = Objects.requireNonNull(unit, "unit == null"); } /** @return the duration value. */ diff --git a/ratis-common/src/main/java/org/apache/ratis/util/WeakValueCache.java b/ratis-common/src/main/java/org/apache/ratis/util/WeakValueCache.java new file mode 100644 index 0000000000..5c6fcd3683 --- /dev/null +++ b/ratis-common/src/main/java/org/apache/ratis/util/WeakValueCache.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.util; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; + +import static org.apache.ratis.util.BiWeakValueCache.newMap; + +/** + * Weak Value Cache: {@link K} -> {@link V}. + *

+ * Note that the cached values are weakly referenced. + * A cached value could be garage-collected (i.e. evicted from the cache) + * when there are no external (strong) references. + *

+ * For key types with two components, use {@link BiWeakValueCache}. + * + * @param the type of the keys. + * @param the type to be cached values. + */ +public final class WeakValueCache { + private final String keyName; + private final String name; + + /** For constructing a value from a key. */ + private final Function constructor; + /** Count the number of values constructed. */ + private final AtomicInteger constructionCount = new AtomicInteger(0); + + /** Map: {@link K} -> {@link V}. */ + private final ConcurrentMap map = newMap(); + + /** + * Create a cache for mapping {@link K} keys to {@link V} values. + * + * @param keyName the name of the key. + * @param constructor for constructing {@link V} values. + */ + public WeakValueCache(String keyName, Function constructor) { + this.keyName = keyName; + this.name = keyName + "-cache"; + this.constructor = constructor; + } + + private V construct(K key) { + final V constructed = constructor.apply(key); + Objects.requireNonNull(constructed, "constructed == null"); + constructionCount.incrementAndGet(); + return constructed; + } + + /** + * If the given key is in the cache, return its cached values. + * Otherwise, create a new value, put it in the cache and then return it. + */ + public V getOrCreate(K key) { + Objects.requireNonNull(key, () -> keyName + " (key) == null"); + return map.computeIfAbsent(key, this::construct); + } + + List getValues() { + return new ArrayList<>(map.values()); + } + + @Override + public String toString() { + return name; + } +} diff --git a/ratis-common/src/test/java/org/apache/ratis/BaseTest.java b/ratis-common/src/test/java/org/apache/ratis/BaseTest.java index 8213694614..2c9f87c0aa 100644 --- a/ratis-common/src/test/java/org/apache/ratis/BaseTest.java +++ b/ratis-common/src/test/java/org/apache/ratis/BaseTest.java @@ -18,27 +18,29 @@ package org.apache.ratis; import org.apache.ratis.conf.ConfUtils; -import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.ReferenceCountedLeakDetector; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.StringUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.function.CheckedRunnable; import org.junit.After; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.rules.TestName; -import org.junit.rules.Timeout; +import org.junit.Before; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.lang.reflect.Method; import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; @@ -47,6 +49,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +@Timeout(value = 100) public abstract class BaseTest { public final Logger LOG = LoggerFactory.getLogger(getClass()); @@ -69,18 +72,36 @@ public void setFirstException(Throwable e) { } } - public List getPeersWithPriority(List peers, RaftPeer suggestedLeader) { - List peersWithPriority = new ArrayList<>(); - for (int i = 0; i < peers.size(); i++) { - RaftPeer peer = peers.get(i); - final int priority = peer.equals(suggestedLeader)? 2: 1; - peersWithPriority.add( - RaftPeer.newBuilder(peer).setPriority(priority).build()); - } - return peersWithPriority; + // TODO: Junit 4 reference should be removed once all the unit tests are migrated to Junit 5. + + private String testCaseName; + + @BeforeEach + public void setup(TestInfo testInfo) { + checkAssumptions(); + + final Method method = testInfo.getTestMethod().orElse(null); + testCaseName = testInfo.getTestClass().orElse(getClass()).getSimpleName() + + "." + (method == null? null : method.getName()); + } + + // @Before annotation is retained to support junit 4 tests. + @Before + @BeforeEach + public void checkAssumptions() { + final int leaks = ReferenceCountedLeakDetector.getLeakDetector().getLeakCount(); + Assumptions.assumeFalse(0 < leaks, () -> "numLeaks " + leaks + " > 0"); + + final Throwable first = firstException.get(); + Assumptions.assumeTrue(first == null, () -> "Already failed with " + first); + + final Throwable exited = ExitUtils.getFirstExitException(); + Assumptions.assumeTrue(exited == null, () -> "Already exited with " + exited); } + // @After annotation is retained to support junit 4 tests. @After + @AfterEach public void assertNoFailures() { final Throwable e = firstException.get(); if (e != null) { @@ -90,17 +111,7 @@ public void assertNoFailures() { ExitUtils.assertNotTerminated(); } - @Rule - public final Timeout globalTimeout = new Timeout(getGlobalTimeoutSeconds(), TimeUnit.SECONDS ); - - @Rule - public final TestName testName = new TestName(); - - public int getGlobalTimeoutSeconds() { - return 100; - } - - private static final Supplier rootTestDir = JavaUtils.memoize( + private static final Supplier ROOT_TEST_DIR = JavaUtils.memoize( () -> JavaUtils.callAsUnchecked(() -> { final File dir = new File(System.getProperty("test.build.data", "target/test/data"), Integer.toHexString(ThreadLocalRandom.current().nextInt())); @@ -114,7 +125,7 @@ public int getGlobalTimeoutSeconds() { public static File getRootTestDir() { - return rootTestDir.get(); + return ROOT_TEST_DIR.get(); } public File getClassTestDir() { @@ -122,7 +133,8 @@ public File getClassTestDir() { } public File getTestDir() { - return new File(getClassTestDir(), testName.getMethodName()); + // This will work for both junit 4 and 5. + return new File(getClassTestDir(), testCaseName); } @SafeVarargs @@ -135,13 +147,13 @@ public static void assertThrowable( description, expectedThrowableClass.getSimpleName(), StringUtils.array2String(expectedCauseClasses, Class::getSimpleName)); } - Assert.assertEquals(expectedThrowableClass, t.getClass()); + Assertions.assertEquals(expectedThrowableClass, t.getClass()); for (Class expectedCause : expectedCauseClasses) { final Throwable previous = t; t = Objects.requireNonNull(previous.getCause(), () -> "previous.getCause() == null for previous=" + previous); - Assert.assertEquals(expectedCause, t.getClass()); + Assertions.assertEquals(expectedCause, t.getClass()); } } diff --git a/ratis-common/src/test/java/org/apache/ratis/JUnit5TestExecutionListener.java b/ratis-common/src/test/java/org/apache/ratis/JUnit5TestExecutionListener.java new file mode 100644 index 0000000000..535d8968cb --- /dev/null +++ b/ratis-common/src/test/java/org/apache/ratis/JUnit5TestExecutionListener.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis; + +import org.apache.ratis.util.JavaUtils; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; + +import java.io.PrintStream; +import java.util.concurrent.TimeoutException; + +/** + * A {@link TestExecutionListener} to dump all threads after a test timeout failure. + */ +public class JUnit5TestExecutionListener implements TestExecutionListener { + private final PrintStream out = System.out; + + @Override + public void executionFinished(TestIdentifier id, TestExecutionResult result) { + final Throwable timeoutException = getTimeoutException(result); + if (timeoutException != null) { + out.format("%n%s %s failed%n", JavaUtils.date(), id.getDisplayName()); + timeoutException.printStackTrace(out); + JavaUtils.dumpAllThreads(out::println); + } + } + + private static Throwable getTimeoutException(TestExecutionResult result) { + if (result == null) { + return null; + } + final Throwable throwable = result.getThrowable().orElse(null); + return throwable instanceof TimeoutException? throwable : null; + } +} diff --git a/ratis-common/src/test/java/org/apache/ratis/JUnitRunListener.java b/ratis-common/src/test/java/org/apache/ratis/JUnitRunListener.java deleted file mode 100644 index 144c8069c6..0000000000 --- a/ratis-common/src/test/java/org/apache/ratis/JUnitRunListener.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis; - -import org.apache.ratis.util.JavaUtils; -import org.junit.internal.runners.statements.FailOnTimeout; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; -import org.junit.runners.model.Statement; -import org.junit.runners.model.TestTimedOutException; - -import java.io.PrintStream; -import java.util.concurrent.TimeUnit; - -/** - * A {@link RunListener} to dump all threads after a test timeout failure. - */ -public class JUnitRunListener extends RunListener { - private static final Throwable TIMEOUT_EXCEPTION = getTimeoutException(); - private static final String TIMEOUT_EXCEPTION_PREFIX; - - private static Throwable getTimeoutException() { - final FailOnTimeout f = FailOnTimeout.builder().withTimeout(1, TimeUnit.NANOSECONDS).build(new Statement() { - @Override - public void evaluate() throws InterruptedException { - Thread.sleep(1000); - } - }); - try { - f.evaluate(); - } catch(Throwable throwable) { - return throwable; - } - throw new IllegalStateException("Failed to getTimeoutException"); - } - - static { - final String message = JUnitRunListener.TIMEOUT_EXCEPTION.getMessage(); - TIMEOUT_EXCEPTION_PREFIX = message.substring(0, message.indexOf('1')); - } - - private final PrintStream out = System.out; - - @Override - public void testFailure(Failure failure) { - final Throwable timeoutException = getTimeoutException(failure); - if (timeoutException != null) { - out.format("%n%s ", JavaUtils.date()); - timeoutException.printStackTrace(out); - JavaUtils.dumpAllThreads(out::println); - } - } - - private static Throwable getTimeoutException(Failure failure) { - if (failure == null) { - return null; - } - final Throwable throwable = failure.getException(); - if (throwable.getClass() != TIMEOUT_EXCEPTION.getClass()) { - return null; - } - final String message = throwable.getMessage(); - if (message == null || !message.startsWith(TIMEOUT_EXCEPTION_PREFIX)) { - return null; - } - return throwable; - } - - public static void main(String[] args) { - final JUnitRunListener listener = new JUnitRunListener(); - listener.out.println("TIMEOUT_EXCEPTION_PREFIX = '" + TIMEOUT_EXCEPTION_PREFIX + "'"); - TIMEOUT_EXCEPTION.printStackTrace(listener.out); - - listener.testFailure(new Failure(null, new TestTimedOutException(999, TimeUnit.MILLISECONDS))); - } -} diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java new file mode 100644 index 0000000000..2d8c63030a --- /dev/null +++ b/ratis-common/src/test/java/org/apache/ratis/test/tag/Flaky.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.test.tag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.junit.jupiter.api.Tag; + +/** + * Annotation to mark JUnit5 test classes or methods that exhibit intermittent + * issues. These are run separately from the normal tests in CI. In case of + * failure they may be repeated a few times. + * Usage: @Flaky("RATIS-123") + */ +@Target({ ElementType.TYPE, ElementType.METHOD }) +@Retention(RetentionPolicy.RUNTIME) +@Tag("flaky") +public @interface Flaky { + /** + * The issue(s) tracking the flaky test. + */ + String[] value(); +} diff --git a/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java new file mode 100644 index 0000000000..f43bd16985 --- /dev/null +++ b/ratis-common/src/test/java/org/apache/ratis/test/tag/FlakyTest.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.test.tag; + +/** + * Interface to mark JUnit4 test classes or methods that exhibit intermittent + * issues. These are run separately from the normal tests in CI. In case of + * failure they may be repeated a few times. + * Usage: @Category(FlakyTest.class) @Flaky("RATIS-123") + */ +public interface FlakyTest { + // category marker +} diff --git a/ratis-metrics-default/src/test/java/org/apache/ratis/metrics/impl/TestRefCountingMap.java b/ratis-common/src/test/java/org/apache/ratis/util/TestRefCountingMap.java similarity index 86% rename from ratis-metrics-default/src/test/java/org/apache/ratis/metrics/impl/TestRefCountingMap.java rename to ratis-common/src/test/java/org/apache/ratis/util/TestRefCountingMap.java index a8c54d7b15..217d6e3da8 100644 --- a/ratis-metrics-default/src/test/java/org/apache/ratis/metrics/impl/TestRefCountingMap.java +++ b/ratis-common/src/test/java/org/apache/ratis/util/TestRefCountingMap.java @@ -15,25 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ratis.metrics.impl; +package org.apache.ratis.util; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Arrays; import java.util.Collection; import java.util.Set; -import org.apache.ratis.thirdparty.com.google.common.collect.Lists; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestRefCountingMap { private RefCountingMap map; - @Before + @BeforeEach public void setUp() { map = new RefCountingMap<>(); } @@ -128,7 +128,7 @@ public void testKeySet() { Set keys = map.keySet(); assertEquals(3, keys.size()); - Lists.newArrayList("foo", "bar", "baz").stream().forEach(v -> assertTrue(keys.contains(v))); + Arrays.asList("foo", "bar", "baz").forEach(v -> assertTrue(keys.contains(v))); } @Test @@ -141,7 +141,7 @@ public void testValues() { Collection values = map.values(); assertEquals(3, values.size()); - Lists.newArrayList("foovalue", "foovalue3", "foovalue4").stream() + Arrays.asList("foovalue", "foovalue3", "foovalue4") .forEach(v -> assertTrue(values.contains(v))); } } diff --git a/ratis-common/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/ratis-common/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 0000000000..71636f29a8 --- /dev/null +++ b/ratis-common/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.apache.ratis.JUnit5TestExecutionListener \ No newline at end of file diff --git a/ratis-docs/pom.xml b/ratis-docs/pom.xml index 47f1175a46..1bb1c64589 100644 --- a/ratis-docs/pom.xml +++ b/ratis-docs/pom.xml @@ -20,10 +20,16 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-docs Apache Ratis Documentation jar + + + + true + + diff --git a/ratis-docs/src/site/markdown/cli.md b/ratis-docs/src/site/markdown/cli.md index 60958fc7ed..ab9f89982f 100644 --- a/ratis-docs/src/site/markdown/cli.md +++ b/ratis-docs/src/site/markdown/cli.md @@ -182,5 +182,5 @@ It has the following subcommands: ### local raftMetaConf Generate a new raft-meta.conf file based on original raft-meta.conf and new peers, which is used to move a raft node to a new node. ``` -$ ratis sh local raftMetaConf -peers -path +$ ratis sh local raftMetaConf -peers <[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT> -path ``` diff --git a/ratis-docs/src/site/markdown/concept/index.md b/ratis-docs/src/site/markdown/concept/index.md index 279338a5fd..6ff77b2203 100644 --- a/ratis-docs/src/site/markdown/concept/index.md +++ b/ratis-docs/src/site/markdown/concept/index.md @@ -15,24 +15,76 @@ limitations under the License. --> # Concepts -TODO: complete this section + ## RaftServer +The `RaftServer` is a core component of Apache Ratis, +responsible for handling all operations related to the RAFT protocol. +A `RaftServer` manages all the `RaftGroup`(s) within the current process. +For each group, a corresponding `RaftServer.Division` handles the core functions +such as replication of log entries, voting, and leader election within that group. +Each division can independently act as a Leader, Candidate, Follower or Listener, +with the specific role depending on the outcome of elections and the state of the protocol. + ## RaftPeer and RaftPeerId +`RaftPeer` represents a participant node in the cluster, +including the node's unique identifier, IP address, and port number. +The unique identifier is represented by the `RaftPeerId` class, +which is crucial for distinguishing different nodes within a cluster. + ## RaftGroup -## Transport (gRPC,Netty ...) +A `RaftGroup` represents a collection of `RaftPeer`(s) in a Raft protocol cluster. +Each group has a unique identifier represented by the `RaftGroupId` class. +Multiple groups can operate independently within a physical network, +while each group managing its own consistency and state replication. -## Raft Log +## Transport (gRPC, Netty, etc.) -## StateMachine -A StateMachine is the abstraction point for user-code +Ratis supports various network transport protocols for node communication, +including gRPC (default) and Netty. +These transport layers in Ratis are used for data serialization and deserialization, +as well as ensuring safe and efficient data transmission between different nodes. + +## RaftLog + +The `RaftLog` is a core component of the Raft algorithm, +used to record all state change transactions. +Once a log entry has been acknowledged by a majority of peers, +the entry becomes committed. +The Raft log is key to achieving distributed data consistency. ## Snapshot +A `Snapshot` is a point-in-time copy of the current state of the `StateMachine`. +It can be used for quick recovery of the state after system restarts, +and for transferring the state to newly joined nodes. +When a snapshot has been taken, +the log entries earlier than the snapshot can be purged +in order to free up the storage space. + ## TermIndex -## Transaction (as in applyTransaction or startTransaction in StateMachine) +`TermIndex` is an order pair of `long` integers (term, index) as defined in the Raft protocol. +Term is the logical clock in Raft. +A newly elected leader starts a new term and remains the leader for the rest of the term. +Index is the position of log entries in the Raft log. + +## StateMachine + +In Ratis, `StateMachine` is the abstraction point for user-defined code. +Developers implement specific business logic or data storage operations at this layer. +The transactions committed through the Raft protocol will be applied to it. + +### The `applyTransaction` method + +In Ratis, transaction processing is implemented by the `StateMachine` +through the `applyTransaction` method. +A transaction usually changes the state of the `StateMachine`. + +### StateMachineStorage -## StateMachineStorage +`StateMachineStorage` is a component for storing data related to the `StateMachine`. +It is for persisting the Raft log and the snapshots +such that the state can be fully recovered even after system failures. diff --git a/ratis-docs/src/site/markdown/configurations.md b/ratis-docs/src/site/markdown/configurations.md index 7c8fb001fb..4caf11b30e 100644 --- a/ratis-docs/src/site/markdown/configurations.md +++ b/ratis-docs/src/site/markdown/configurations.md @@ -104,6 +104,16 @@ When bootstrapping a new peer, If the gap between the match index of the peer and the leader's latest committed index is less than this gap, we treat the peer as caught-up. Increase this number when write throughput is high. +--------------------------------------------------------------------------------- +| **Property** | `raft.server.staging.timeout` | +|:----------------|:-----------------------------------------| +| **Description** | timeout of bootstrapping a new peer | +| **Type** | TimeDuration | +| **Default** | 3 times of `raft.server.rpc.timeout.max` | + +During the initialization of a new peer, the leader will classify the bootstrap process as "NO PROGRESS" +if it fails to receive any RPC responses from this peer within this specified timeout period. + --------------------------------------------------------------------------------- ### ThreadPool - Configurations related to server thread pools. @@ -494,6 +504,11 @@ further wait for 5sec for max times ((5sec*980)/2 times ~= 40min) | **Type** | boolean | | **Default** | true | +| **Property** | `raft.server.snapshot.trigger-when-remove.enabled` | +|:----------------|:--------------------------------------------------------| +| **Description** | whether to trigger snapshot when raft server is removed | +| **Type** | boolean | +| **Default** | true | | **Property** | `raft.server.snapshot.creation.gap` | |:----------------|:-----------------------------------------------------| @@ -670,4 +685,65 @@ For examples, 2. However, setConf to a 3-member group by removing 2 of members and adding 2 new members is a majority-add. - Note also that adding 1 new member to an 1-member group is always allowed, - although it is a majority-add. \ No newline at end of file + although it is a majority-add. + + +## Client Configurations + +Client configurations are located at `RaftClientConfigKeys`. + +### RPC - Configurations related to Client RPC timeout. + +| **Property** | `raft.client.rpc.request.timeout` | +|:----------------|:------------------------------------------| +| **Description** | client side timeout for sending a request | +| **Type** | TimeDuration | +| **Default** | 3s | + +| **Property** | `raft.client.rpc.watch.request.timeout` | +|:----------------|:------------------------------------------------| +| **Description** | client side timeout for sending a watch request | +| **Type** | TimeDuration | +| **Default** | 10s | + +### Async - Configurations related to async requests. + +| **Property** | `raft.client.async.outstanding-requests.max` | +|:----------------|:---------------------------------------------| +| **Description** | maximum number of outstanding async requests | +| **Type** | int | +| **Default** | 100 | + +### DataStream - Configurations related to DataStream Api. + +| **Property** | `raft.client.data-stream.outstanding-requests.max` | +|:----------------|:---------------------------------------------------| +| **Description** | maximum number of outstanding data stream requests | +| **Type** | int | +| **Default** | 100 | + +| **Property** | `raft.client.data-stream.flush.request.count.min` | +|:----------------|:-----------------------------------------------------------------| +| **Description** | minimum number of requests before data stream flush would happen | +| **Type** | int | +| **Default** | 0 | + +| **Property** | `raft.client.data-stream.flush.request.bytes.min` | +|:----------------|:--------------------------------------------------------------| +| **Description** | minimum number of bytes before data stream flush would happen | +| **Type** | SizeInBytes | +| **Default** | 1MB | + +| **Property** | `raft.client.data-stream.request.timeout` | +|:----------------|:------------------------------------------| +| **Description** | timeout for data stream request | +| **Type** | TimeDuration | +| **Default** | 10s | + +### MessageStream - Configurations related to MessageStream Api. + +| **Property** | `raft.client.message-stream.submessage-size` | +|:----------------|:---------------------------------------------| +| **Description** | maximum size of a sub message | +| **Type** | SizeInBytes | +| **Default** | 1MB | diff --git a/ratis-docs/src/site/markdown/metrics.md b/ratis-docs/src/site/markdown/metrics.md index 10c78ccbbb..23486792e6 100644 --- a/ratis-docs/src/site/markdown/metrics.md +++ b/ratis-docs/src/site/markdown/metrics.md @@ -80,31 +80,34 @@ ### Raft Server Metrics -| Application | Component | Name | Type | Description | -|-------------|-----------|----------------------------------|---------|---------------------------------------------------------------------| -| ratis | server | {peer}_lastHeartbeatElapsedTime | Gauge | Time elapsed since last heartbeat rpc response | -| ratis | server | follower_append_entry_latency | Timer | Time taken for followers to append log entries | -| ratis | server | {peer}_peerCommitIndex | Gauge | Commit index of peer | -| ratis | server | clientReadRequest | Timer | Time taken to process read requests from client | -| ratis | server | clientStaleReadRequest | Timer | Time taken to process stale-read requests from client | -| ratis | server | clientWriteRequest | Timer | Time taken to process write requests from client | -| ratis | server | clientWatch{level}Request | Timer | Time taken to process watch(replication_level) requests from client | -| ratis | server | numRequestQueueLimitHits | Counter | Number of (total client requests in queue) limit hits | -| ratis | server | numRequestsByteSizeLimitHits | Counter | Number of (total size of client requests in queue) limit hits | -| ratis | server | numResourceLimitHits | Counter | Sum of numRequestQueueLimitHits and numRequestsByteSizeLimitHits | -| ratis | server | numPendingRequestInQueue | Gauge | Number of pending client requests in queue | -| ratis | server | numPendingRequestMegaByteSize | Gauge | Total size of pending client requests in queue | -| ratis | server | retryCacheEntryCount | Gauge | Number of entries in retry cache | -| ratis | server | retryCacheHitCount | Gauge | Number of retry cache hits | -| ratis | server | retryCacheHitRate | Gauge | Retry cache hit rate | -| ratis | server | retryCacheMissCount | Gauge | Number of retry cache misses | -| ratis | server | retryCacheMissRate | Gauge | Retry cache miss rate | -| ratis | server | numFailedClientStaleReadOnServer | Counter | Number of failed stale-read requests | -| ratis | server | numFailedClientReadOnServer | Counter | Number of failed read requests | -| ratis | server | numFailedClientWriteOnServer | Counter | Number of failed write requests | -| ratis | server | numFailedClientWatchOnServer | Counter | Number of failed watch requests | -| ratis | server | numFailedClientStreamOnServer | Counter | Number of failed stream requests | -| ratis | server | numInstallSnapshot | Counter | Number of install-snapshot requests | +| Application | Component | Name | Type | Description | +|-------------|-----------|--------------------------------------|---------|---------------------------------------------------------------------| +| ratis | server | {peer}_lastHeartbeatElapsedTime | Gauge | Time elapsed since last heartbeat rpc response | +| ratis | server | follower_append_entry_latency | Timer | Time taken for followers to append log entries | +| ratis | server | {peer}_peerCommitIndex | Gauge | Commit index of peer | +| ratis | server | clientReadRequest | Timer | Time taken to process read requests from client | +| ratis | server | clientStaleReadRequest | Timer | Time taken to process stale-read requests from client | +| ratis | server | clientWriteRequest | Timer | Time taken to process write requests from client | +| ratis | server | clientWatch{level}Request | Timer | Time taken to process watch(replication_level) requests from client | +| ratis | server | numRequestQueueLimitHits | Counter | Number of (total client requests in queue) limit hits | +| ratis | server | numRequestsByteSizeLimitHits | Counter | Number of (total size of client requests in queue) limit hits | +| ratis | server | numResourceLimitHits | Counter | Sum of numRequestQueueLimitHits and numRequestsByteSizeLimitHits | +| ratis | server | numPendingRequestInQueue | Gauge | Number of pending client requests in queue | +| ratis | server | numPendingRequestMegaByteSize | Gauge | Total size of pending client requests in queue | +| ratis | server | retryCacheEntryCount | Gauge | Number of entries in retry cache | +| ratis | server | retryCacheHitCount | Gauge | Number of retry cache hits | +| ratis | server | retryCacheHitRate | Gauge | Retry cache hit rate | +| ratis | server | retryCacheMissCount | Gauge | Number of retry cache misses | +| ratis | server | retryCacheMissRate | Gauge | Retry cache miss rate | +| ratis | server | numFailedClientStaleReadOnServer | Counter | Number of failed stale-read requests | +| ratis | server | numFailedClientReadOnServer | Counter | Number of failed read requests | +| ratis | server | numFailedClientWriteOnServer | Counter | Number of failed write requests | +| ratis | server | numFailedClientWatchOnServer | Counter | Number of failed watch requests | +| ratis | server | numFailedClientStreamOnServer | Counter | Number of failed stream requests | +| ratis | server | numInstallSnapshot | Counter | Number of install-snapshot requests | +| ratis | server | numWatch{level}RequestTimeout | Counter | Number of watch(replication_level) request timeout | +| ratis | server | numWatch{level}RequestInQueue | Gauge | Number of watch(replication_level) requests in queue | +| ratis | server | numWatch{level}RequestQueueLimitHits | Counter | Number of (total watch request in queue) limit hits | ## Ratis Netty Metrics diff --git a/ratis-docs/src/site/markdown/snapshot.md b/ratis-docs/src/site/markdown/snapshot.md index f20dc19d71..a9f06511fc 100644 --- a/ratis-docs/src/site/markdown/snapshot.md +++ b/ratis-docs/src/site/markdown/snapshot.md @@ -75,6 +75,8 @@ Note that Ratis imposes a minimal creation gap between two subsequent snapshot c ```java // SnapshotManagementApi RaftClientReply create(long timeoutMs) throws IOException; + RaftClientReply create(boolean force, long timeoutMs) throws IOException; + RaftClientReply create(long creationGap, long timeoutMs) throws IOException; ``` ```java diff --git a/ratis-examples/pom.xml b/ratis-examples/pom.xml index 5e899f8fff..ce594d332a 100644 --- a/ratis-examples/pom.xml +++ b/ratis-examples/pom.xml @@ -17,7 +17,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-examples @@ -122,12 +122,20 @@ junit test - - com.github.spotbugs - spotbugs-annotations - provided - true + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-params + test @@ -142,8 +150,8 @@ shade - ${project.build.directory}/dependency-reduced-pom.xml - true + false + false org.apache.ratis.examples.common.Runner @@ -176,23 +184,6 @@ - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-file - package - - copy - - - target/${project.artifactId}-${project.version}-shaded.jar - target/${project.artifactId}-${project.version}.jar - - - - diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/ArithmeticStateMachine.java b/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/ArithmeticStateMachine.java index 28e3fb1c7d..c4adff5987 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/ArithmeticStateMachine.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/ArithmeticStateMachine.java @@ -81,7 +81,7 @@ public void initialize(RaftServer server, RaftGroupId groupId, @Override public void reinitialize() throws IOException { close(); - loadSnapshot(storage.getLatestSnapshot()); + loadSnapshot(storage.loadLatestSnapshot()); } @Override @@ -164,7 +164,7 @@ public void close() { @Override public CompletableFuture applyTransaction(TransactionContext trx) { - final LogEntryProto entry = trx.getLogEntry(); + final LogEntryProto entry = trx.getLogEntryUnsafe(); final AssignmentMessage assignment = new AssignmentMessage(entry.getStateMachineLogEntry().getLogData()); final long index = entry.getIndex(); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/counter/server/CounterStateMachine.java b/ratis-examples/src/main/java/org/apache/ratis/examples/counter/server/CounterStateMachine.java index b88a763e0f..a65582ea00 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/counter/server/CounterStateMachine.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/counter/server/CounterStateMachine.java @@ -138,7 +138,7 @@ public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftS */ @Override public void reinitialize() throws IOException { - load(storage.getLatestSnapshot()); + load(storage.loadLatestSnapshot()); } /** @@ -247,7 +247,7 @@ public TransactionContext startTransaction(RaftClientRequest request) throws IOE */ @Override public CompletableFuture applyTransaction(TransactionContext trx) { - final LogEntryProto entry = trx.getLogEntry(); + final LogEntryProto entry = trx.getLogEntryUnsafe(); //increment the counter and update term-index final TermIndex termIndex = TermIndex.valueOf(entry); final int incremented = incrementCounter(termIndex); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java b/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java index 4377a1420f..07f7b20f44 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/debug/server/Server.java @@ -34,6 +34,7 @@ public final class Server { private Server(){ } + @SuppressWarnings({"squid:S2095"}) // Suppress closeable warning public static void main(String[] args) throws IOException { if (args.length < 1) { System.err.println("The arguments should be "); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileInfo.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileInfo.java index c7d8cb7cd1..bba001002a 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileInfo.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileInfo.java @@ -19,6 +19,7 @@ import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; import org.apache.ratis.util.CollectionUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LogUtils; @@ -84,7 +85,7 @@ ByteString read(CheckedFunction resolver, long offset, final ByteBuffer buffer = ByteBuffer.allocateDirect(FileStoreCommon.getChunkSize(length)); in.position(offset).read(buffer); buffer.flip(); - return ByteString.copyFrom(buffer); + return UnsafeByteOperations.unsafeWrap(buffer); } } diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileStoreStateMachine.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileStoreStateMachine.java index 858e300ec9..2fb9cace66 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileStoreStateMachine.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/FileStoreStateMachine.java @@ -42,6 +42,7 @@ import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.ReferenceCountedObject; import java.io.IOException; import java.nio.file.Path; @@ -114,16 +115,18 @@ public TransactionContext startTransaction(RaftClientRequest request) throws IOE @Override public TransactionContext startTransaction(LogEntryProto entry, RaftProtos.RaftPeerRole role) { + ByteString copied = ByteString.copyFrom(entry.getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer()); return TransactionContext.newBuilder() .setStateMachine(this) .setLogEntry(entry) .setServerRole(role) - .setStateMachineContext(getProto(entry)) + .setStateMachineContext(getProto(copied)) .build(); } @Override - public CompletableFuture write(LogEntryProto entry, TransactionContext context) { + public CompletableFuture write(ReferenceCountedObject entryRef, TransactionContext context) { + LogEntryProto entry = entryRef.retain(); final FileStoreRequestProto proto = getProto(context, entry); if (proto.getRequestCase() != FileStoreRequestProto.RequestCase.WRITEHEADER) { return null; @@ -132,9 +135,10 @@ public CompletableFuture write(LogEntryProto entry, TransactionContext final WriteRequestHeaderProto h = proto.getWriteHeader(); final CompletableFuture f = files.write(entry.getIndex(), h.getPath().toStringUtf8(), h.getClose(), h.getSync(), h.getOffset(), - entry.getStateMachineLogEntry().getStateMachineEntry().getStateMachineData()); + entry.getStateMachineLogEntry().getStateMachineEntry().getStateMachineData() + ).whenComplete((r, e) -> entryRef.release()); // sync only if closing the file - return h.getClose()? f: null; + return h.getClose() ? f: null; } static FileStoreRequestProto getProto(TransactionContext context, LogEntryProto entry) { @@ -144,14 +148,14 @@ static FileStoreRequestProto getProto(TransactionContext context, LogEntryProto return proto; } } - return getProto(entry); + return getProto(entry.getStateMachineLogEntry().getLogData()); } - static FileStoreRequestProto getProto(LogEntryProto entry) { + static FileStoreRequestProto getProto(ByteString bytes) { try { - return FileStoreRequestProto.parseFrom(entry.getStateMachineLogEntry().getLogData()); + return FileStoreRequestProto.parseFrom(bytes); } catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Failed to parse data, entry=" + entry, e); + throw new IllegalArgumentException("Failed to parse data", e); } } @@ -224,7 +228,7 @@ public CompletableFuture link(DataStream stream, LogEntryProto entry) { @Override public CompletableFuture applyTransaction(TransactionContext trx) { - final LogEntryProto entry = trx.getLogEntry(); + final LogEntryProto entry = trx.getLogEntryUnsafe(); final long index = entry.getIndex(); updateLastAppliedTermIndex(entry.getTerm(), index); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java index 086b791e49..caf2aa59b2 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java @@ -18,7 +18,6 @@ package org.apache.ratis.examples.filestore.cli; import com.beust.jcommander.Parameter; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.ratis.RaftConfigKeys; import org.apache.ratis.client.RaftClient; import org.apache.ratis.client.RaftClientConfigKeys; @@ -143,12 +142,10 @@ public List getClients(RaftProperties raftProperties) { } - @SuppressFBWarnings("DM_EXIT") - protected void stop(List clients) throws IOException { + protected void close(List clients) throws IOException { for (FileStoreClient client : clients) { client.close(); } - System.exit(0); } public String getPath(String fileName) { @@ -162,6 +159,9 @@ protected void dropCache() { Process pro = Runtime.getRuntime().exec(cmds); pro.waitFor(); } catch (Throwable t) { + if (t instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } System.err.println("Failed to run command:" + Arrays.toString(cmds) + ":" + t.getMessage()); } } diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/DataStream.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/DataStream.java index 4e4b4dbc64..b15b860570 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/DataStream.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/DataStream.java @@ -116,7 +116,8 @@ private boolean checkParam() { @Override protected void operation(List clients) throws IOException, ExecutionException, InterruptedException { if (!checkParam()) { - stop(clients); + close(clients); + return; } final ExecutorService executor = Executors.newFixedThreadPool(getNumThread()); @@ -136,7 +137,7 @@ protected void operation(List clients) throws IOException, Exec System.out.println("Total data written: " + totalWrittenBytes + " bytes"); System.out.println("Total time taken: " + (endTime - startTime) + " millis"); - stop(clients); + close(clients); } private Map>>> streamWrite( diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/LoadGen.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/LoadGen.java index 8225df9eef..42ffac4e1c 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/LoadGen.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/LoadGen.java @@ -64,7 +64,7 @@ protected void operation(List clients) throws IOException, Exec System.out.println("Total data written: " + totalWrittenBytes + " bytes"); System.out.println("Total time taken: " + (endTime - startTime) + " millis"); - stop(clients); + close(clients); } long write(FileChannel in, long offset, FileStoreClient fileStoreClient, String path, diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java b/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java index a846cd1e77..2145412e04 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/membership/server/CServer.java @@ -32,7 +32,6 @@ import org.apache.ratis.thirdparty.com.google.common.base.MoreObjects; import org.apache.ratis.util.FileUtils; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.util.Collections; @@ -40,7 +39,7 @@ /** * A simple raft server using {@link CounterStateMachine}. */ -public class CServer implements Closeable { +public class CServer { public static final RaftGroupId GROUP_ID = RaftGroupId.randomId(); public static final String LOCAL_ADDR = "0.0.0.0"; @@ -78,7 +77,6 @@ public RaftPeer getPeer() { return server.getPeer(); } - @Override public void close() throws IOException { server.close(); FileUtils.deleteFully(storageDir); diff --git a/ratis-examples/src/main/resources/log4j.properties b/ratis-examples/src/main/resources/log4j.properties index f5b4baf67c..eedefe287e 100644 --- a/ratis-examples/src/main/resources/log4j.properties +++ b/ratis-examples/src/main/resources/log4j.properties @@ -16,10 +16,8 @@ log4j.rootLogger=INFO, stdout -log4j.logger.org.apache.ratis.server.impl.RatisServerImpl=DEBUG -log4j.logger.org.apache.ratis.client.RaftClient=DEBUG log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java index 49a9102940..5ce96da39a 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java +++ b/ratis-examples/src/test/java/org/apache/ratis/TestMultiRaftGroup.java @@ -22,50 +22,42 @@ import org.apache.ratis.examples.arithmetic.ArithmeticStateMachine; import org.apache.ratis.examples.arithmetic.TestArithmetic; import org.apache.ratis.protocol.RaftGroup; -import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.impl.GroupManagementBaseTest; import org.apache.ratis.server.impl.MiniRaftCluster; -import org.apache.ratis.util.Slf4jUtils; +import org.apache.ratis.test.tag.Flaky; import org.apache.ratis.util.function.CheckedBiConsumer; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.event.Level; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.Collection; import java.util.concurrent.atomic.AtomicInteger; -@RunWith(Parameterized.class) +@Flaky("RATIS-2218") +@Timeout(value = 300) public class TestMultiRaftGroup extends BaseTest { - static { - Slf4jUtils.setLogLevel(RaftServer.Division.LOG, Level.DEBUG); - } - - @Parameterized.Parameters - public static Collection data() throws IOException { + public static Collection data() { return ParameterizedBaseTest.getMiniRaftClusters(ArithmeticStateMachine.class, 0); } - @Parameterized.Parameter - public MiniRaftCluster cluster; - - @Test - public void testMultiRaftGroup() throws Exception { - runTestMultiRaftGroup(3, 6, 9, 12, 15); + @ParameterizedTest + @MethodSource("data") + public void testMultiRaftGroup(MiniRaftCluster cluster) throws Exception { + runTestMultiRaftGroup(cluster, 3, 6, 9, 12, 15); } - private void runTestMultiRaftGroup(int... idIndex) throws Exception { - runTestMultiRaftGroup(idIndex, -1); + private void runTestMultiRaftGroup(MiniRaftCluster cluster, int... idIndex) throws Exception { + runTestMultiRaftGroup(cluster, idIndex, -1); } private final AtomicInteger start = new AtomicInteger(3); private final int count = 10; - private void runTestMultiRaftGroup(int[] idIndex, int chosen) throws Exception { + private void runTestMultiRaftGroup(MiniRaftCluster cluster, int[] idIndex, int chosen) throws Exception { - final CheckedBiConsumer checker = (cluster, group) -> { - try (final RaftClient client = cluster.createClient(group)) { + final CheckedBiConsumer checker = (c, group) -> { + try (final RaftClient client = c.createClient(group)) { TestArithmetic.runTestPythagorean(client, start.getAndAdd(2*count), count); } }; diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java b/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java index 9352a24e80..12445d1d2c 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java @@ -28,34 +28,36 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.AfterClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -@RunWith(Parameterized.class) public abstract class ParameterizedBaseTest extends BaseTest { public static final Logger LOG = LoggerFactory.getLogger(ParameterizedBaseTest.class); /** Subclasses should override this method to provide real data parameters. */ - @Parameterized.Parameters public static Collection data() { return Collections.emptyList(); } - /** For {@link Parameterized} test so that a cluster can be shared by multiple {@link Test} */ - private static final AtomicReference currentCluster = new AtomicReference<>(); + /** For {@link ParameterizedTest} so that a cluster can be shared by multiple {@link Test} */ + private static final AtomicReference CURRENT_CLUSTER = new AtomicReference<>(); - /** Set {@link #currentCluster} to the given cluster and start it if {@link #currentCluster} is changed. */ + /** Set {@link #CURRENT_CLUSTER} to the given cluster and start it if {@link #CURRENT_CLUSTER} is changed. */ public static void setAndStart(MiniRaftCluster cluster) throws InterruptedException, IOException { - final MiniRaftCluster previous = currentCluster.getAndSet(cluster); + final MiniRaftCluster previous = CURRENT_CLUSTER.getAndSet(cluster); if (previous != cluster) { if (previous != null) { previous.shutdown(); @@ -66,9 +68,9 @@ public static void setAndStart(MiniRaftCluster cluster) throws InterruptedExcept } } - @AfterClass + @AfterAll public static void shutdownCurrentCluster() { - final MiniRaftCluster cluster = currentCluster.getAndSet(null); + final MiniRaftCluster cluster = CURRENT_CLUSTER.getAndSet(null); if (cluster != null) { cluster.shutdown(); } diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmetic.java b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmetic.java index 0fa295c544..3d9d89a7db 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmetic.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmetic.java @@ -27,9 +27,9 @@ import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.Preconditions; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.event.Level; import java.io.IOException; @@ -44,16 +44,13 @@ public class TestArithmetic extends ParameterizedBaseTest { Slf4jUtils.setLogLevel(ArithmeticStateMachine.LOG, Level.DEBUG); } - @Parameterized.Parameters public static Collection data() { return getMiniRaftClusters(ArithmeticStateMachine.class, 3); } - @Parameterized.Parameter - public MiniRaftCluster cluster; - - @Test - public void testPythagorean() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testPythagorean(MiniRaftCluster cluster) throws Exception { setAndStart(cluster); try (final RaftClient client = cluster.createClient()) { runTestPythagorean(client, 3, 10); @@ -85,8 +82,9 @@ public static void runTestPythagorean( } } - @Test - public void testGaussLegendre() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testGaussLegendre(MiniRaftCluster cluster) throws Exception { setAndStart(cluster); try (final RaftClient client = cluster.createClient()) { runGaussLegendre(client); @@ -109,7 +107,8 @@ void runGaussLegendre(RaftClient client) throws IOException { final Variable p0 = new Variable("p" + i_1); final Variable a1 = defineVariable(client, "a"+i, DIV.apply(ADD.apply(a0, b0), 2)); final Variable b1 = defineVariable(client, "b"+i, SQRT.apply(MULT.apply(a0, b0))); - final Variable t1 = defineVariable(client, "t"+i, SUBTRACT.apply(t0, MULT.apply(p0, SQUARE.apply(SUBTRACT.apply(a0, a1))))); + final Variable t1 = defineVariable(client, "t"+i, + SUBTRACT.apply(t0, MULT.apply(p0, SQUARE.apply(SUBTRACT.apply(a0, a1))))); final Variable p1 = defineVariable(client, "p"+i, MULT.apply(2, p0)); final Variable pi_i = new Variable("pi_"+i); @@ -117,14 +116,14 @@ void runGaussLegendre(RaftClient client) throws IOException { final double pi = e.evaluate(null); if (converged) { - Assert.assertTrue(pi == previous); + Assertions.assertEquals(pi, previous); } else if (pi == previous) { converged = true; } LOG.info("{} = {}, converged? {}", pi_i, pi, converged); previous = pi; } - Assert.assertTrue(converged); + Assertions.assertTrue(converged); } static Variable defineVariable(RaftClient client, String name, double value) throws IOException { @@ -145,7 +144,7 @@ static Expression assign(RaftClient client, Variable x, double value) throws IOE static void assignNull(RaftClient client, Variable x) throws IOException { final Expression e = assign(client, x, NullValue.getInstance()); - Assert.assertEquals(NullValue.getInstance(), e); + Assertions.assertEquals(NullValue.getInstance(), e); } static Expression assign(RaftClient client, Variable x, Expression e) throws IOException { @@ -158,11 +157,11 @@ static Expression assign(RaftClient client, Variable x, Expression e, Double exp } static Expression assertRaftClientReply(RaftClientReply reply, Double expected) { - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); final Expression e = Expression.Utils.bytes2Expression( reply.getMessage().getContent().toByteArray(), 0); if (expected != null) { - Assert.assertEquals(expected, e.evaluate(null)); + Assertions.assertEquals(expected, e.evaluate(null)); } return e; } diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmeticLogDump.java b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmeticLogDump.java index 564afaac2c..f17ebe2604 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmeticLogDump.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/TestArithmeticLogDump.java @@ -34,10 +34,10 @@ import org.apache.ratis.tools.ParseRatisLog; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; public class TestArithmeticLogDump extends BaseTest { @@ -47,26 +47,26 @@ public class TestArithmeticLogDump extends BaseTest { public static final int NUM_SERVERS = 1; - protected static final RaftProperties properties = new RaftProperties(); + protected static final RaftProperties PROPERTIES = new RaftProperties(); private final MiniRaftClusterWithSimulatedRpc cluster = MiniRaftClusterWithSimulatedRpc .FACTORY.newCluster(NUM_SERVERS, getProperties()); public RaftProperties getProperties() { RaftServerConfigKeys.Rpc - .setSlownessTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS)); - properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, + .setSlownessTimeout(PROPERTIES, TimeDuration.valueOf(1, TimeUnit.SECONDS)); + PROPERTIES.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class); - return properties; + return PROPERTIES; } - @Before + @BeforeEach public void setup() throws IOException { - Assert.assertNull(cluster.getLeader()); + Assertions.assertNull(cluster.getLeader()); cluster.start(); } - @After + @AfterEach public void tearDown() { if (cluster != null) { cluster.shutdown(); @@ -77,7 +77,7 @@ public void tearDown() { public void testLogDump() throws Exception { final RaftServer.Division leaderServer = RaftTestUtil.waitForLeader(cluster); final List files = LogSegmentPath.getLogSegmentPaths(leaderServer.getRaftStorage()); - Assert.assertEquals(1, files.size()); + Assertions.assertEquals(1, files.size()); cluster.shutdown(); ParseRatisLog.Builder builder = new ParseRatisLog.Builder(); diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/cli/TestAssignCli.java b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/cli/TestAssignCli.java index 3052adba10..c28b4d2ddf 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/cli/TestAssignCli.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/cli/TestAssignCli.java @@ -19,8 +19,8 @@ import org.apache.ratis.examples.arithmetic.expression.DoubleValue; import org.apache.ratis.examples.arithmetic.expression.Variable; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import static org.apache.ratis.examples.arithmetic.expression.BinaryExpression.Op.ADD; import static org.apache.ratis.examples.arithmetic.expression.BinaryExpression.Op.MULT; @@ -32,55 +32,55 @@ public class TestAssignCli { @Test public void createExpression() { - Assert.assertEquals( + Assertions.assertEquals( new DoubleValue(2.0), new Assign().createExpression("2.0")); - Assert.assertEquals( + Assertions.assertEquals( new DoubleValue(42.0), new Assign().createExpression("42")); - Assert.assertEquals( + Assertions.assertEquals( MULT.apply(2.0, new Variable("a")), new Assign().createExpression("2*a")); - Assert.assertEquals( + Assertions.assertEquals( MULT.apply(new Variable("v1"), 2.0), new Assign().createExpression("v1 * 2")); - Assert.assertEquals( + Assertions.assertEquals( ADD.apply(2.0, 1.0), new Assign().createExpression("2+1")); - Assert.assertEquals( + Assertions.assertEquals( SUBTRACT.apply(1.0, 6.0), new Assign().createExpression("1 - 6")); - Assert.assertEquals( + Assertions.assertEquals( ADD.apply(new Variable("a"), new Variable("v2")), new Assign().createExpression("a+v2")); - Assert.assertEquals( + Assertions.assertEquals( ADD.apply(new Variable("v1"), new Variable("b")), new Assign().createExpression("v1 + b")); - Assert.assertEquals( + Assertions.assertEquals( SQRT.apply(new Variable("a")), new Assign().createExpression("√a")); - Assert.assertEquals( + Assertions.assertEquals( SQRT.apply(new Variable("ABC")), new Assign().createExpression("√ABC")); - Assert.assertEquals( + Assertions.assertEquals( SQRT.apply(2.0), new Assign().createExpression("√2")); - Assert.assertEquals( + Assertions.assertEquals( NEG.apply(2.0), new Assign().createExpression("~2.0")); - Assert.assertEquals( + Assertions.assertEquals( MINUS.apply(6.0), new Assign().createExpression("-6.0")); } diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/expression/TestExpression.java b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/expression/TestExpression.java index 6996fe9786..b512c617f4 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/expression/TestExpression.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/arithmetic/expression/TestExpression.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,18 +19,15 @@ import org.apache.ratis.BaseTest; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; +@Timeout(value = 1) public class TestExpression extends BaseTest { - @Override - public int getGlobalTimeoutSeconds() { - return 1; - } - @Test public void testArithmeticUtils() throws Exception { final Random ran = ThreadLocalRandom.current(); @@ -42,21 +39,21 @@ public void testArithmeticUtils() throws Exception { final int n = ran.nextInt(); Expression.Utils.int2bytes(n, buf, offset); final int m = Expression.Utils.bytes2int(buf, offset); - Assert.assertEquals(n, m); + Assertions.assertEquals(n, m); offset += 4; } { final long n = ran.nextLong(); Expression.Utils.long2bytes(n, buf, offset); final long m = Expression.Utils.bytes2long(buf, offset); - Assert.assertEquals(n, m); + Assertions.assertEquals(n, m); offset += 8; } { final double n = ran.nextDouble(); Expression.Utils.double2bytes(n, buf, offset); final double m = Expression.Utils.bytes2double(buf, offset); - Assert.assertTrue(n == m); + Assertions.assertEquals(n, m); offset += 8; } } @@ -65,11 +62,11 @@ public void testArithmeticUtils() throws Exception { public void testOp() throws Exception { for(BinaryExpression.Op op : BinaryExpression.Op.values()) { final byte b = op.byteValue(); - Assert.assertEquals(op, BinaryExpression.Op.valueOf(b)); + Assertions.assertEquals(op, BinaryExpression.Op.valueOf(b)); } for(UnaryExpression.Op op : UnaryExpression.Op.values()) { final byte b = op.byteValue(); - Assert.assertEquals(op, UnaryExpression.Op.valueOf(b)); + Assertions.assertEquals(op, UnaryExpression.Op.valueOf(b)); } } @@ -84,9 +81,9 @@ public void testExpression() throws Exception { final int len = a.toBytes(buf, offset); final Variable a2 = new Variable(buf, offset); LOG.info("var a2: " + a2); - Assert.assertEquals(a.getName(), a2.getName()); - Assert.assertEquals(len, a.length()); - Assert.assertEquals(len, a2.length()); + Assertions.assertEquals(a.getName(), a2.getName()); + Assertions.assertEquals(len, a.length()); + Assertions.assertEquals(len, a2.length()); offset += len; } @@ -96,9 +93,9 @@ public void testExpression() throws Exception { final int len = three.toBytes(buf, offset); final DoubleValue three2 = new DoubleValue(buf, offset); LOG.info("double three2: " + three2.evaluate(null)); - Assert.assertTrue(three.evaluate(null).equals(three2.evaluate(null))); - Assert.assertEquals(len, three.length()); - Assert.assertEquals(len, three2.length()); + Assertions.assertEquals(three.evaluate(null), three2.evaluate(null)); + Assertions.assertEquals(len, three.length()); + Assertions.assertEquals(len, three2.length()); } } } diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/common/TestSubCommand.java b/ratis-examples/src/test/java/org/apache/ratis/examples/common/TestSubCommand.java index 71fe7e6a4a..c47dbb82a1 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/common/TestSubCommand.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/common/TestSubCommand.java @@ -21,24 +21,21 @@ import java.util.Collection; import java.util.Collections; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; -@RunWith(Parameterized.class) public class TestSubCommand { - @Parameterized.Parameters public static Collection data() { return Collections.singleton("127.0.0.1:6667"); } - @Parameterized.Parameter - public String peers; - - @Test(expected = IllegalArgumentException.class) - public void testParsePeers() { - SubCommandBase.parsePeers(peers); + @ParameterizedTest + @MethodSource("data") + public void testParsePeers(String peers) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> SubCommandBase.parsePeers(peers)); } } diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/counter/TestCounter.java b/ratis-examples/src/test/java/org/apache/ratis/examples/counter/TestCounter.java index 6fbe8770e3..3d188010d6 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/counter/TestCounter.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/counter/TestCounter.java @@ -24,42 +24,39 @@ import org.apache.ratis.examples.counter.server.CounterStateMachine; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientReply; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; import java.util.Collection; public class TestCounter extends ParameterizedBaseTest { - @Parameterized.Parameters public static Collection data() { return getMiniRaftClusters(CounterStateMachine.class, 3); } - @Parameterized.Parameter - public MiniRaftCluster cluster; - - @Test - public void testSeveralCounter() throws IOException, InterruptedException { + @ParameterizedTest + @MethodSource("data") + public void testSeveralCounter(MiniRaftCluster cluster) throws IOException, InterruptedException { setAndStart(cluster); try (final RaftClient client = cluster.createClient()) { for (int i = 0; i < 10; i++) { client.io().send(Message.valueOf("INCREMENT")); } RaftClientReply reply1 = client.io().sendReadOnly(Message.valueOf("GET")); - Assert.assertEquals(10, reply1.getMessage().getContent().asReadOnlyByteBuffer().getInt()); + Assertions.assertEquals(10, reply1.getMessage().getContent().asReadOnlyByteBuffer().getInt()); for (int i = 0; i < 10; i++) { client.io().send(Message.valueOf("INCREMENT")); } RaftClientReply reply2 = client.io().sendReadOnly(Message.valueOf("GET")); - Assert.assertEquals(20, reply2.getMessage().getContent().asReadOnlyByteBuffer().getInt()); + Assertions.assertEquals(20, reply2.getMessage().getContent().asReadOnlyByteBuffer().getInt()); for (int i = 0; i < 10; i++) { client.io().send(Message.valueOf("INCREMENT")); } RaftClientReply reply3 = client.io().sendReadOnly(Message.valueOf("GET")); - Assert.assertEquals(30, reply3.getMessage().getContent().asReadOnlyByteBuffer().getInt()); + Assertions.assertEquals(30, reply3.getMessage().getContent().asReadOnlyByteBuffer().getInt()); } } } diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreAsyncBaseTest.java b/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreAsyncBaseTest.java index eb51e643b0..0b5071c448 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreAsyncBaseTest.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreAsyncBaseTest.java @@ -32,7 +32,11 @@ import java.io.File; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; public abstract class FileStoreAsyncBaseTest extends BaseTest diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreStreamingBaseTest.java b/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreStreamingBaseTest.java index 92147de4e9..490f12c8da 100644 --- a/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreStreamingBaseTest.java +++ b/ratis-examples/src/test/java/org/apache/ratis/examples/filestore/FileStoreStreamingBaseTest.java @@ -56,6 +56,7 @@ public abstract class FileStoreStreamingBaseTest +public class TestFileStoreStreamingWithGrpcCluster + extends FileStoreStreamingBaseTest implements MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.FactoryGet { } diff --git a/ratis-experiments/pom.xml b/ratis-experiments/pom.xml index 2099142ea3..39ad9668ff 100644 --- a/ratis-experiments/pom.xml +++ b/ratis-experiments/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-experiments Apache Ratis Experiments + + + true + + org.apache.ratis @@ -120,8 +125,8 @@ shade - ${project.build.directory}/dependency-reduced-pom.xml - true + false + false *:* @@ -140,23 +145,6 @@ - - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - copy-file - package - - copy - - - target/${project.artifactId}-${project.version}-shaded.jar - target/${project.artifactId}-${project.version}.jar - - - - diff --git a/ratis-grpc/pom.xml b/ratis-grpc/pom.xml index 3352ce16fb..c555b8e4c2 100644 --- a/ratis-grpc/pom.xml +++ b/ratis-grpc/pom.xml @@ -17,7 +17,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-grpc @@ -53,6 +53,10 @@ test test-jar + + org.apache.ratis + ratis-server-api + ratis-server org.apache.ratis @@ -71,8 +75,8 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api test @@ -80,12 +84,5 @@ mockito-core test - - - com.github.spotbugs - spotbugs-annotations - provided - true - diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcConfigKeys.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcConfigKeys.java index 8caacfeeb5..2fcb9b6b0a 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcConfigKeys.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcConfigKeys.java @@ -19,6 +19,7 @@ import org.apache.ratis.conf.Parameters; import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.grpc.server.GrpcServices; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; @@ -230,15 +231,6 @@ static void setAsyncRequestThreadPoolSize(RaftProperties properties, int port) { setInt(properties::setInt, ASYNC_REQUEST_THREAD_POOL_SIZE_KEY, port); } - String TLS_CONF_PARAMETER = PREFIX + ".tls.conf"; - Class TLS_CONF_CLASS = TLS.CONF_CLASS; - static GrpcTlsConfig tlsConf(Parameters parameters) { - return parameters != null ? parameters.get(TLS_CONF_PARAMETER, TLS_CONF_CLASS): null; - } - static void setTlsConf(Parameters parameters, GrpcTlsConfig conf) { - parameters.put(TLS_CONF_PARAMETER, conf, TLS_CONF_CLASS); - } - String LEADER_OUTSTANDING_APPENDS_MAX_KEY = PREFIX + ".leader.outstanding.appends.max"; int LEADER_OUTSTANDING_APPENDS_MAX_DEFAULT = 8; static int leaderOutstandingAppendsMax(RaftProperties properties) { @@ -292,6 +284,34 @@ static void setLogMessageBatchDuration(RaftProperties properties, setTimeDuration(properties::setTimeDuration, LOG_MESSAGE_BATCH_DURATION_KEY, logMessageBatchDuration); } + + String ZERO_COPY_ENABLED_KEY = PREFIX + ".zerocopy.enabled"; + boolean ZERO_COPY_ENABLED_DEFAULT = false; + static boolean zeroCopyEnabled(RaftProperties properties) { + return getBoolean(properties::getBoolean, ZERO_COPY_ENABLED_KEY, ZERO_COPY_ENABLED_DEFAULT, getDefaultLog()); + } + static void setZeroCopyEnabled(RaftProperties properties, boolean enabled) { + setBoolean(properties::setBoolean, ZERO_COPY_ENABLED_KEY, enabled); + } + + String SERVICES_CUSTOMIZER_PARAMETER = PREFIX + ".services.customizer"; + Class SERVICES_CUSTOMIZER_CLASS = GrpcServices.Customizer.class; + static GrpcServices.Customizer servicesCustomizer(Parameters parameters) { + return parameters == null ? null + : parameters.get(SERVICES_CUSTOMIZER_PARAMETER, SERVICES_CUSTOMIZER_CLASS); + } + static void setServicesCustomizer(Parameters parameters, GrpcServices.Customizer customizer) { + parameters.put(SERVICES_CUSTOMIZER_PARAMETER, customizer, SERVICES_CUSTOMIZER_CLASS); + } + + String TLS_CONF_PARAMETER = PREFIX + ".tls.conf"; + Class TLS_CONF_CLASS = TLS.CONF_CLASS; + static GrpcTlsConfig tlsConf(Parameters parameters) { + return parameters != null ? parameters.get(TLS_CONF_PARAMETER, TLS_CONF_CLASS): null; + } + static void setTlsConf(Parameters parameters, GrpcTlsConfig conf) { + parameters.put(TLS_CONF_PARAMETER, conf, TLS_CONF_CLASS); + } } String MESSAGE_SIZE_MAX_KEY = PREFIX + ".message.size.max"; diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcFactory.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcFactory.java index 75eb34a2d1..331d1a8585 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcFactory.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcFactory.java @@ -22,7 +22,8 @@ import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.grpc.client.GrpcClientRpc; import org.apache.ratis.grpc.server.GrpcLogAppender; -import org.apache.ratis.grpc.server.GrpcService; +import org.apache.ratis.grpc.server.GrpcServices; +import org.apache.ratis.grpc.server.GrpcServicesImpl; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.rpc.SupportedRpcType; import org.apache.ratis.server.RaftServer; @@ -64,6 +65,8 @@ static boolean checkPooledByteBufAllocatorUseCacheForAllThreads(Consumer return value; } + private final GrpcServices.Customizer servicesCustomizer; + private final GrpcTlsConfig tlsConfig; private final GrpcTlsConfig adminTlsConfig; private final GrpcTlsConfig clientTlsConfig; @@ -76,7 +79,7 @@ public static Parameters newRaftParameters(GrpcTlsConfig conf) { } public GrpcFactory(Parameters parameters) { - this( + this(GrpcConfigKeys.Server.servicesCustomizer(parameters), GrpcConfigKeys.TLS.conf(parameters), GrpcConfigKeys.Admin.tlsConf(parameters), GrpcConfigKeys.Client.tlsConf(parameters), @@ -85,11 +88,14 @@ public GrpcFactory(Parameters parameters) { } public GrpcFactory(GrpcTlsConfig tlsConfig) { - this(tlsConfig, null, null, null); + this(null, tlsConfig, null, null, null); } - private GrpcFactory(GrpcTlsConfig tlsConfig, GrpcTlsConfig adminTlsConfig, + private GrpcFactory(GrpcServices.Customizer servicesCustomizer, + GrpcTlsConfig tlsConfig, GrpcTlsConfig adminTlsConfig, GrpcTlsConfig clientTlsConfig, GrpcTlsConfig serverTlsConfig) { + this.servicesCustomizer = servicesCustomizer; + this.tlsConfig = tlsConfig; this.adminTlsConfig = adminTlsConfig; this.clientTlsConfig = clientTlsConfig; @@ -123,10 +129,11 @@ public LogAppender newLogAppender(RaftServer.Division server, LeaderState state, } @Override - public GrpcService newRaftServerRpc(RaftServer server) { + public GrpcServices newRaftServerRpc(RaftServer server) { checkPooledByteBufAllocatorUseCacheForAllThreads(LOG::info); - return GrpcService.newBuilder() + return GrpcServicesImpl.newBuilder() .setServer(server) + .setCustomizer(servicesCustomizer) .setAdminTlsConfig(getAdminTlsConfig()) .setServerTlsConfig(getServerTlsConfig()) .setClientTlsConfig(getClientTlsConfig()) diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcUtil.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcUtil.java index 22653b6efb..e17d17bff6 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcUtil.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/GrpcUtil.java @@ -24,8 +24,12 @@ import org.apache.ratis.security.TlsConf.CertificatesConf; import org.apache.ratis.security.TlsConf.PrivateKeyConf; import org.apache.ratis.security.TlsConf.KeyManagerConf; +import org.apache.ratis.thirdparty.com.google.protobuf.MessageLite; import org.apache.ratis.thirdparty.io.grpc.ManagedChannel; import org.apache.ratis.thirdparty.io.grpc.Metadata; +import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor; +import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; import org.apache.ratis.thirdparty.io.grpc.Status; import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; @@ -59,14 +63,8 @@ public interface GrpcUtil { Metadata.Key.of("heartbeat", Metadata.ASCII_STRING_MARSHALLER); static StatusRuntimeException wrapException(Throwable t) { - return wrapException(t, -1); - } - - static StatusRuntimeException wrapException(Throwable t, long callId) { t = JavaUtils.unwrapCompletionException(t); - Metadata trailers = new StatusRuntimeExceptionMetadataBuilder(t) - .addCallId(callId) - .build(); + Metadata trailers = new StatusRuntimeExceptionMetadataBuilder(t).build(); return wrapException(t, trailers); } @@ -163,13 +161,6 @@ static IOException unwrapIOException(Throwable t) { return e; } - static void asyncCall( - StreamObserver responseObserver, - CheckedSupplier, IOException> supplier, - Function toProto) { - asyncCall(responseObserver, supplier, toProto, throwable -> {}); - } - static void asyncCall( StreamObserver responseObserver, CheckedSupplier, IOException> supplier, @@ -304,4 +295,26 @@ static void setKeyManager(SslContextBuilder b, KeyManagerConf keyManagerConfig) b.keyManager(privateKey.get(), certificates.get()); } } + + /** + * Used to add a method to Service definition with a custom request marshaller. + * + * @param orig original service definition. + * @param newServiceBuilder builder of the new service definition. + * @param origMethod the original method definition. + * @param customMarshaller custom marshaller to be set for the method. + * @param + * @param + */ + static void addMethodWithCustomMarshaller( + ServerServiceDefinition orig, ServerServiceDefinition.Builder newServiceBuilder, + MethodDescriptor origMethod, MethodDescriptor.PrototypeMarshaller customMarshaller) { + MethodDescriptor newMethod = origMethod.toBuilder() + .setRequestMarshaller(customMarshaller) + .build(); + @SuppressWarnings("unchecked") + ServerCallHandler serverCallHandler = + (ServerCallHandler) orig.getMethod(newMethod.getFullMethodName()).getServerCallHandler(); + newServiceBuilder.addMethod(newMethod, serverCallHandler); + } } diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/client/GrpcClientProtocolClient.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/client/GrpcClientProtocolClient.java index 08bacdb73b..3b9d512683 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/client/GrpcClientProtocolClient.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/client/GrpcClientProtocolClient.java @@ -128,8 +128,11 @@ private ManagedChannel buildChannel(String address, GrpcTlsConfig tlsConf, SizeInBytes flowControlWindow, SizeInBytes maxMessageSize) { NettyChannelBuilder channelBuilder = NettyChannelBuilder.forTarget(address); + // ignore any http proxy for grpc + channelBuilder.proxyDetector(uri -> null); if (tlsConf != null) { + LOG.debug("Setting TLS for {}", address); SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); GrpcUtil.setTrustManager(sslContextBuilder, tlsConf.getTrustManager()); if (tlsConf.getMtlsEnabled()) { diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/ZeroCopyMetrics.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/ZeroCopyMetrics.java new file mode 100644 index 0000000000..1fcc317f9d --- /dev/null +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/ZeroCopyMetrics.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.grpc.metrics; + +import org.apache.ratis.metrics.LongCounter; +import org.apache.ratis.metrics.MetricRegistryInfo; +import org.apache.ratis.metrics.RatisMetricRegistry; +import org.apache.ratis.metrics.RatisMetrics; +import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.ratis.thirdparty.com.google.protobuf.AbstractMessage; + +import java.util.function.Supplier; + +public class ZeroCopyMetrics extends RatisMetrics { + private static final String RATIS_GRPC_METRICS_APP_NAME = "ratis_grpc"; + private static final String RATIS_GRPC_METRICS_COMP_NAME = "zero_copy"; + private static final String RATIS_GRPC_METRICS_DESC = "Metrics for Ratis Grpc Zero copy"; + + private final LongCounter zeroCopyMessages = getRegistry().counter("num_zero_copy_messages"); + private final LongCounter nonZeroCopyMessages = getRegistry().counter("num_non_zero_copy_messages"); + private final LongCounter releasedMessages = getRegistry().counter("num_released_messages"); + + public ZeroCopyMetrics() { + super(createRegistry()); + } + + private static RatisMetricRegistry createRegistry() { + return create(new MetricRegistryInfo("", + RATIS_GRPC_METRICS_APP_NAME, + RATIS_GRPC_METRICS_COMP_NAME, RATIS_GRPC_METRICS_DESC)); + } + + public void addUnreleased(String name, Supplier unreleased) { + getRegistry().gauge(name + "_num_unreleased_messages", () -> unreleased); + } + + + public void onZeroCopyMessage(AbstractMessage ignored) { + zeroCopyMessages.inc(); + } + + public void onNonZeroCopyMessage(AbstractMessage ignored) { + nonZeroCopyMessages.inc(); + } + + public void onReleasedMessage(AbstractMessage ignored) { + releasedMessages.inc(); + } + + @VisibleForTesting + public long zeroCopyMessages() { + return zeroCopyMessages.getCount(); + } + + @VisibleForTesting + public long nonZeroCopyMessages() { + return nonZeroCopyMessages.getCount(); + } + + @VisibleForTesting + public long releasedMessages() { + return releasedMessages.getCount(); + } +} \ No newline at end of file diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerInterceptor.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerInterceptor.java index 7e20946add..e7c1dd17ce 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerInterceptor.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerInterceptor.java @@ -75,8 +75,7 @@ public ServerCall.Listener interceptCall( } String metricNamePrefix = getMethodMetricPrefix(method); ServerCall monitoringCall = new MetricServerCall<>(call, metricNamePrefix, metrics); - return new MetricServerCallListener<>( - next.startCall(monitoringCall, requestHeaders), metrics); + return next.startCall(monitoringCall, requestHeaders); } @Override diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcAdminProtocolService.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcAdminProtocolService.java index feb780302e..f6a6cfba6a 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcAdminProtocolService.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcAdminProtocolService.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -32,8 +32,12 @@ import org.apache.ratis.proto.RaftProtos.RaftClientReplyProto; import org.apache.ratis.proto.RaftProtos.GroupManagementRequestProto; import org.apache.ratis.proto.grpc.AdminProtocolServiceGrpc.AdminProtocolServiceImplBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GrpcAdminProtocolService extends AdminProtocolServiceImplBase { + static final Logger LOG = LoggerFactory.getLogger(GrpcAdminProtocolService.class); + private final AdminAsynchronousProtocol protocol; public GrpcAdminProtocolService(AdminAsynchronousProtocol protocol) { @@ -45,7 +49,8 @@ public void groupManagement(GroupManagementRequestProto proto, StreamObserver responseObserver) { final GroupManagementRequest request = ClientProtoUtils.toGroupManagementRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.groupManagementAsync(request), - ClientProtoUtils::toRaftClientReplyProto); + ClientProtoUtils::toRaftClientReplyProto, + t -> LOG.warn("Failed groupManagement: {}, {}", proto.getOpCase(), request, t)); } @Override @@ -53,14 +58,16 @@ public void groupList(GroupListRequestProto proto, StreamObserver responseObserver) { final GroupListRequest request = ClientProtoUtils.toGroupListRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.getGroupListAsync(request), - ClientProtoUtils::toGroupListReplyProto); + ClientProtoUtils::toGroupListReplyProto, + t -> LOG.warn("Failed to groupList: {}", request, t)); } @Override public void groupInfo(GroupInfoRequestProto proto, StreamObserver responseObserver) { final GroupInfoRequest request = ClientProtoUtils.toGroupInfoRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.getGroupInfoAsync(request), - ClientProtoUtils::toGroupInfoReplyProto); + ClientProtoUtils::toGroupInfoReplyProto, + t -> LOG.warn("Failed to groupInfo: {}", request, t)); } @Override @@ -68,7 +75,8 @@ public void setConfiguration(SetConfigurationRequestProto proto, StreamObserver responseObserver) { final SetConfigurationRequest request = ClientProtoUtils.toSetConfigurationRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.setConfigurationAsync(request), - ClientProtoUtils::toRaftClientReplyProto); + ClientProtoUtils::toRaftClientReplyProto, + t -> LOG.warn("Failed to setConfiguration: {}", request, t)); } @Override @@ -76,7 +84,8 @@ public void transferLeadership(TransferLeadershipRequestProto proto, StreamObserver responseObserver) { final TransferLeadershipRequest request = ClientProtoUtils.toTransferLeadershipRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.transferLeadershipAsync(request), - ClientProtoUtils::toRaftClientReplyProto); + ClientProtoUtils::toRaftClientReplyProto, + t -> LOG.warn("Failed to transferLeadership: {}", request, t)); } @Override @@ -84,7 +93,8 @@ public void snapshotManagement(SnapshotManagementRequestProto proto, StreamObserver responseObserver) { final SnapshotManagementRequest request = ClientProtoUtils.toSnapshotManagementRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.snapshotManagementAsync(request), - ClientProtoUtils::toRaftClientReplyProto); + ClientProtoUtils::toRaftClientReplyProto, + t -> LOG.warn("Failed snapshotManagement: {}, {}", proto.getOpCase(), request, t)); } @Override @@ -92,6 +102,7 @@ public void leaderElectionManagement(LeaderElectionManagementRequestProto proto, StreamObserver responseObserver) { final LeaderElectionManagementRequest request = ClientProtoUtils.toLeaderElectionManagementRequest(proto); GrpcUtil.asyncCall(responseObserver, () -> protocol.leaderElectionManagementAsync(request), - ClientProtoUtils::toRaftClientReplyProto); + ClientProtoUtils::toRaftClientReplyProto, + t -> LOG.warn("Failed leaderElectionManagement: {}, {}", proto.getOpCase(), request, t)); } } diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcClientProtocolService.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcClientProtocolService.java index 9c19684677..b7548780cd 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcClientProtocolService.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcClientProtocolService.java @@ -19,10 +19,13 @@ import org.apache.ratis.client.impl.ClientProtoUtils; import org.apache.ratis.grpc.GrpcUtil; +import org.apache.ratis.grpc.metrics.ZeroCopyMetrics; +import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller; import org.apache.ratis.protocol.*; import org.apache.ratis.protocol.exceptions.AlreadyClosedException; import org.apache.ratis.protocol.exceptions.GroupMismatchException; import org.apache.ratis.protocol.exceptions.RaftException; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.apache.ratis.proto.RaftProtos.RaftClientReplyProto; import org.apache.ratis.proto.RaftProtos.RaftClientRequestProto; @@ -30,16 +33,15 @@ import org.apache.ratis.util.CollectionUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.SlidingWindow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; @@ -48,15 +50,21 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import static org.apache.ratis.grpc.GrpcUtil.addMethodWithCustomMarshaller; +import static org.apache.ratis.proto.grpc.RaftClientProtocolServiceGrpc.getOrderedMethod; +import static org.apache.ratis.proto.grpc.RaftClientProtocolServiceGrpc.getUnorderedMethod; + class GrpcClientProtocolService extends RaftClientProtocolServiceImplBase { private static final Logger LOG = LoggerFactory.getLogger(GrpcClientProtocolService.class); private static class PendingOrderedRequest implements SlidingWindow.ServerSideRequest { + private final ReferenceCountedObject requestRef; private final RaftClientRequest request; private final AtomicReference reply = new AtomicReference<>(); - PendingOrderedRequest(RaftClientRequest request) { - this.request = request; + PendingOrderedRequest(ReferenceCountedObject requestRef) { + this.requestRef = requestRef; + this.request = requestRef != null ? requestRef.retain() : null; } @Override @@ -76,15 +84,23 @@ public boolean hasReply() { @Override public void setReply(RaftClientReply r) { final boolean set = reply.compareAndSet(null, r); - Preconditions.assertTrue(set, () -> "Reply is already set: request=" + request + ", reply=" + reply); + Preconditions.assertTrue(set, () -> "Reply is already set: request=" + + request.toStringShort() + ", reply=" + reply); } RaftClientReply getReply() { return reply.get(); } - RaftClientRequest getRequest() { - return request; + ReferenceCountedObject getRequestRef() { + return requestRef; + } + + @Override + public void release() { + if (requestRef != null) { + requestRef.release(); + } } @Override @@ -135,18 +151,38 @@ void closeAllExisting(RaftGroupId groupId) { private final ExecutorService executor; private final OrderedStreamObservers orderedStreamObservers = new OrderedStreamObservers(); + private final boolean zeroCopyEnabled; + private final ZeroCopyMessageMarshaller zeroCopyRequestMarshaller; GrpcClientProtocolService(Supplier idSupplier, RaftClientAsynchronousProtocol protocol, - ExecutorService executor) { + ExecutorService executor, boolean zeroCopyEnabled, ZeroCopyMetrics zeroCopyMetrics) { this.idSupplier = idSupplier; this.protocol = protocol; this.executor = executor; + this.zeroCopyEnabled = zeroCopyEnabled; + this.zeroCopyRequestMarshaller = new ZeroCopyMessageMarshaller<>(RaftClientRequestProto.getDefaultInstance(), + zeroCopyMetrics::onZeroCopyMessage, zeroCopyMetrics::onNonZeroCopyMessage, zeroCopyMetrics::onReleasedMessage); + zeroCopyMetrics.addUnreleased("client_protocol", zeroCopyRequestMarshaller::getUnclosedCount); } RaftPeerId getId() { return idSupplier.get(); } + ServerServiceDefinition bindServiceWithZeroCopy() { + ServerServiceDefinition orig = super.bindService(); + if (!zeroCopyEnabled) { + LOG.info("{}: Zero copy is disabled.", getId()); + return orig; + } + ServerServiceDefinition.Builder builder = ServerServiceDefinition.builder(orig.getServiceDescriptor().getName()); + + addMethodWithCustomMarshaller(orig, builder, getOrderedMethod(), zeroCopyRequestMarshaller); + addMethodWithCustomMarshaller(orig, builder, getUnorderedMethod(), zeroCopyRequestMarshaller); + + return builder.build(); + } + @Override public StreamObserver ordered(StreamObserver responseObserver) { final OrderedRequestStreamObserver so = new OrderedRequestStreamObserver(responseObserver); @@ -220,31 +256,38 @@ boolean isClosed() { return isClosed.get(); } - CompletableFuture processClientRequest(RaftClientRequest request, Consumer replyHandler) { - try { - final String errMsg = LOG.isDebugEnabled() ? "processClientRequest for " + request : ""; - return protocol.submitClientRequestAsync(request - ).thenAcceptAsync(replyHandler, executor - ).exceptionally(exception -> { - // TODO: the exception may be from either raft or state machine. - // Currently we skip all the following responses when getting an - // exception from the state machine. - responseError(exception, () -> errMsg); - return null; - }); - } catch (IOException e) { - throw new CompletionException("Failed processClientRequest for " + request + " in " + name, e); - } + CompletableFuture processClientRequest(ReferenceCountedObject requestRef, + Consumer replyHandler) { + final String errMsg = LOG.isDebugEnabled() ? "processClientRequest for " + requestRef.get() : ""; + return protocol.submitClientRequestAsync(requestRef + ).thenAcceptAsync(replyHandler, executor + ).exceptionally(exception -> { + // TODO: the exception may be from either raft or state machine. + // Currently we skip all the following responses when getting an + // exception from the state machine. + responseError(exception, () -> errMsg); + return null; + }); } - abstract void processClientRequest(RaftClientRequest request); + abstract void processClientRequest(ReferenceCountedObject requestRef); @Override public void onNext(RaftClientRequestProto request) { + ReferenceCountedObject requestRef = null; try { final RaftClientRequest r = ClientProtoUtils.toRaftClientRequest(request); - processClientRequest(r); + requestRef = ReferenceCountedObject.wrap(r, () -> {}, released -> { + if (released) { + zeroCopyRequestMarshaller.release(request); + } + }); + + processClientRequest(requestRef); } catch (Exception e) { + if (requestRef == null) { + zeroCopyRequestMarshaller.release(request); + } responseError(e, () -> "onNext for " + ClientProtoUtils.toString(request) + " in " + name); } } @@ -278,15 +321,21 @@ private class UnorderedRequestStreamObserver extends RequestStreamObserver { } @Override - void processClientRequest(RaftClientRequest request) { - final CompletableFuture f = processClientRequest(request, reply -> { - if (!reply.isSuccess()) { - LOG.info("Failed " + request + ", reply=" + reply); - } - final RaftClientReplyProto proto = ClientProtoUtils.toRaftClientReplyProto(reply); - responseNext(proto); - }); - final long callId = request.getCallId(); + void processClientRequest(ReferenceCountedObject requestRef) { + final long callId = requestRef.retain().getCallId(); + final CompletableFuture f; + try { + f = processClientRequest(requestRef, reply -> { + if (!reply.isSuccess()) { + LOG.info("Failed request cid={}, reply={}", callId, reply); + } + final RaftClientReplyProto proto = ClientProtoUtils.toRaftClientReplyProto(reply); + responseNext(proto); + }); + } finally { + requestRef.release(); + } + put(callId, f); f.thenAccept(dummy -> remove(callId)); } @@ -329,31 +378,40 @@ RaftGroupId getGroupId() { void processClientRequest(PendingOrderedRequest pending) { final long seq = pending.getSeqNum(); - processClientRequest(pending.getRequest(), + processClientRequest(pending.getRequestRef(), reply -> slidingWindow.receiveReply(seq, reply, this::sendReply)); } @Override - void processClientRequest(RaftClientRequest r) { - if (isClosed()) { - final AlreadyClosedException exception = new AlreadyClosedException(getName() + ": the stream is closed"); - responseError(exception, () -> "processClientRequest (stream already closed) for " + r); - } + void processClientRequest(ReferenceCountedObject requestRef) { + final RaftClientRequest request = requestRef.retain(); + try { + if (isClosed()) { + final AlreadyClosedException exception = new AlreadyClosedException(getName() + ": the stream is closed"); + responseError(exception, () -> "processClientRequest (stream already closed) for " + request); + } - final RaftGroupId requestGroupId = r.getRaftGroupId(); - // use the group id in the first request as the group id of this observer - final RaftGroupId updated = groupId.updateAndGet(g -> g != null ? g: requestGroupId); - final PendingOrderedRequest pending = new PendingOrderedRequest(r); - - if (!requestGroupId.equals(updated)) { - final GroupMismatchException exception = new GroupMismatchException(getId() - + ": The group (" + requestGroupId + ") of " + r.getClientId() - + " does not match the group (" + updated + ") of the " + JavaUtils.getClassSimpleName(getClass())); - responseError(exception, () -> "processClientRequest (Group mismatched) for " + r); - return; - } + final RaftGroupId requestGroupId = request.getRaftGroupId(); + // use the group id in the first request as the group id of this observer + final RaftGroupId updated = groupId.updateAndGet(g -> g != null ? g : requestGroupId); - slidingWindow.receivedRequest(pending, this::processClientRequest); + if (!requestGroupId.equals(updated)) { + final GroupMismatchException exception = new GroupMismatchException(getId() + + ": The group (" + requestGroupId + ") of " + request.getClientId() + + " does not match the group (" + updated + ") of the " + JavaUtils.getClassSimpleName(getClass())); + responseError(exception, () -> "processClientRequest (Group mismatched) for " + request); + return; + } + final PendingOrderedRequest pending = new PendingOrderedRequest(requestRef); + try { + slidingWindow.receivedRequest(pending, this::processClientRequest); + } catch (Exception e) { + pending.release(); + throw e; + } + } finally { + requestRef.release(); + } } private void sendReply(PendingOrderedRequest ready) { diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java index 9a2e84f21c..c016dabfe4 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcLogAppender.java @@ -41,7 +41,9 @@ import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto.AppendResult; import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto; import org.apache.ratis.proto.RaftProtos.InstallSnapshotReplyProto; +import org.apache.ratis.proto.RaftProtos.InstallSnapshotReplyProto.InstallSnapshotReplyBodyCase; import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto; +import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto.InstallSnapshotRequestBodyCase; import org.apache.ratis.statemachine.SnapshotInfo; import org.apache.ratis.util.*; import org.slf4j.Logger; @@ -49,11 +51,17 @@ import java.io.IOException; import java.io.InterruptedIOException; -import java.util.*; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Queue; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; /** @@ -64,9 +72,12 @@ public class GrpcLogAppender extends LogAppenderBase { private enum BatchLogKey implements BatchLogger.Key { RESET_CLIENT, + INCONSISTENCY_REPLY, APPEND_LOG_RESPONSE_HANDLER_ON_ERROR } + public static final int INSTALL_SNAPSHOT_NOTIFICATION_INDEX = 0; + private static final Comparator CALL_ID_COMPARATOR = (left, right) -> { // calculate diff in order to take care the possibility of numerical overflow final long diff = left - right; @@ -152,7 +163,7 @@ synchronized int process(Event event) { private final TimeDuration logMessageBatchDuration; private final int maxOutstandingInstallSnapshots; private final TimeoutExecutor scheduler = TimeoutExecutor.getInstance(); - + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile StreamObservers appendLogRequestObserver; private final boolean useSeparateHBChannel; @@ -166,7 +177,7 @@ synchronized int process(Event event) { public GrpcLogAppender(RaftServer.Division server, LeaderState leaderState, FollowerInfo f) { super(server, leaderState, f); - Preconditions.assertNotNull(getServerRpc(), "getServerRpc()"); + Objects.requireNonNull(getServerRpc(), "getServerRpc() == null"); final RaftProperties properties = server.getRaftServer().getProperties(); this.maxPendingRequestsNum = GrpcConfigKeys.Server.leaderOutstandingAppendsMax(properties); @@ -188,8 +199,8 @@ public GrpcLogAppender(RaftServer.Division server, LeaderState leaderState, Foll } @Override - public GrpcService getServerRpc() { - return (GrpcService)super.getServerRpc(); + public GrpcServicesImpl getServerRpc() { + return (GrpcServicesImpl)super.getServerRpc(); } private GrpcServerProtocolClient getClient() throws IOException { @@ -213,7 +224,7 @@ private void resetClient(AppendEntriesRequest request, Event event) { .orElseGet(f::getMatchIndex); if (event.isError() && request == null) { final long followerNextIndex = f.getNextIndex(); - BatchLogger.warn(BatchLogKey.RESET_CLIENT, f.getId() + "-" + followerNextIndex, suffix -> + BatchLogger.print(BatchLogKey.RESET_CLIENT, f.getId() + "-" + followerNextIndex, suffix -> LOG.warn("{}: Follower failed (request=null, errorCount={}); keep nextIndex ({}) unchanged and retry.{}", this, errorCount, followerNextIndex, suffix), logMessageBatchDuration); return; @@ -298,8 +309,14 @@ private long errorWaitTimeMs() { @Override public CompletableFuture stopAsync() { - grpcServerMetrics.unregister(); - return super.stopAsync(); + try (AutoCloseableLock ignored = lock.writeLock(caller, LOG::trace)) { + if (appendLogRequestObserver != null) { + appendLogRequestObserver.stop(); + appendLogRequestObserver = null; + } + grpcServerMetrics.unregister(); + return super.stopAsync(); + } } @Override @@ -375,30 +392,42 @@ public Comparator getCallIdComparator() { } private void appendLog(boolean heartbeat) throws IOException { - final AppendEntriesRequestProto pending; + final ReferenceCountedObject pending; final AppendEntriesRequest request; try (AutoCloseableLock writeLock = lock.writeLock(caller, LOG::trace)) { + if (!isRunning()) { + return; + } // Prepare and send the append request. // Note changes on follower's nextIndex and ops on pendingRequests should always be done under the write-lock - pending = newAppendEntriesRequest(callId.getAndIncrement(), heartbeat); + pending = nextAppendEntriesRequest(callId.getAndIncrement(), heartbeat); if (pending == null) { return; } - request = new AppendEntriesRequest(pending, getFollowerId(), grpcServerMetrics); - pendingRequests.put(request); - increaseNextIndex(pending); - if (appendLogRequestObserver == null) { - appendLogRequestObserver = new StreamObservers( - getClient(), new AppendLogResponseHandler(), useSeparateHBChannel, getWaitTimeMin()); + try { + request = new AppendEntriesRequest(pending.get(), getFollowerId(), grpcServerMetrics); + pendingRequests.put(request); + increaseNextIndex(pending.get()); + if (appendLogRequestObserver == null) { + appendLogRequestObserver = new StreamObservers( + getClient(), new AppendLogResponseHandler(), useSeparateHBChannel, getWaitTimeMin()); + } + } catch (Exception e) { + pending.release(); + throw e; } } - final TimeDuration remaining = getRemainingWaitTime(); - if (remaining.isPositive()) { - sleep(remaining, heartbeat); - } - if (isRunning()) { - sendRequest(request, pending); + try { + final TimeDuration remaining = getRemainingWaitTime(); + if (remaining.isPositive()) { + sleep(remaining, heartbeat); + } + if (isRunning()) { + sendRequest(request, pending.get()); + } + } finally { + pending.release(); } } @@ -415,7 +444,7 @@ private static void sleep(TimeDuration waitTime, boolean heartbeat) private void sendRequest(AppendEntriesRequest request, AppendEntriesRequestProto proto) throws InterruptedIOException { - CodeInjectionForTesting.execute(GrpcService.GRPC_SEND_SERVER_REQUEST, + CodeInjectionForTesting.execute(GrpcServicesImpl.GRPC_SEND_SERVER_REQUEST, getServer().getId(), null, proto); resetHeartbeatTrigger(); @@ -431,7 +460,7 @@ private void sendRequest(AppendEntriesRequest request, } private void timeoutAppendRequest(long cid, boolean heartbeat) { - final AppendEntriesRequest pending = pendingRequests.handleTimeout(cid, heartbeat); + final AppendEntriesRequest pending = pendingRequests.remove(cid, heartbeat); if (pending != null) { final int errorCount = replyState.process(Event.TIMEOUT); LOG.warn("{}: Timed out {}appendEntries, errorCount={}, request={}", @@ -512,8 +541,9 @@ private void onNextImpl(AppendEntriesRequest request, AppendEntriesReplyProto re break; case INCONSISTENCY: grpcServerMetrics.onRequestInconsistency(getFollowerId().toString()); - LOG.warn("{}: received {} reply with nextIndex {}, errorCount={}, request={}", - this, reply.getResult(), reply.getNextIndex(), errorCount, request); + BatchLogger.print(BatchLogKey.INCONSISTENCY_REPLY, getFollower().getName() + "_" + reply.getNextIndex(), + suffix -> LOG.warn("{}: received {} reply with nextIndex {}, errorCount={}, request={} {}", + this, reply.getResult(), reply.getNextIndex(), errorCount, request, suffix)); final long requestFirstIndex = request != null? request.getFirstIndex(): RaftLog.INVALID_LOG_INDEX; updateNextIndex(getNextIndexForInconsistency(requestFirstIndex, reply.getNextIndex())); break; @@ -533,7 +563,7 @@ public void onError(Throwable t) { LOG.info("{} is already stopped", GrpcLogAppender.this); return; } - BatchLogger.warn(BatchLogKey.APPEND_LOG_RESPONSE_HANDLER_ON_ERROR, AppendLogResponseHandler.this.name, + BatchLogger.print(BatchLogKey.APPEND_LOG_RESPONSE_HANDLER_ON_ERROR, AppendLogResponseHandler.this.name, suffix -> GrpcUtil.warn(LOG, () -> this + ": Failed appendEntries" + suffix, t), logMessageBatchDuration, t instanceof StatusRuntimeException); grpcServerMetrics.onRequestRetry(); // Update try counter @@ -563,7 +593,7 @@ private void updateNextIndex(long replyNextIndex) { private class InstallSnapshotResponseHandler implements StreamObserver { private final String name = getFollower().getName() + "-" + JavaUtils.getClassSimpleName(getClass()); private final Queue pending; - private final AtomicBoolean done = new AtomicBoolean(false); + private final CompletableFuture done = new CompletableFuture<>(); private final boolean isNotificationOnly; InstallSnapshotResponseHandler() { @@ -577,15 +607,35 @@ private class InstallSnapshotResponseHandler implements StreamObserver null); if (tlsConfig!= null) { SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); @@ -139,7 +141,7 @@ CallStreamObserver appendEntries( StreamObserver installSnapshot( String name, TimeDuration timeout, int limit, StreamObserver responseHandler) { return StreamObserverWithTimeout.newInstance(name, ServerStringUtils::toInstallSnapshotRequestString, - timeout, limit, i -> asyncStub.withInterceptors(i).installSnapshot(responseHandler)); + () -> timeout, limit, i -> asyncStub.withInterceptors(i).installSnapshot(responseHandler)); } // short-circuit the backoff timer and make them reconnect immediately. diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java index 766e14321a..7e17cb3cf4 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServerProtocolService.java @@ -20,16 +20,21 @@ import java.util.function.Consumer; import java.util.function.Function; import org.apache.ratis.grpc.GrpcUtil; +import org.apache.ratis.grpc.metrics.ZeroCopyMetrics; +import org.apache.ratis.grpc.util.ZeroCopyMessageMarshaller; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.protocol.RaftServerProtocol; import org.apache.ratis.server.util.ServerStringUtils; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; import org.apache.ratis.thirdparty.io.grpc.Status; -import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.apache.ratis.proto.RaftProtos.*; import org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc.RaftServerProtocolServiceImplBase; +import org.apache.ratis.util.BatchLogger; +import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.ProtoUtils; +import org.apache.ratis.util.ReferenceCountedObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,28 +45,45 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import static org.apache.ratis.grpc.GrpcUtil.addMethodWithCustomMarshaller; +import static org.apache.ratis.proto.grpc.RaftServerProtocolServiceGrpc.getAppendEntriesMethod; + class GrpcServerProtocolService extends RaftServerProtocolServiceImplBase { public static final Logger LOG = LoggerFactory.getLogger(GrpcServerProtocolService.class); + private enum BatchLogKey implements BatchLogger.Key { + COMPLETED_REQUEST, + COMPLETED_REPLY + } + static class PendingServerRequest { - private final REQUEST request; + private final AtomicReference> requestRef; private final CompletableFuture future = new CompletableFuture<>(); - PendingServerRequest(REQUEST request) { - this.request = request; + PendingServerRequest(ReferenceCountedObject requestRef) { + requestRef.retain(); + this.requestRef = new AtomicReference<>(requestRef); } REQUEST getRequest() { - return request; + return Optional.ofNullable(requestRef.get()) + .map(ReferenceCountedObject::get) + .orElse(null); } CompletableFuture getFuture() { return future; } + + void release() { + Optional.ofNullable(requestRef.getAndSet(null)) + .ifPresent(ReferenceCountedObject::release); + } } abstract class ServerRequestStreamObserver implements StreamObserver { private final RaftServer.Op op; + private final Supplier nameSupplier; private final StreamObserver responseObserver; /** For ordered {@link #onNext(Object)} requests. */ private final AtomicReference> previousOnNext = new AtomicReference<>(); @@ -72,9 +94,14 @@ abstract class ServerRequestStreamObserver implements StreamObse ServerRequestStreamObserver(RaftServer.Op op, StreamObserver responseObserver) { this.op = op; + this.nameSupplier = MemoizedSupplier.valueOf(() -> getId() + "_" + op); this.responseObserver = responseObserver; } + String getName() { + return nameSupplier.get(); + } + private String getPreviousRequestString() { return Optional.ofNullable(previousOnNext.get()) .map(PendingServerRequest::getRequest) @@ -82,24 +109,39 @@ private String getPreviousRequestString() { .orElse(null); } - abstract CompletableFuture process(REQUEST request) throws IOException; + CompletableFuture process(REQUEST request) throws IOException { + throw new UnsupportedOperationException("This method is not supported."); + } + + CompletableFuture process(ReferenceCountedObject requestRef) + throws IOException { + try { + return process(requestRef.retain()); + } finally { + requestRef.release(); + } + } + + void release(REQUEST req) { + } abstract long getCallId(REQUEST request); + boolean isHeartbeat(REQUEST request) { + return false; + } + abstract String requestToString(REQUEST request); abstract String replyToString(REPLY reply); abstract boolean replyInOrder(REQUEST request); - StatusRuntimeException wrapException(Throwable e, REQUEST request) { - return GrpcUtil.wrapException(e, getCallId(request)); - } - - private void handleError(Throwable e, REQUEST request) { - GrpcUtil.warn(LOG, () -> getId() + ": Failed " + op + " request " + requestToString(request), e); + private synchronized void handleError(Throwable e, long callId, boolean isHeartbeat) { + GrpcUtil.warn(LOG, () -> getId() + ": Failed " + op + " request cid=" + callId + ", isHeartbeat? " + + isHeartbeat, e); if (isClosed.compareAndSet(false, true)) { - responseObserver.onError(wrapException(e, request)); + responseObserver.onError(GrpcUtil.wrapException(e, callId, isHeartbeat)); } } @@ -119,24 +161,32 @@ void composeRequest(CompletableFuture current) { @Override public void onNext(REQUEST request) { + ReferenceCountedObject requestRef = ReferenceCountedObject.wrap(request, () -> {}, released -> { + if (released) { + release(request); + } + }); + if (!replyInOrder(request)) { try { - composeRequest(process(request).thenApply(this::handleReply)); + composeRequest(process(requestRef).thenApply(this::handleReply)); } catch (Exception e) { - handleError(e, request); + handleError(e, getCallId(request), isHeartbeat(request)); + release(request); } return; } - final PendingServerRequest current = new PendingServerRequest<>(request); - final PendingServerRequest previous = previousOnNext.getAndSet(current); - final CompletableFuture previousFuture = Optional.ofNullable(previous) - .map(PendingServerRequest::getFuture) + final PendingServerRequest current = new PendingServerRequest<>(requestRef); + final long callId = getCallId(current.getRequest()); + final boolean isHeartbeat = isHeartbeat(current.getRequest()); + final Optional> previous = Optional.ofNullable(previousOnNext.getAndSet(current)); + final CompletableFuture previousFuture = previous.map(PendingServerRequest::getFuture) .orElse(CompletableFuture.completedFuture(null)); try { - final CompletableFuture f = process(request).exceptionally(e -> { + final CompletableFuture f = process(requestRef).exceptionally(e -> { // Handle cases, such as RaftServer is paused - handleError(e, request); + handleError(e, callId, isHeartbeat); current.getFuture().completeExceptionally(e); return null; }).thenCombine(previousFuture, (reply, v) -> { @@ -146,21 +196,32 @@ public void onNext(REQUEST request) { }); composeRequest(f); } catch (Exception e) { - handleError(e, request); + handleError(e, callId, isHeartbeat); current.getFuture().completeExceptionally(e); + } finally { + previous.ifPresent(PendingServerRequest::release); + if (isClosed.get()) { + // Some requests may come after onCompleted or onError, ensure they're released. + releaseLast(); + } } } @Override public void onCompleted() { if (isClosed.compareAndSet(false, true)) { - LOG.info("{}: Completed {}, lastRequest: {}", getId(), op, getPreviousRequestString()); + BatchLogger.print(BatchLogKey.COMPLETED_REQUEST, getName(), + suffix -> LOG.info("{}: Completed {}, lastRequest: {} {}", + getId(), op, getPreviousRequestString(), suffix)); requestFuture.get().thenAccept(reply -> { - LOG.info("{}: Completed {}, lastReply: {}", getId(), op, reply); + BatchLogger.print(BatchLogKey.COMPLETED_REPLY, getName(), + suffix -> LOG.info("{}: Completed {}, lastReply: {} {}", getId(), op, reply, suffix)); responseObserver.onCompleted(); }); + releaseLast(); } } + @Override public void onError(Throwable t) { GrpcUtil.warn(LOG, () -> getId() + ": "+ op + " onError, lastRequest: " + getPreviousRequestString(), t); @@ -169,22 +230,54 @@ public void onError(Throwable t) { if (status != null && status.getCode() != Status.Code.CANCELLED) { responseObserver.onCompleted(); } + releaseLast(); } } + + private void releaseLast() { + Optional.ofNullable(previousOnNext.get()).ifPresent(PendingServerRequest::release); + } } private final Supplier idSupplier; private final RaftServer server; + private final boolean zeroCopyEnabled; + private final ZeroCopyMessageMarshaller zeroCopyRequestMarshaller; - GrpcServerProtocolService(Supplier idSupplier, RaftServer server) { + GrpcServerProtocolService(Supplier idSupplier, RaftServer server, boolean zeroCopyEnabled, + ZeroCopyMetrics zeroCopyMetrics) { this.idSupplier = idSupplier; this.server = server; + this.zeroCopyEnabled = zeroCopyEnabled; + this.zeroCopyRequestMarshaller = new ZeroCopyMessageMarshaller<>(AppendEntriesRequestProto.getDefaultInstance(), + zeroCopyMetrics::onZeroCopyMessage, zeroCopyMetrics::onNonZeroCopyMessage, zeroCopyMetrics::onReleasedMessage); + zeroCopyMetrics.addUnreleased("server_protocol", zeroCopyRequestMarshaller::getUnclosedCount); } RaftPeerId getId() { return idSupplier.get(); } + ServerServiceDefinition bindServiceWithZeroCopy() { + ServerServiceDefinition orig = super.bindService(); + if (!zeroCopyEnabled) { + LOG.info("{}: Zero copy is disabled.", getId()); + return orig; + } + ServerServiceDefinition.Builder builder = ServerServiceDefinition.builder(orig.getServiceDescriptor().getName()); + + // Add appendEntries with zero copy marshaller. + addMethodWithCustomMarshaller(orig, builder, getAppendEntriesMethod(), zeroCopyRequestMarshaller); + // Add remaining methods as is. + orig.getMethods().stream().filter( + x -> !x.getMethodDescriptor().getFullMethodName().equals(getAppendEntriesMethod().getFullMethodName()) + ).forEach( + builder::addMethod + ); + + return builder.build(); + } + @Override public void requestVote(RequestVoteRequestProto request, StreamObserver responseObserver) { @@ -225,8 +318,14 @@ public StreamObserver appendEntries( return new ServerRequestStreamObserver( RaftServerProtocol.Op.APPEND_ENTRIES, responseObserver) { @Override - CompletableFuture process(AppendEntriesRequestProto request) throws IOException { - return server.appendEntriesAsync(request); + CompletableFuture process(ReferenceCountedObject requestRef) + throws IOException { + return server.appendEntriesAsync(requestRef); + } + + @Override + void release(AppendEntriesRequestProto req) { + zeroCopyRequestMarshaller.release(req); } @Override @@ -234,9 +333,14 @@ long getCallId(AppendEntriesRequestProto request) { return request.getServerRequest().getCallId(); } + @Override + boolean isHeartbeat(AppendEntriesRequestProto request) { + return request.getEntriesCount() == 0; + } + @Override String requestToString(AppendEntriesRequestProto request) { - return ServerStringUtils.toAppendEntriesRequestString(request); + return ServerStringUtils.toAppendEntriesRequestString(request, null); } @Override @@ -248,11 +352,6 @@ String replyToString(AppendEntriesReplyProto reply) { boolean replyInOrder(AppendEntriesRequestProto request) { return request.getEntriesCount() != 0; } - - @Override - StatusRuntimeException wrapException(Throwable e, AppendEntriesRequestProto request) { - return GrpcUtil.wrapException(e, getCallId(request), request.getEntriesCount() == 0); - } }; } diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcService.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcService.java deleted file mode 100644 index 097900a0fb..0000000000 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcService.java +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis.grpc.server; - -import org.apache.ratis.conf.RaftProperties; -import org.apache.ratis.grpc.GrpcConfigKeys; -import org.apache.ratis.grpc.GrpcTlsConfig; -import org.apache.ratis.grpc.GrpcUtil; -import org.apache.ratis.grpc.metrics.intercept.server.MetricServerInterceptor; -import org.apache.ratis.protocol.RaftGroupId; -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.rpc.SupportedRpcType; -import org.apache.ratis.server.RaftServer; -import org.apache.ratis.server.RaftServerConfigKeys; -import org.apache.ratis.server.RaftServerRpcWithProxy; -import org.apache.ratis.server.protocol.RaftServerAsynchronousProtocol; -import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors; -import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; -import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; -import org.apache.ratis.thirdparty.io.grpc.Server; -import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; -import org.apache.ratis.thirdparty.io.netty.channel.ChannelOption; -import org.apache.ratis.thirdparty.io.netty.handler.ssl.ClientAuth; -import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; - -import org.apache.ratis.proto.RaftProtos.*; -import org.apache.ratis.util.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.function.Supplier; - -import static org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider.OPENSSL; - -/** A grpc implementation of {@link org.apache.ratis.server.RaftServerRpc}. */ -public final class GrpcService extends RaftServerRpcWithProxy> { - static final Logger LOG = LoggerFactory.getLogger(GrpcService.class); - public static final String GRPC_SEND_SERVER_REQUEST = - JavaUtils.getClassSimpleName(GrpcService.class) + ".sendRequest"; - - class AsyncService implements RaftServerAsynchronousProtocol { - - @Override - public CompletableFuture appendEntriesAsync(AppendEntriesRequestProto request) - throws IOException { - throw new UnsupportedOperationException("This method is not supported"); - } - - @Override - public CompletableFuture readIndexAsync(ReadIndexRequestProto request) throws IOException { - CodeInjectionForTesting.execute(GRPC_SEND_SERVER_REQUEST, getId(), null, request); - - final CompletableFuture f = new CompletableFuture<>(); - final StreamObserver s = new StreamObserver() { - @Override - public void onNext(ReadIndexReplyProto reply) { - f.complete(reply); - } - - @Override - public void onError(Throwable throwable) { - f.completeExceptionally(throwable); - } - - @Override - public void onCompleted() { - } - }; - - final RaftPeerId target = RaftPeerId.valueOf(request.getServerRequest().getReplyId()); - getProxies().getProxy(target).readIndex(request, s); - return f; - } - } - - public static final class Builder { - private RaftServer server; - private GrpcTlsConfig tlsConfig; - private GrpcTlsConfig adminTlsConfig; - private GrpcTlsConfig clientTlsConfig; - private GrpcTlsConfig serverTlsConfig; - - private Builder() {} - - public Builder setServer(RaftServer raftServer) { - this.server = raftServer; - return this; - } - - public GrpcService build() { - return new GrpcService(server, adminTlsConfig, clientTlsConfig, serverTlsConfig); - } - - public Builder setTlsConfig(GrpcTlsConfig tlsConfig) { - this.tlsConfig = tlsConfig; - return this; - } - - public Builder setAdminTlsConfig(GrpcTlsConfig config) { - this.adminTlsConfig = config; - return this; - } - - public Builder setClientTlsConfig(GrpcTlsConfig config) { - this.clientTlsConfig = config; - return this; - } - - public Builder setServerTlsConfig(GrpcTlsConfig config) { - this.serverTlsConfig = config; - return this; - } - - public GrpcTlsConfig getTlsConfig() { - return tlsConfig; - } - } - - public static Builder newBuilder() { - return new Builder(); - } - - private final Map servers = new HashMap<>(); - private final Supplier addressSupplier; - private final Supplier clientServerAddressSupplier; - private final Supplier adminServerAddressSupplier; - - private final AsyncService asyncService = new AsyncService(); - - private final ExecutorService executor; - private final GrpcClientProtocolService clientProtocolService; - - private final MetricServerInterceptor serverInterceptor; - - public MetricServerInterceptor getServerInterceptor() { - return serverInterceptor; - } - - private GrpcService(RaftServer server, - GrpcTlsConfig adminTlsConfig, GrpcTlsConfig clientTlsConfig, GrpcTlsConfig serverTlsConfig) { - this(server, server::getId, - GrpcConfigKeys.Admin.host(server.getProperties()), - GrpcConfigKeys.Admin.port(server.getProperties()), - adminTlsConfig, - GrpcConfigKeys.Client.host(server.getProperties()), - GrpcConfigKeys.Client.port(server.getProperties()), - clientTlsConfig, - GrpcConfigKeys.Server.host(server.getProperties()), - GrpcConfigKeys.Server.port(server.getProperties()), - serverTlsConfig, - GrpcConfigKeys.messageSizeMax(server.getProperties(), LOG::info), - RaftServerConfigKeys.Log.Appender.bufferByteLimit(server.getProperties()), - GrpcConfigKeys.flowControlWindow(server.getProperties(), LOG::info), - RaftServerConfigKeys.Rpc.requestTimeout(server.getProperties()), - GrpcConfigKeys.Server.heartbeatChannel(server.getProperties())); - } - - @SuppressWarnings("checkstyle:ParameterNumber") // private constructor - private GrpcService(RaftServer raftServer, Supplier idSupplier, - String adminHost, int adminPort, GrpcTlsConfig adminTlsConfig, - String clientHost, int clientPort, GrpcTlsConfig clientTlsConfig, - String serverHost, int serverPort, GrpcTlsConfig serverTlsConfig, - SizeInBytes grpcMessageSizeMax, SizeInBytes appenderBufferSize, - SizeInBytes flowControlWindow,TimeDuration requestTimeoutDuration, - boolean useSeparateHBChannel) { - super(idSupplier, id -> new PeerProxyMap<>(id.toString(), - p -> new GrpcServerProtocolClient(p, flowControlWindow.getSizeInt(), - requestTimeoutDuration, serverTlsConfig, useSeparateHBChannel))); - if (appenderBufferSize.getSize() > grpcMessageSizeMax.getSize()) { - throw new IllegalArgumentException("Illegal configuration: " - + RaftServerConfigKeys.Log.Appender.BUFFER_BYTE_LIMIT_KEY + " = " + appenderBufferSize - + " > " + GrpcConfigKeys.MESSAGE_SIZE_MAX_KEY + " = " + grpcMessageSizeMax); - } - - final RaftProperties properties = raftServer.getProperties(); - this.executor = ConcurrentUtils.newThreadPoolWithMax( - GrpcConfigKeys.Server.asyncRequestThreadPoolCached(properties), - GrpcConfigKeys.Server.asyncRequestThreadPoolSize(properties), - getId() + "-request-"); - this.clientProtocolService = new GrpcClientProtocolService(idSupplier, raftServer, executor); - - this.serverInterceptor = new MetricServerInterceptor( - idSupplier, - JavaUtils.getClassSimpleName(getClass()) + "_" + serverPort - ); - - final boolean separateAdminServer = adminPort != serverPort && adminPort > 0; - final boolean separateClientServer = clientPort != serverPort && clientPort > 0; - - final NettyServerBuilder serverBuilder = - startBuildingNettyServer(serverHost, serverPort, serverTlsConfig, grpcMessageSizeMax, flowControlWindow); - serverBuilder.addService(ServerInterceptors.intercept( - new GrpcServerProtocolService(idSupplier, raftServer), serverInterceptor)); - if (!separateAdminServer) { - addAdminService(raftServer, serverBuilder); - } - if (!separateClientServer) { - addClientService(serverBuilder); - } - - final Server server = serverBuilder.build(); - servers.put(GrpcServerProtocolService.class.getSimpleName(), server); - addressSupplier = newAddressSupplier(serverPort, server); - - if (separateAdminServer) { - final NettyServerBuilder builder = - startBuildingNettyServer(adminHost, adminPort, adminTlsConfig, grpcMessageSizeMax, flowControlWindow); - addAdminService(raftServer, builder); - final Server adminServer = builder.build(); - servers.put(GrpcAdminProtocolService.class.getName(), adminServer); - adminServerAddressSupplier = newAddressSupplier(adminPort, adminServer); - } else { - adminServerAddressSupplier = addressSupplier; - } - - if (separateClientServer) { - final NettyServerBuilder builder = - startBuildingNettyServer(clientHost, clientPort, clientTlsConfig, grpcMessageSizeMax, flowControlWindow); - addClientService(builder); - final Server clientServer = builder.build(); - servers.put(GrpcClientProtocolService.class.getName(), clientServer); - clientServerAddressSupplier = newAddressSupplier(clientPort, clientServer); - } else { - clientServerAddressSupplier = addressSupplier; - } - } - - private MemoizedSupplier newAddressSupplier(int port, Server server) { - return JavaUtils.memoize(() -> new InetSocketAddress(port != 0 ? port : server.getPort())); - } - - private void addClientService(NettyServerBuilder builder) { - builder.addService(ServerInterceptors.intercept(clientProtocolService, serverInterceptor)); - } - - private void addAdminService(RaftServer raftServer, NettyServerBuilder nettyServerBuilder) { - nettyServerBuilder.addService(ServerInterceptors.intercept( - new GrpcAdminProtocolService(raftServer), - serverInterceptor)); - } - - private static NettyServerBuilder startBuildingNettyServer(String hostname, int port, GrpcTlsConfig tlsConfig, - SizeInBytes grpcMessageSizeMax, SizeInBytes flowControlWindow) { - InetSocketAddress address = hostname == null || hostname.isEmpty() ? - new InetSocketAddress(port) : new InetSocketAddress(hostname, port); - NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forAddress(address) - .withChildOption(ChannelOption.SO_REUSEADDR, true) - .maxInboundMessageSize(grpcMessageSizeMax.getSizeInt()) - .flowControlWindow(flowControlWindow.getSizeInt()); - - if (tlsConfig != null) { - SslContextBuilder sslContextBuilder = GrpcUtil.initSslContextBuilderForServer(tlsConfig.getKeyManager()); - if (tlsConfig.getMtlsEnabled()) { - sslContextBuilder.clientAuth(ClientAuth.REQUIRE); - GrpcUtil.setTrustManager(sslContextBuilder, tlsConfig.getTrustManager()); - } - sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder, OPENSSL); - try { - nettyServerBuilder.sslContext(sslContextBuilder.build()); - } catch (Exception ex) { - throw new IllegalArgumentException("Failed to build SslContext, tlsConfig=" + tlsConfig, ex); - } - } - return nettyServerBuilder; - } - - @Override - public SupportedRpcType getRpcType() { - return SupportedRpcType.GRPC; - } - - @Override - public void startImpl() { - for (Server server : servers.values()) { - try { - server.start(); - } catch (IOException e) { - ExitUtils.terminate(1, "Failed to start Grpc server", e, LOG); - } - LOG.info("{}: {} started, listening on {}", - getId(), JavaUtils.getClassSimpleName(getClass()), server.getPort()); - } - } - - @Override - public void closeImpl() throws IOException { - for (Map.Entry server : servers.entrySet()) { - final String name = getId() + ": shutdown server " + server.getKey(); - LOG.info("{} now", name); - final Server s = server.getValue().shutdownNow(); - super.closeImpl(); - try { - s.awaitTermination(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw IOUtils.toInterruptedIOException(name + " failed", e); - } - LOG.info("{} successfully", name); - } - - serverInterceptor.close(); - ConcurrentUtils.shutdownAndWait(executor); - } - - @Override - public void notifyNotLeader(RaftGroupId groupId) { - clientProtocolService.closeAllOrderedRequestStreamObservers(groupId); - } - - @Override - public InetSocketAddress getInetSocketAddress() { - return addressSupplier.get(); - } - - @Override - public InetSocketAddress getClientServerAddress() { - return clientServerAddressSupplier.get(); - } - - @Override - public InetSocketAddress getAdminServerAddress() { - return adminServerAddressSupplier.get(); - } - - @Override - public RaftServerAsynchronousProtocol async() { - return asyncService; - } - - @Override - public AppendEntriesReplyProto appendEntries(AppendEntriesRequestProto request) { - throw new UnsupportedOperationException( - "Blocking " + JavaUtils.getCurrentStackTraceElement().getMethodName() + " call is not supported"); - } - - @Override - public InstallSnapshotReplyProto installSnapshot(InstallSnapshotRequestProto request) { - throw new UnsupportedOperationException( - "Blocking " + JavaUtils.getCurrentStackTraceElement().getMethodName() + " call is not supported"); - } - - @Override - public RequestVoteReplyProto requestVote(RequestVoteRequestProto request) - throws IOException { - CodeInjectionForTesting.execute(GRPC_SEND_SERVER_REQUEST, getId(), - null, request); - - final RaftPeerId target = RaftPeerId.valueOf(request.getServerRequest().getReplyId()); - return getProxies().getProxy(target).requestVote(request); - } - - @Override - public StartLeaderElectionReplyProto startLeaderElection(StartLeaderElectionRequestProto request) throws IOException { - CodeInjectionForTesting.execute(GRPC_SEND_SERVER_REQUEST, getId(), null, request); - - final RaftPeerId target = RaftPeerId.valueOf(request.getServerRequest().getReplyId()); - return getProxies().getProxy(target).startLeaderElection(request); - } - -} diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServices.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServices.java new file mode 100644 index 0000000000..663fd6d743 --- /dev/null +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServices.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.grpc.server; + +import org.apache.ratis.server.RaftServerRpc; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; + +import java.util.EnumSet; + +/** The gRPC services extending {@link RaftServerRpc}. */ +public interface GrpcServices extends RaftServerRpc { + /** The type of the services. */ + enum Type {ADMIN, CLIENT, SERVER} + + /** + * To customize the services. + * For example, add a custom service. + */ + interface Customizer { + /** The default NOOP {@link Customizer}. */ + class Default implements Customizer { + private static final Default INSTANCE = new Default(); + + @Override + public NettyServerBuilder customize(NettyServerBuilder builder, EnumSet types) { + return builder; + } + } + + static Customizer getDefaultInstance() { + return Default.INSTANCE; + } + + /** + * Customize the given builder for the given types. + * + * @return the customized builder. + */ + NettyServerBuilder customize(NettyServerBuilder builder, EnumSet types); + } +} diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServicesImpl.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServicesImpl.java new file mode 100644 index 0000000000..d6f6a0c866 --- /dev/null +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/server/GrpcServicesImpl.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.grpc.server; + +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.grpc.GrpcConfigKeys; +import org.apache.ratis.grpc.GrpcTlsConfig; +import org.apache.ratis.grpc.GrpcUtil; +import org.apache.ratis.grpc.metrics.MessageMetrics; +import org.apache.ratis.grpc.metrics.ZeroCopyMetrics; +import org.apache.ratis.grpc.metrics.intercept.server.MetricServerInterceptor; +import org.apache.ratis.protocol.AdminAsynchronousProtocol; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.rpc.SupportedRpcType; +import org.apache.ratis.server.RaftServer; +import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.RaftServerRpcWithProxy; +import org.apache.ratis.server.protocol.RaftServerAsynchronousProtocol; +import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor; +import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors; +import org.apache.ratis.thirdparty.io.grpc.ServerServiceDefinition; +import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.grpc.Server; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; +import org.apache.ratis.thirdparty.io.netty.channel.ChannelOption; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.ClientAuth; +import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; + +import org.apache.ratis.proto.RaftProtos.*; +import org.apache.ratis.util.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; + +import static org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider.OPENSSL; + +/** A grpc implementation of {@link org.apache.ratis.server.RaftServerRpc}. */ +public final class GrpcServicesImpl + extends RaftServerRpcWithProxy> + implements GrpcServices { + static final Logger LOG = LoggerFactory.getLogger(GrpcServicesImpl.class); + public static final String GRPC_SEND_SERVER_REQUEST = + JavaUtils.getClassSimpleName(GrpcServicesImpl.class) + ".sendRequest"; + + class AsyncService implements RaftServerAsynchronousProtocol { + + @Override + public CompletableFuture appendEntriesAsync(AppendEntriesRequestProto request) { + throw new UnsupportedOperationException("This method is not supported"); + } + + @Override + public CompletableFuture readIndexAsync(ReadIndexRequestProto request) throws IOException { + CodeInjectionForTesting.execute(GRPC_SEND_SERVER_REQUEST, getId(), null, request); + + final CompletableFuture f = new CompletableFuture<>(); + final StreamObserver s = new StreamObserver() { + @Override + public void onNext(ReadIndexReplyProto reply) { + f.complete(reply); + } + + @Override + public void onError(Throwable throwable) { + f.completeExceptionally(throwable); + } + + @Override + public void onCompleted() { + } + }; + + final RaftPeerId target = RaftPeerId.valueOf(request.getServerRequest().getReplyId()); + getProxies().getProxy(target).readIndex(request, s); + return f; + } + } + + public static final class Builder { + private RaftServer server; + private Customizer customizer; + + private String adminHost; + private int adminPort; + private GrpcTlsConfig adminTlsConfig; + private String clientHost; + private int clientPort; + private GrpcTlsConfig clientTlsConfig; + private String serverHost; + private int serverPort; + private GrpcTlsConfig serverTlsConfig; + + private SizeInBytes messageSizeMax; + private SizeInBytes flowControlWindow; + private TimeDuration requestTimeoutDuration; + private boolean separateHeartbeatChannel; + private boolean zeroCopyEnabled; + + private Builder() {} + + public Builder setServer(RaftServer raftServer) { + this.server = raftServer; + + final RaftProperties properties = server.getProperties(); + this.adminHost = GrpcConfigKeys.Admin.host(properties); + this.adminPort = GrpcConfigKeys.Admin.port(properties); + this.clientHost = GrpcConfigKeys.Client.host(properties); + this.clientPort = GrpcConfigKeys.Client.port(properties); + this.serverHost = GrpcConfigKeys.Server.host(properties); + this.serverPort = GrpcConfigKeys.Server.port(properties); + this.messageSizeMax = GrpcConfigKeys.messageSizeMax(properties, LOG::info); + this.flowControlWindow = GrpcConfigKeys.flowControlWindow(properties, LOG::info); + this.requestTimeoutDuration = RaftServerConfigKeys.Rpc.requestTimeout(properties); + this.separateHeartbeatChannel = GrpcConfigKeys.Server.heartbeatChannel(properties); + this.zeroCopyEnabled = GrpcConfigKeys.Server.zeroCopyEnabled(properties); + + final SizeInBytes appenderBufferSize = RaftServerConfigKeys.Log.Appender.bufferByteLimit(properties); + final SizeInBytes gap = SizeInBytes.ONE_MB; + final long diff = messageSizeMax.getSize() - appenderBufferSize.getSize(); + if (diff < gap.getSize()) { + throw new IllegalArgumentException("Illegal configuration: " + + GrpcConfigKeys.MESSAGE_SIZE_MAX_KEY + "(= " + messageSizeMax + + ") must be " + gap + " larger than " + + RaftServerConfigKeys.Log.Appender.BUFFER_BYTE_LIMIT_KEY + "(= " + appenderBufferSize + ")."); + } + + return this; + } + + public Builder setCustomizer(Customizer customizer) { + this.customizer = customizer != null? customizer : Customizer.getDefaultInstance(); + return this; + } + + private GrpcServerProtocolClient newGrpcServerProtocolClient(RaftPeer target) { + return new GrpcServerProtocolClient(target, flowControlWindow.getSizeInt(), + requestTimeoutDuration, serverTlsConfig, separateHeartbeatChannel); + } + + private ExecutorService newExecutor() { + final RaftProperties properties = server.getProperties(); + return ConcurrentUtils.newThreadPoolWithMax( + GrpcConfigKeys.Server.asyncRequestThreadPoolCached(properties), + GrpcConfigKeys.Server.asyncRequestThreadPoolSize(properties), + server.getId() + "-request-"); + } + + private GrpcClientProtocolService newGrpcClientProtocolService( + ExecutorService executor, ZeroCopyMetrics zeroCopyMetrics) { + return new GrpcClientProtocolService(server::getId, server, executor, zeroCopyEnabled, zeroCopyMetrics); + } + + private GrpcServerProtocolService newGrpcServerProtocolService(ZeroCopyMetrics zeroCopyMetrics) { + return new GrpcServerProtocolService(server::getId, server, zeroCopyEnabled, zeroCopyMetrics); + } + + private MetricServerInterceptor newMetricServerInterceptor() { + return new MetricServerInterceptor(server::getId, + JavaUtils.getClassSimpleName(getClass()) + "_" + serverPort); + } + + Server buildServer(NettyServerBuilder builder, EnumSet types) { + return customizer.customize(builder, types).build(); + } + + private NettyServerBuilder newNettyServerBuilderForServer() { + return newNettyServerBuilder(serverHost, serverPort, serverTlsConfig); + } + + private NettyServerBuilder newNettyServerBuilderForAdmin() { + return newNettyServerBuilder(adminHost, adminPort, adminTlsConfig); + } + + private NettyServerBuilder newNettyServerBuilderForClient() { + return newNettyServerBuilder(clientHost, clientPort, clientTlsConfig); + } + + private NettyServerBuilder newNettyServerBuilder(String hostname, int port, GrpcTlsConfig tlsConfig) { + final InetSocketAddress address = hostname == null || hostname.isEmpty() ? + new InetSocketAddress(port) : new InetSocketAddress(hostname, port); + final NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forAddress(address) + .withChildOption(ChannelOption.SO_REUSEADDR, true) + .maxInboundMessageSize(messageSizeMax.getSizeInt()) + .flowControlWindow(flowControlWindow.getSizeInt()); + + if (tlsConfig != null) { + LOG.info("Setting TLS for {}", address); + SslContextBuilder sslContextBuilder = GrpcUtil.initSslContextBuilderForServer(tlsConfig.getKeyManager()); + if (tlsConfig.getMtlsEnabled()) { + sslContextBuilder.clientAuth(ClientAuth.REQUIRE); + GrpcUtil.setTrustManager(sslContextBuilder, tlsConfig.getTrustManager()); + } + sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder, OPENSSL); + try { + nettyServerBuilder.sslContext(sslContextBuilder.build()); + } catch (Exception ex) { + throw new IllegalArgumentException("Failed to build SslContext, tlsConfig=" + tlsConfig, ex); + } + } + return nettyServerBuilder; + } + + private boolean separateAdminServer() { + return adminPort > 0 && adminPort != serverPort; + } + + private boolean separateClientServer() { + return clientPort > 0 && clientPort != serverPort; + } + + Server newServer(GrpcClientProtocolService client, ZeroCopyMetrics zeroCopyMetrics, ServerInterceptor interceptor) { + final EnumSet types = EnumSet.of(GrpcServices.Type.SERVER); + final NettyServerBuilder serverBuilder = newNettyServerBuilderForServer(); + final ServerServiceDefinition service = newGrpcServerProtocolService(zeroCopyMetrics).bindServiceWithZeroCopy(); + serverBuilder.addService(ServerInterceptors.intercept(service, interceptor)); + + if (!separateAdminServer()) { + types.add(GrpcServices.Type.ADMIN); + addAdminService(serverBuilder, server, interceptor); + } + if (!separateClientServer()) { + types.add(GrpcServices.Type.CLIENT); + addClientService(serverBuilder, client, interceptor); + } + return buildServer(serverBuilder, types); + } + + public GrpcServicesImpl build() { + return new GrpcServicesImpl(this); + } + + public Builder setAdminTlsConfig(GrpcTlsConfig config) { + this.adminTlsConfig = config; + return this; + } + + public Builder setClientTlsConfig(GrpcTlsConfig config) { + this.clientTlsConfig = config; + return this; + } + + public Builder setServerTlsConfig(GrpcTlsConfig config) { + this.serverTlsConfig = config; + return this; + } + } + + public static Builder newBuilder() { + return new Builder(); + } + + private final Map servers = new HashMap<>(); + private final Supplier addressSupplier; + private final Supplier clientServerAddressSupplier; + private final Supplier adminServerAddressSupplier; + + private final AsyncService asyncService = new AsyncService(); + + private final ExecutorService executor; + private final GrpcClientProtocolService clientProtocolService; + + private final MetricServerInterceptor serverInterceptor; + private final ZeroCopyMetrics zeroCopyMetrics = new ZeroCopyMetrics(); + + private GrpcServicesImpl(Builder b) { + super(b.server::getId, id -> new PeerProxyMap<>(id.toString(), b::newGrpcServerProtocolClient)); + + this.executor = b.newExecutor(); + this.clientProtocolService = b.newGrpcClientProtocolService(executor, zeroCopyMetrics); + this.serverInterceptor = b.newMetricServerInterceptor(); + final Server server = b.newServer(clientProtocolService, zeroCopyMetrics, serverInterceptor); + + servers.put(GrpcServerProtocolService.class.getSimpleName(), server); + addressSupplier = newAddressSupplier(b.serverPort, server); + + if (b.separateAdminServer()) { + final NettyServerBuilder builder = b.newNettyServerBuilderForAdmin(); + addAdminService(builder, b.server, serverInterceptor); + final Server adminServer = b.buildServer(builder, EnumSet.of(GrpcServices.Type.ADMIN)); + servers.put(GrpcAdminProtocolService.class.getName(), adminServer); + adminServerAddressSupplier = newAddressSupplier(b.adminPort, adminServer); + } else { + adminServerAddressSupplier = addressSupplier; + } + + if (b.separateClientServer()) { + final NettyServerBuilder builder = b.newNettyServerBuilderForClient(); + addClientService(builder, clientProtocolService, serverInterceptor); + final Server clientServer = b.buildServer(builder, EnumSet.of(GrpcServices.Type.CLIENT)); + servers.put(GrpcClientProtocolService.class.getName(), clientServer); + clientServerAddressSupplier = newAddressSupplier(b.clientPort, clientServer); + } else { + clientServerAddressSupplier = addressSupplier; + } + } + + static MemoizedSupplier newAddressSupplier(int port, Server server) { + return JavaUtils.memoize(() -> new InetSocketAddress(port != 0 ? port : server.getPort())); + } + + static void addClientService(NettyServerBuilder builder, GrpcClientProtocolService client, + ServerInterceptor interceptor) { + final ServerServiceDefinition service = client.bindServiceWithZeroCopy(); + builder.addService(ServerInterceptors.intercept(service, interceptor)); + } + + static void addAdminService(NettyServerBuilder builder, AdminAsynchronousProtocol admin, + ServerInterceptor interceptor) { + final GrpcAdminProtocolService service = new GrpcAdminProtocolService(admin); + builder.addService(ServerInterceptors.intercept(service, interceptor)); + } + + @Override + public SupportedRpcType getRpcType() { + return SupportedRpcType.GRPC; + } + + @Override + public void startImpl() { + for (Server server : servers.values()) { + try { + server.start(); + } catch (IOException e) { + ExitUtils.terminate(1, "Failed to start Grpc server", e, LOG); + } + LOG.info("{}: {} started, listening on {}", + getId(), JavaUtils.getClassSimpleName(getClass()), server.getPort()); + } + } + + @Override + public void closeImpl() throws IOException { + for (Map.Entry server : servers.entrySet()) { + final String name = getId() + ": shutdown server " + server.getKey(); + LOG.info("{} now", name); + final Server s = server.getValue().shutdownNow(); + super.closeImpl(); + try { + s.awaitTermination(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw IOUtils.toInterruptedIOException(name + " failed", e); + } + LOG.info("{} successfully", name); + } + + serverInterceptor.close(); + ConcurrentUtils.shutdownAndWait(executor); + zeroCopyMetrics.unregister(); + } + + @Override + public void notifyNotLeader(RaftGroupId groupId) { + clientProtocolService.closeAllOrderedRequestStreamObservers(groupId); + } + + @Override + public InetSocketAddress getInetSocketAddress() { + return addressSupplier.get(); + } + + @Override + public InetSocketAddress getClientServerAddress() { + return clientServerAddressSupplier.get(); + } + + @Override + public InetSocketAddress getAdminServerAddress() { + return adminServerAddressSupplier.get(); + } + + @Override + public RaftServerAsynchronousProtocol async() { + return asyncService; + } + + @Override + public AppendEntriesReplyProto appendEntries(AppendEntriesRequestProto request) { + throw new UnsupportedOperationException( + "Blocking " + JavaUtils.getCurrentStackTraceElement().getMethodName() + " call is not supported"); + } + + @Override + public InstallSnapshotReplyProto installSnapshot(InstallSnapshotRequestProto request) { + throw new UnsupportedOperationException( + "Blocking " + JavaUtils.getCurrentStackTraceElement().getMethodName() + " call is not supported"); + } + + @Override + public RequestVoteReplyProto requestVote(RequestVoteRequestProto request) + throws IOException { + CodeInjectionForTesting.execute(GRPC_SEND_SERVER_REQUEST, getId(), + null, request); + + final RaftPeerId target = RaftPeerId.valueOf(request.getServerRequest().getReplyId()); + return getProxies().getProxy(target).requestVote(request); + } + + @Override + public StartLeaderElectionReplyProto startLeaderElection(StartLeaderElectionRequestProto request) throws IOException { + CodeInjectionForTesting.execute(GRPC_SEND_SERVER_REQUEST, getId(), null, request); + + final RaftPeerId target = RaftPeerId.valueOf(request.getServerRequest().getReplyId()); + return getProxies().getProxy(target).startLeaderElection(request); + } + + @VisibleForTesting + MessageMetrics getMessageMetrics() { + return serverInterceptor.getMetrics(); + } + + @VisibleForTesting + public ZeroCopyMetrics getZeroCopyMetrics() { + return zeroCopyMetrics; + } +} diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/StreamObserverWithTimeout.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/StreamObserverWithTimeout.java index ff89e7d4b7..3cc754e565 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/StreamObserverWithTimeout.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/StreamObserverWithTimeout.java @@ -32,13 +32,14 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.IntSupplier; +import java.util.function.Supplier; public final class StreamObserverWithTimeout implements StreamObserver { public static final Logger LOG = LoggerFactory.getLogger(StreamObserverWithTimeout.class); public static StreamObserverWithTimeout newInstance( String name, Function request2String, - TimeDuration timeout, int outstandingLimit, + Supplier timeout, int outstandingLimit, Function> newStreamObserver) { final AtomicInteger responseCount = new AtomicInteger(); final ResourceSemaphore semaphore = outstandingLimit > 0? new ResourceSemaphore(outstandingLimit): null; @@ -55,7 +56,7 @@ public static StreamObserverWithTimeout newInstance( private final String name; private final Function requestToStringFunction; - private final TimeDuration timeout; + private final Supplier timeoutSupplier; private final StreamObserver observer; private final TimeoutExecutor scheduler = TimeoutExecutor.getInstance(); @@ -65,17 +66,18 @@ public static StreamObserverWithTimeout newInstance( private final ResourceSemaphore semaphore; private StreamObserverWithTimeout(String name, Function requestToStringFunction, - TimeDuration timeout, IntSupplier responseCount, ResourceSemaphore semaphore, StreamObserver observer) { + Supplier timeoutSupplier, IntSupplier responseCount, ResourceSemaphore semaphore, + StreamObserver observer) { this.name = JavaUtils.getClassSimpleName(getClass()) + "-" + name; this.requestToStringFunction = requestToStringFunction; - this.timeout = timeout; + this.timeoutSupplier = timeoutSupplier; this.responseCount = responseCount; this.semaphore = semaphore; this.observer = observer; } - private void acquire(StringSupplier request) { + private void acquire(StringSupplier request, TimeDuration timeout) { if (semaphore == null) { return; } @@ -96,14 +98,16 @@ private void acquire(StringSupplier request) { @Override public void onNext(T request) { final StringSupplier requestString = StringSupplier.get(() -> requestToStringFunction.apply(request)); - acquire(requestString); + final TimeDuration timeout = timeoutSupplier.get(); + acquire(requestString, timeout); observer.onNext(request); final int id = requestCount.incrementAndGet(); - scheduler.onTimeout(timeout, () -> handleTimeout(id, requestString), + LOG.debug("{}: send {} with timeout={}: {}", name, id, timeout, requestString); + scheduler.onTimeout(timeout, () -> handleTimeout(id, timeout, requestString), LOG, () -> name + ": Timeout check failed for request: " + requestString); } - private void handleTimeout(int id, StringSupplier request) { + private void handleTimeout(int id, TimeDuration timeout, StringSupplier request) { if (id > responseCount.getAsInt()) { onError(new TimeoutIOException(name + ": Timed out " + timeout + " for sending request " + request)); } diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/ZeroCopyMessageMarshaller.java b/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/ZeroCopyMessageMarshaller.java index bb8183a247..3cdbc07c71 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/ZeroCopyMessageMarshaller.java +++ b/ratis-grpc/src/main/java/org/apache/ratis/grpc/util/ZeroCopyMessageMarshaller.java @@ -62,12 +62,14 @@ public class ZeroCopyMessageMarshaller implements Prototy private final Consumer zeroCopyCount; private final Consumer nonZeroCopyCount; + private final Consumer releasedCount; public ZeroCopyMessageMarshaller(T defaultInstance) { - this(defaultInstance, m -> {}, m -> {}); + this(defaultInstance, m -> {}, m -> {}, m -> {}); } - public ZeroCopyMessageMarshaller(T defaultInstance, Consumer zeroCopyCount, Consumer nonZeroCopyCount) { + public ZeroCopyMessageMarshaller(T defaultInstance, Consumer zeroCopyCount, Consumer nonZeroCopyCount, + Consumer releasedCount) { this.name = JavaUtils.getClassSimpleName(defaultInstance.getClass()) + "-Marshaller"; @SuppressWarnings("unchecked") final Parser p = (Parser) defaultInstance.getParserForType(); @@ -76,6 +78,7 @@ public ZeroCopyMessageMarshaller(T defaultInstance, Consumer zeroCopyCount, C this.zeroCopyCount = zeroCopyCount; this.nonZeroCopyCount = nonZeroCopyCount; + this.releasedCount = releasedCount; } @Override @@ -124,6 +127,7 @@ public void release(T message) { } try { stream.close(); + releasedCount.accept(message); } catch (IOException e) { LOG.error(name + ": Failed to close stream.", e); } @@ -222,4 +226,8 @@ private T parseFrom(CodedInputStream stream) throws InvalidProtocolBufferExcepti public InputStream popStream(T message) { return unclosedStreams.remove(message); } + + public int getUnclosedCount() { + return unclosedStreams.size(); + } } diff --git a/ratis-grpc/src/test/java/org/apache/ratis/grpc/MiniRaftClusterWithGrpc.java b/ratis-grpc/src/test/java/org/apache/ratis/grpc/MiniRaftClusterWithGrpc.java index 18c65c5be4..bd1c72b241 100644 --- a/ratis-grpc/src/test/java/org/apache/ratis/grpc/MiniRaftClusterWithGrpc.java +++ b/ratis-grpc/src/test/java/org/apache/ratis/grpc/MiniRaftClusterWithGrpc.java @@ -21,14 +21,19 @@ import org.apache.ratis.RaftTestUtil; import org.apache.ratis.conf.Parameters; import org.apache.ratis.conf.RaftProperties; -import org.apache.ratis.grpc.server.GrpcService; +import org.apache.ratis.grpc.metrics.ZeroCopyMetrics; +import org.apache.ratis.grpc.server.GrpcServicesImpl; import org.apache.ratis.protocol.RaftGroup; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.rpc.SupportedRpcType; +import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.impl.DelayLocalExecutionInjection; import org.apache.ratis.server.impl.MiniRaftCluster; +import org.apache.ratis.server.impl.RaftServerTestUtil; import org.apache.ratis.util.NetUtils; +import org.apache.ratis.util.ReferenceCountedLeakDetector; +import org.junit.jupiter.api.Assertions; import java.util.Optional; @@ -45,6 +50,11 @@ public MiniRaftClusterWithGrpc newCluster(String[] ids, String[] listenerIds, Ra } }; + static { + // TODO move it to MiniRaftCluster for detecting non-gRPC cases + ReferenceCountedLeakDetector.enable(false); + } + public interface FactoryGet extends Factory.Get { @Override default Factory getFactory() { @@ -52,10 +62,14 @@ default Factory getFactory() { } } - public static final DelayLocalExecutionInjection sendServerRequestInjection = - new DelayLocalExecutionInjection(GrpcService.GRPC_SEND_SERVER_REQUEST); + public static final DelayLocalExecutionInjection SEND_SERVER_REQUEST_INJECTION = + new DelayLocalExecutionInjection(GrpcServicesImpl.GRPC_SEND_SERVER_REQUEST); - protected MiniRaftClusterWithGrpc(String[] ids, String[] listenerIds, RaftProperties properties, Parameters parameters) { + public MiniRaftClusterWithGrpc(String[] ids, RaftProperties properties, Parameters parameters) { + this(ids, new String[0], properties, parameters); + } + + public MiniRaftClusterWithGrpc(String[] ids, String[] listenerIds, RaftProperties properties, Parameters parameters) { super(ids, listenerIds, properties, parameters); } @@ -66,13 +80,33 @@ protected Parameters setPropertiesAndInitParameters(RaftPeerId id, RaftGroup gro GrpcConfigKeys.Client.setPort(properties, NetUtils.createSocketAddr(address).getPort())); Optional.ofNullable(getAddress(id, group, RaftPeer::getAdminAddress)).ifPresent(address -> GrpcConfigKeys.Admin.setPort(properties, NetUtils.createSocketAddr(address).getPort())); + // Always run grpc integration tests with zero-copy enabled because the path of nonzero-copy is not risky. + GrpcConfigKeys.Server.setZeroCopyEnabled(properties, true); return parameters; } @Override protected void blockQueueAndSetDelay(String leaderId, int delayMs) throws InterruptedException { - RaftTestUtil.blockQueueAndSetDelay(getServers(), sendServerRequestInjection, + RaftTestUtil.blockQueueAndSetDelay(getServers(), SEND_SERVER_REQUEST_INJECTION, leaderId, delayMs, getTimeoutMax()); } + + @Override + public void shutdown() { + super.shutdown(); + assertZeroCopyMetrics(); + } + + public void assertZeroCopyMetrics() { + getServers().forEach(server -> server.getGroupIds().forEach(id -> { + LOG.info("Checking {}-{}", server.getId(), id); + RaftServer.Division division = RaftServerTestUtil.getDivision(server, id); + final GrpcServicesImpl service = (GrpcServicesImpl) RaftServerTestUtil.getServerRpc(division); + ZeroCopyMetrics zeroCopyMetrics = service.getZeroCopyMetrics(); + Assertions.assertEquals(0, zeroCopyMetrics.nonZeroCopyMessages()); + Assertions.assertEquals(zeroCopyMetrics.zeroCopyMessages(), zeroCopyMetrics.releasedMessages(), + "Unreleased zero copy messages: please check logs to find the leaks. "); + })); + } } diff --git a/ratis-metrics-api/pom.xml b/ratis-metrics-api/pom.xml index e51a9f9632..cc11e91d53 100644 --- a/ratis-metrics-api/pom.xml +++ b/ratis-metrics-api/pom.xml @@ -18,12 +18,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-metrics-api Apache Ratis Metrics API + + + true + + ratis-common diff --git a/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/MetricRegistriesLoader.java b/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/MetricRegistriesLoader.java index 0837b0bdd1..8baac7a465 100644 --- a/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/MetricRegistriesLoader.java +++ b/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/MetricRegistriesLoader.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; +import java.util.stream.Collectors; import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.ratis.util.ReflectionUtils; @@ -30,10 +31,9 @@ public final class MetricRegistriesLoader { - private static final Logger LOG = LoggerFactory.getLogger(MetricRegistries.class); + private static final Logger LOG = LoggerFactory.getLogger(MetricRegistriesLoader.class); - private static final String DEFAULT_CLASS - = "org.apache.ratis.metrics.impl.MetricRegistriesImpl"; + static final String DEFAULT_CLASS = "org.apache.ratis.metrics.impl.MetricRegistriesImpl"; private MetricRegistriesLoader() { } @@ -56,32 +56,25 @@ public static MetricRegistries load() { * @return A {@link MetricRegistries} implementation. */ @VisibleForTesting - static MetricRegistries load(List availableImplementations) { - - if (availableImplementations.size() == 1) { - // One and only one instance -- what we want/expect - MetricRegistries impl = availableImplementations.get(0); - LOG.info("Loaded MetricRegistries " + impl.getClass()); - return impl; - } else if (availableImplementations.isEmpty()) { + static MetricRegistries load(List registries) { + if (registries.isEmpty()) { try { return ReflectionUtils.newInstance(Class.forName(DEFAULT_CLASS).asSubclass(MetricRegistries.class)); } catch (ClassNotFoundException e) { - throw new RuntimeException(e); + throw new IllegalStateException("Failed to load default MetricRegistries " + DEFAULT_CLASS, e); } + } + + final MetricRegistries first = registries.get(0); + if (registries.size() == 1) { + // One and only one instance -- what we want/expect + LOG.debug("Loaded {}", first.getClass()); } else { // Tell the user they're doing something wrong, and choose the first impl. - StringBuilder sb = new StringBuilder(); - for (MetricRegistries factory : availableImplementations) { - if (sb.length() > 0) { - sb.append(", "); - } - sb.append(factory.getClass()); - } - LOG.warn("Found multiple MetricRegistries implementations: " + sb - + ". Using first found implementation: " + availableImplementations.get(0)); - return availableImplementations.get(0); + final List> classes = registries.stream().map(Object::getClass).collect(Collectors.toList()); + LOG.warn("Found multiple MetricRegistries: {}. Using the first: {}", classes, first.getClass()); } + return first; } private static List getDefinedImplementations() { diff --git a/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/RatisMetrics.java b/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/RatisMetrics.java index 32bcf52e3c..eafc3849d1 100644 --- a/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/RatisMetrics.java +++ b/ratis-metrics-api/src/main/java/org/apache/ratis/metrics/RatisMetrics.java @@ -49,6 +49,13 @@ protected static > Map> newCounter return Collections.unmodifiableMap(maps); } + protected static > Map newCounterMap( + Class clazz, Function constructor) { + final EnumMap map = new EnumMap<>(clazz); + Arrays.stream(clazz.getEnumConstants()).forEach(t -> map.put(t, constructor.apply(t))); + return Collections.unmodifiableMap(map); + } + protected static > Map newTimerMap( Class clazz, Function constructor) { final EnumMap map = new EnumMap<>(clazz); diff --git a/ratis-metrics-default/pom.xml b/ratis-metrics-default/pom.xml index 7caccf634b..cad2150c7d 100644 --- a/ratis-metrics-default/pom.xml +++ b/ratis-metrics-default/pom.xml @@ -18,7 +18,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-metrics-default @@ -50,10 +50,16 @@ - junit - junit + org.junit.jupiter + junit-jupiter-engine test + + org.junit.jupiter + junit-jupiter-api + test + + org.mockito mockito-core diff --git a/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java b/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java index 17968ae9f8..088508fab9 100644 --- a/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java +++ b/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/MetricRegistriesImpl.java @@ -28,6 +28,7 @@ import org.apache.ratis.metrics.MetricRegistries; import org.apache.ratis.metrics.MetricRegistryInfo; import org.apache.ratis.metrics.RatisMetricRegistry; +import org.apache.ratis.util.RefCountingMap; import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/RefCountingMap.java b/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/RefCountingMap.java deleted file mode 100644 index 49759781f8..0000000000 --- a/ratis-metrics-default/src/main/java/org/apache/ratis/metrics/impl/RefCountingMap.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis.metrics.impl; - -import java.util.Collection; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -/** - * A map of K to V, but does ref counting for added and removed values. The values are - * not added directly, but instead requested from the given Supplier if ref count == 0. Each put() - * call will increment the ref count, and each remove() will decrement it. The values are removed - * from the map iff ref count == 0. - */ -class RefCountingMap { - private static class Payload { - private final V value; - private final AtomicInteger refCount = new AtomicInteger(); - - Payload(V v) { - this.value = v; - } - - V get() { - return value; - } - - V increment() { - return refCount.incrementAndGet() > 0? value: null; - } - - Payload decrement() { - return refCount.decrementAndGet() > 0? this: null; - } - } - - private final ConcurrentMap> map = new ConcurrentHashMap<>(); - - V put(K k, Supplier supplier) { - return map.compute(k, (k1, old) -> old != null? old: new Payload<>(supplier.get())).increment(); - } - - static V get(Payload p) { - return p == null ? null : p.get(); - } - - V get(K k) { - return get(map.get(k)); - } - - /** - * Decrements the ref count of k, and removes from map if ref count == 0. - * @param k the key to remove - * @return the value associated with the specified key or null if key is removed from map. - */ - V remove(K k) { - return get(map.computeIfPresent(k, (k1, v) -> v.decrement())); - } - - void clear() { - map.clear(); - } - - Set keySet() { - return map.keySet(); - } - - Collection values() { - return map.values().stream().map(Payload::get).collect(Collectors.toList()); - } - - int size() { - return map.size(); - } -} diff --git a/ratis-metrics-default/src/test/java/org/apache/ratis/metrics/TestMetricRegistriesLoader.java b/ratis-metrics-default/src/test/java/org/apache/ratis/metrics/TestMetricRegistriesLoader.java index 0093fa759f..9816cc99c5 100644 --- a/ratis-metrics-default/src/test/java/org/apache/ratis/metrics/TestMetricRegistriesLoader.java +++ b/ratis-metrics-default/src/test/java/org/apache/ratis/metrics/TestMetricRegistriesLoader.java @@ -17,15 +17,15 @@ */ package org.apache.ratis.metrics; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.mockito.Mockito.mock; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import org.apache.ratis.metrics.impl.MetricRegistriesImpl; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collections; @@ -34,6 +34,12 @@ * Test class for {@link MetricRegistriesLoader}. */ public class TestMetricRegistriesLoader { + @Test + public void testLoadEmptyInstance() { + MetricRegistries instance = MetricRegistriesLoader.load(Collections.emptyList()); + assertEquals(MetricRegistriesLoader.DEFAULT_CLASS, instance.getClass().getName()); + } + @Test public void testLoadSingleInstance() { MetricRegistries loader = mock(MetricRegistries.class); @@ -57,7 +63,7 @@ public void testLoadMultipleInstances() { @Test public void testLoadDefault() { final MetricRegistries r = MetricRegistriesLoader.load(); - Assert.assertSame(MetricRegistriesImpl.class, r.getClass()); + Assertions.assertSame(MetricRegistriesImpl.class, r.getClass()); } @Test @@ -67,20 +73,20 @@ public void testAddRemoveReporter() { Consumer reporter = v-> cntr.incrementAndGet(); Consumer stopReporter = v-> cntr.incrementAndGet(); r.addReporterRegistration(reporter, stopReporter); - + // check if add and remove of metric do reporting counter increase MetricRegistryInfo info = new MetricRegistryInfo("t1", "t1", "t1", "t1"); r.create(info); - Assert.assertTrue(cntr.get() == 1); + assertEquals(1, cntr.get()); r.remove(info); - Assert.assertTrue(cntr.get() == 2); + assertEquals(2, cntr.get()); // after removal, add and remove of metric must not do any increase r.removeReporterRegistration(reporter, stopReporter); r.create(info); - Assert.assertTrue(cntr.get() == 2); + assertEquals(2, cntr.get()); r.remove(info); - Assert.assertTrue(cntr.get() == 2); + assertEquals(2, cntr.get()); } } diff --git a/ratis-metrics-dropwizard3/pom.xml b/ratis-metrics-dropwizard3/pom.xml index 04b86cbaa8..cb9f504276 100644 --- a/ratis-metrics-dropwizard3/pom.xml +++ b/ratis-metrics-dropwizard3/pom.xml @@ -18,7 +18,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-metrics-dropwizard3 @@ -54,8 +54,13 @@ - junit - junit + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-api test diff --git a/ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/Dm3MetricRegistriesImpl.java b/ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/Dm3MetricRegistriesImpl.java index b26f2e27a4..a90c5a0cee 100644 --- a/ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/Dm3MetricRegistriesImpl.java +++ b/ratis-metrics-dropwizard3/src/main/java/org/apache/ratis/metrics/dropwizard3/Dm3MetricRegistriesImpl.java @@ -28,6 +28,7 @@ import org.apache.ratis.metrics.MetricRegistries; import org.apache.ratis.metrics.MetricRegistryInfo; import org.apache.ratis.metrics.RatisMetricRegistry; +import org.apache.ratis.util.RefCountingMap; import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestLoadDm3MetricRegistries.java b/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestLoadDm3MetricRegistries.java index 92a36f5c86..5b3897ffd4 100644 --- a/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestLoadDm3MetricRegistries.java +++ b/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestLoadDm3MetricRegistries.java @@ -23,8 +23,8 @@ import org.apache.ratis.metrics.MetricRegistriesLoader; import org.apache.ratis.metrics.MetricRegistryInfo; import org.apache.ratis.metrics.RatisMetricRegistry; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; /** * Test class for {@link MetricRegistriesLoader}. @@ -33,7 +33,7 @@ public class TestLoadDm3MetricRegistries { @Test public void testLoadDm3() { final MetricRegistries r = MetricRegistriesLoader.load(); - Assert.assertSame(Dm3MetricRegistriesImpl.class, r.getClass()); + Assertions.assertSame(Dm3MetricRegistriesImpl.class, r.getClass()); } @Test @@ -47,15 +47,15 @@ public void testAddRemoveReporter() { // check if add and remove of metric do reporting counter increase MetricRegistryInfo info = new MetricRegistryInfo("t1", "t1", "t1", "t1"); r.create(info); - Assert.assertTrue(cntr.get() == 1); + Assertions.assertEquals(1, cntr.get()); r.remove(info); - Assert.assertTrue(cntr.get() == 2); + Assertions.assertEquals(2, cntr.get()); // after removal, add and remove of metric must not do any increase r.removeReporterRegistration(reporter, stopReporter); r.create(info); - Assert.assertTrue(cntr.get() == 2); + Assertions.assertEquals(2, cntr.get()); r.remove(info); - Assert.assertTrue(cntr.get() == 2); + Assertions.assertEquals(2, cntr.get()); } } diff --git a/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestRefCountingMap.java b/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestRefCountingMap.java deleted file mode 100644 index 1aa15d37c7..0000000000 --- a/ratis-metrics-dropwizard3/src/test/java/org/apache/ratis/metrics/dropwizard3/TestRefCountingMap.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis.metrics.dropwizard3; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.util.Collection; -import java.util.Set; - -import org.apache.ratis.thirdparty.com.google.common.collect.Lists; -import org.junit.Before; -import org.junit.Test; - -public class TestRefCountingMap { - - private RefCountingMap map; - - @Before - public void setUp() { - map = new RefCountingMap<>(); - } - - @Test - public void testPutGet() { - map.put("foo", () -> "foovalue"); - - String v = map.get("foo"); - assertNotNull(v); - assertEquals("foovalue", v); - } - - @Test - public void testPutMulti() { - String v1 = map.put("foo", () -> "foovalue"); - String v2 = map.put("foo", () -> "foovalue2"); - String v3 = map.put("foo", () -> "foovalue3"); - - String v = map.get("foo"); - assertEquals("foovalue", v); - assertEquals(v, v1); - assertEquals(v, v2); - assertEquals(v, v3); - } - - @Test - public void testPutRemove() { - map.put("foo", () -> "foovalue"); - String v = map.remove("foo"); - assertNull(v); - v = map.get("foo"); - assertNull(v); - } - - @Test - public void testPutRemoveMulti() { - map.put("foo", () -> "foovalue"); - map.put("foo", () -> "foovalue2"); - map.put("foo", () -> "foovalue3"); - - // remove 1 - String v = map.remove("foo"); - assertEquals("foovalue", v); - - // remove 2 - v = map.remove("foo"); - assertEquals("foovalue", v); - - // remove 3 - v = map.remove("foo"); - assertNull(v); - v = map.get("foo"); - assertNull(v); - } - - @Test - public void testSize() { - assertEquals(0, map.size()); - - // put a key - map.put("foo", () -> "foovalue"); - assertEquals(1, map.size()); - - // put a different key - map.put("bar", () -> "foovalue2"); - assertEquals(2, map.size()); - - // put the same key again - map.put("bar", () -> "foovalue3"); - assertEquals(2, map.size()); // map should be same size - } - - @Test - public void testClear() { - map.put("foo", () -> "foovalue"); - map.put("bar", () -> "foovalue2"); - map.put("baz", () -> "foovalue3"); - - map.clear(); - - assertEquals(0, map.size()); - } - - - @Test - public void testKeySet() { - map.put("foo", () -> "foovalue"); - map.put("bar", () -> "foovalue2"); - map.put("baz", () -> "foovalue3"); - - Set keys = map.keySet(); - assertEquals(3, keys.size()); - - Lists.newArrayList("foo", "bar", "baz").stream().forEach(v -> assertTrue(keys.contains(v))); - } - - @Test - public void testValues() { - map.put("foo", () -> "foovalue"); - map.put("foo", () -> "foovalue2"); - map.put("bar", () -> "foovalue3"); - map.put("baz", () -> "foovalue4"); - - Collection values = map.values(); - assertEquals(3, values.size()); - - Lists.newArrayList("foovalue", "foovalue3", "foovalue4").stream() - .forEach(v -> assertTrue(values.contains(v))); - } -} diff --git a/ratis-netty/pom.xml b/ratis-netty/pom.xml index c29f35171f..482fcb4687 100644 --- a/ratis-netty/pom.xml +++ b/ratis-netty/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-netty Apache Ratis Netty Support + + + true + + org.apache.ratis @@ -73,11 +78,5 @@ slf4j-api - - com.github.spotbugs - spotbugs-annotations - provided - true - diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyClient.java b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyClient.java index 0cf4bd3833..efea5fd0f8 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyClient.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyClient.java @@ -33,13 +33,17 @@ import java.net.InetSocketAddress; public class NettyClient implements Closeable { - private final LifeCycle lifeCycle = new LifeCycle(JavaUtils.getClassSimpleName(getClass())); - + private final LifeCycle lifeCycle; + private final String serverAddress; private Channel channel; + NettyClient(String serverAddress) { + this.lifeCycle = new LifeCycle(JavaUtils.getClassSimpleName(getClass()) + "-" + serverAddress); + this.serverAddress = serverAddress; + } + /** Connects to the given server address. */ - public void connect(String serverAddress, EventLoopGroup group, - ChannelInitializer initializer) + public void connect(EventLoopGroup group, ChannelInitializer initializer) throws InterruptedException { final InetSocketAddress address = NetUtils.createSocketAddr(serverAddress); @@ -57,13 +61,16 @@ public void connect(String serverAddress, EventLoopGroup group, @Override public void close() { - lifeCycle.checkStateAndClose(() -> { - channel.close().syncUninterruptibly(); - }); + lifeCycle.checkStateAndClose(() -> NettyUtils.closeChannel(channel, serverAddress)); } public ChannelFuture writeAndFlush(Object msg) { lifeCycle.assertCurrentState(LifeCycle.States.RUNNING); return channel.writeAndFlush(msg); } + + @Override + public String toString() { + return lifeCycle.toString(); + } } diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyDataStreamUtils.java b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyDataStreamUtils.java index bd03fefcc3..583d6e3e94 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyDataStreamUtils.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyDataStreamUtils.java @@ -23,7 +23,7 @@ import org.apache.ratis.io.FilePositionCount; import org.apache.ratis.io.StandardWriteOption; import org.apache.ratis.io.WriteOption; -import org.apache.ratis.netty.server.DataStreamRequestByteBuf; +import org.apache.ratis.datastream.impl.DataStreamRequestByteBuf; import org.apache.ratis.proto.RaftProtos.DataStreamReplyHeaderProto; import org.apache.ratis.proto.RaftProtos.DataStreamRequestHeaderProto; import org.apache.ratis.proto.RaftProtos.DataStreamPacketHeaderProto; @@ -103,11 +103,11 @@ static void encodeDataStreamRequestHeader(DataStreamRequest request, Consumer out) { out.accept(Unpooled.wrappedBuffer(buffer)); } + static void encodeDataStreamRequestByteBuf(DataStreamRequestByteBuf request, Consumer out, + ByteBufAllocator allocator) { + encodeDataStreamRequestHeader(request, out, allocator); + encodeByteBuf(request.slice(), out); + } + + static void encodeByteBuf(ByteBuf buffer, Consumer out) { + if (buffer.readableBytes() == 0) { + out.accept(Unpooled.EMPTY_BUFFER); // to avoid EncoderException: must produce at least one message + return; + } + out.accept(buffer); + } + static void encodeDataStreamRequestFilePositionCount( DataStreamRequestFilePositionCount request, Consumer out, ByteBufAllocator allocator) { encodeDataStreamRequestHeader(request, out, allocator); @@ -138,7 +152,7 @@ static void encodeDataStreamRequestFilePositionCount( static void encodeDataStreamReplyByteBuffer(DataStreamReplyByteBuffer reply, Consumer out, ByteBufAllocator allocator) { ByteBuffer headerBuf = getDataStreamReplyHeaderProtoByteBuf(reply); - final ByteBuf headerLenBuf = allocator.directBuffer(DataStreamPacketHeader.getSizeOfHeaderLen()); + final ByteBuf headerLenBuf = allocator.ioBuffer(DataStreamPacketHeader.getSizeOfHeaderLen()); headerLenBuf.writeInt(headerBuf.remaining()); out.accept(headerLenBuf); out.accept(Unpooled.wrappedBuffer(headerBuf)); diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyRpcProxy.java b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyRpcProxy.java index b7a04b050b..41269f76e3 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyRpcProxy.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyRpcProxy.java @@ -99,7 +99,7 @@ public static long getCallId(RaftNettyServerReplyProto proto) { class Connection implements Closeable { - private final NettyClient client = new NettyClient(); + private final NettyClient client = new NettyClient(peer.getAddress()); private final Queue> replies = new LinkedList<>(); @@ -137,7 +137,7 @@ protected void initChannel(SocketChannel ch) throws Exception { } }; - client.connect(peer.getAddress(), group, initializer); + client.connect(group, initializer); } synchronized ChannelFuture offer(RaftNettyServerRequestProto request, @@ -176,6 +176,12 @@ public void close() { connection.close(); } + public CompletableFuture sendAsync(RaftNettyServerRequestProto proto) { + final CompletableFuture reply = new CompletableFuture<>(); + connection.offer(proto, reply); + return reply; + } + public RaftNettyServerReplyProto send( RaftRpcRequestProto request, RaftNettyServerRequestProto proto) throws IOException { diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyUtils.java b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyUtils.java index 8cce291af9..37666bf189 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/NettyUtils.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/NettyUtils.java @@ -22,6 +22,8 @@ import org.apache.ratis.security.TlsConf.KeyManagerConf; import org.apache.ratis.security.TlsConf.PrivateKeyConf; import org.apache.ratis.security.TlsConf.TrustManagerConf; +import org.apache.ratis.thirdparty.io.netty.channel.Channel; +import org.apache.ratis.thirdparty.io.netty.channel.ChannelFuture; import org.apache.ratis.thirdparty.io.netty.channel.EventLoopGroup; import org.apache.ratis.thirdparty.io.netty.channel.ServerChannel; import org.apache.ratis.thirdparty.io.netty.channel.epoll.Epoll; @@ -35,16 +37,19 @@ import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContext; import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder; import org.apache.ratis.util.ConcurrentUtils; +import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.KeyManager; import javax.net.ssl.TrustManager; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; public interface NettyUtils { Logger LOG = LoggerFactory.getLogger(NettyUtils.class); + TimeDuration CLOSE_TIMEOUT = TimeDuration.valueOf(5, TimeUnit.SECONDS); class Print { private static final AtomicBoolean PRINTED_EPOLL_UNAVAILABILITY_CAUSE = new AtomicBoolean(); @@ -176,4 +181,19 @@ static Class getServerChannelClass(EventLoopGroup event return eventLoopGroup instanceof EpollEventLoopGroup ? EpollServerSocketChannel.class : NioServerSocketChannel.class; } + + static void closeChannel(Channel channel, String name) { + final ChannelFuture f = channel.close(); + final boolean completed; + try { + completed = f.await(CLOSE_TIMEOUT.getDuration(), CLOSE_TIMEOUT.getUnit()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.info("Interrupted closeChannel {} ", name, e); + return; + } + if (!completed) { + LOG.warn("closeChannel {} is not yet completed in {}", name, CLOSE_TIMEOUT); + } + } } \ No newline at end of file diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientReplies.java b/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientReplies.java index fc97b6fe34..4c49b1d160 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientReplies.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientReplies.java @@ -30,11 +30,10 @@ import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; public class NettyClientReplies { public static final Logger LOG = LoggerFactory.getLogger(NettyClientReplies.class); @@ -56,8 +55,8 @@ class ReplyMap { ReplyEntry submitRequest(RequestEntry requestEntry, boolean isClose, CompletableFuture f) { LOG.debug("put {} to the map for {}", requestEntry, clientInvocationId); - final MemoizedSupplier replySupplier = MemoizedSupplier.valueOf(() -> new ReplyEntry(isClose, f)); - return map.computeIfAbsent(requestEntry, r -> replySupplier.get()); + // ConcurrentHashMap.computeIfAbsent javadoc: the function is applied at most once per key. + return map.computeIfAbsent(requestEntry, r -> new ReplyEntry(isClose, f)); } void receiveReply(DataStreamReply reply) { @@ -147,7 +146,7 @@ public String toString() { static class ReplyEntry { private final boolean isClosed; private final CompletableFuture replyFuture; - private final AtomicReference> timeoutFuture = new AtomicReference<>(); + private ScheduledFuture timeoutFuture; // for reply timeout ReplyEntry(boolean isClosed, CompletableFuture replyFuture) { this.isClosed = isClosed; @@ -158,22 +157,26 @@ boolean isClosed() { return isClosed; } - void complete(DataStreamReply reply) { - cancelTimeoutFuture(); + synchronized void complete(DataStreamReply reply) { + cancel(timeoutFuture); replyFuture.complete(reply); } - void completeExceptionally(Throwable t) { - cancelTimeoutFuture(); + synchronized void completeExceptionally(Throwable t) { + cancel(timeoutFuture); replyFuture.completeExceptionally(t); } - private void cancelTimeoutFuture() { - Optional.ofNullable(timeoutFuture.get()).ifPresent(f -> f.cancel(false)); + static void cancel(ScheduledFuture future) { + if (future != null) { + future.cancel(true); + } } - void setTimeoutFuture(ScheduledFuture timeoutFuture) { - this.timeoutFuture.compareAndSet(null, timeoutFuture); + synchronized void scheduleTimeout(Supplier> scheduleMethod) { + if (!replyFuture.isDone()) { + timeoutFuture = scheduleMethod.get(); + } } } } diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientRpc.java b/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientRpc.java index c816e29ee8..ef34caf17d 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientRpc.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientRpc.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.netty.client; +import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.client.impl.ClientProtoUtils; import org.apache.ratis.client.impl.RaftClientRpcWithProxy; import org.apache.ratis.conf.RaftProperties; @@ -28,12 +29,79 @@ import org.apache.ratis.proto.RaftProtos.GroupManagementRequestProto; import org.apache.ratis.proto.RaftProtos.SetConfigurationRequestProto; import org.apache.ratis.proto.netty.NettyProtos.RaftNettyServerRequestProto; +import org.apache.ratis.protocol.exceptions.TimeoutIOException; +import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.TimeDuration; +import org.apache.ratis.util.TimeoutExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.concurrent.CompletableFuture; public class NettyClientRpc extends RaftClientRpcWithProxy { + + public static final Logger LOG = LoggerFactory.getLogger(NettyClientRpc.class); + + private ClientId clientId; + private final TimeDuration requestTimeout; + private final TimeoutExecutor scheduler = TimeoutExecutor.getInstance(); + public NettyClientRpc(ClientId clientId, RaftProperties properties) { super(new NettyRpcProxy.PeerMap(clientId.toString(), properties)); + this.clientId = clientId; + this.requestTimeout = RaftClientConfigKeys.Rpc.requestTimeout(properties); + } + + @Override + public CompletableFuture sendRequestAsync(RaftClientRequest request) { + final RaftPeerId serverId = request.getServerId(); + long callId = request.getCallId(); + try { + final NettyRpcProxy proxy = getProxies().getProxy(serverId); + final RaftNettyServerRequestProto serverRequestProto = buildRequestProto(request); + final CompletableFuture replyFuture = new CompletableFuture<>(); + + proxy.sendAsync(serverRequestProto).thenApply(replyProto -> { + if (request instanceof GroupListRequest) { + return ClientProtoUtils.toGroupListReply(replyProto.getGroupListReply()); + } else if (request instanceof GroupInfoRequest) { + return ClientProtoUtils.toGroupInfoReply(replyProto.getGroupInfoReply()); + } else { + return ClientProtoUtils.toRaftClientReply(replyProto.getRaftClientReply()); + } + }).whenComplete((reply, e) -> { + if (e == null) { + if (reply == null) { + e = new NullPointerException("Both reply==null && e==null"); + } + if (e == null) { + e = reply.getNotLeaderException(); + } + if (e == null) { + e = reply.getLeaderNotReadyException(); + } + } + + if (e != null) { + replyFuture.completeExceptionally(e); + } else { + replyFuture.complete(reply); + } + }); + + scheduler.onTimeout(requestTimeout, () -> { + if (!replyFuture.isDone()) { + final String s = clientId + "->" + serverId + " request #" + + callId + " timeout " + requestTimeout.getDuration(); + replyFuture.completeExceptionally(new TimeoutIOException(s)); + } + }, LOG, () -> "Timeout check for client request #" + callId); + + return replyFuture; + } catch (Throwable e) { + return JavaUtils.completeExceptionally(e); + } } @Override @@ -41,58 +109,69 @@ public RaftClientReply sendRequest(RaftClientRequest request) throws IOException final RaftPeerId serverId = request.getServerId(); final NettyRpcProxy proxy = getProxies().getProxy(serverId); + final RaftNettyServerRequestProto serverRequestProto = buildRequestProto(request); + final RaftRpcRequestProto rpcRequest = getRpcRequestProto(serverRequestProto); + if (request instanceof GroupListRequest) { + return ClientProtoUtils.toGroupListReply( + proxy.send(rpcRequest, serverRequestProto).getGroupListReply()); + } else if (request instanceof GroupInfoRequest) { + return ClientProtoUtils.toGroupInfoReply( + proxy.send(rpcRequest, serverRequestProto).getGroupInfoReply()); + } else { + return ClientProtoUtils.toRaftClientReply( + proxy.send(rpcRequest, serverRequestProto).getRaftClientReply()); + } + } + + private RaftNettyServerRequestProto buildRequestProto(RaftClientRequest request) { final RaftNettyServerRequestProto.Builder b = RaftNettyServerRequestProto.newBuilder(); - final RaftRpcRequestProto rpcRequest; if (request instanceof GroupManagementRequest) { final GroupManagementRequestProto proto = ClientProtoUtils.toGroupManagementRequestProto( (GroupManagementRequest)request); b.setGroupManagementRequest(proto); - rpcRequest = proto.getRpcRequest(); } else if (request instanceof SetConfigurationRequest) { final SetConfigurationRequestProto proto = ClientProtoUtils.toSetConfigurationRequestProto( (SetConfigurationRequest)request); b.setSetConfigurationRequest(proto); - rpcRequest = proto.getRpcRequest(); } else if (request instanceof GroupListRequest) { final RaftProtos.GroupListRequestProto proto = ClientProtoUtils.toGroupListRequestProto( (GroupListRequest)request); b.setGroupListRequest(proto); - rpcRequest = proto.getRpcRequest(); } else if (request instanceof GroupInfoRequest) { final RaftProtos.GroupInfoRequestProto proto = ClientProtoUtils.toGroupInfoRequestProto( (GroupInfoRequest)request); b.setGroupInfoRequest(proto); - rpcRequest = proto.getRpcRequest(); } else if (request instanceof TransferLeadershipRequest) { final RaftProtos.TransferLeadershipRequestProto proto = ClientProtoUtils.toTransferLeadershipRequestProto( (TransferLeadershipRequest)request); b.setTransferLeadershipRequest(proto); - rpcRequest = proto.getRpcRequest(); } else if (request instanceof SnapshotManagementRequest) { final RaftProtos.SnapshotManagementRequestProto proto = ClientProtoUtils.toSnapshotManagementRequestProto( (SnapshotManagementRequest) request); b.setSnapshotManagementRequest(proto); - rpcRequest = proto.getRpcRequest(); } else if (request instanceof LeaderElectionManagementRequest) { final RaftProtos.LeaderElectionManagementRequestProto proto = ClientProtoUtils.toLeaderElectionManagementRequestProto( (LeaderElectionManagementRequest) request); b.setLeaderElectionManagementRequest(proto); - rpcRequest = proto.getRpcRequest(); } else { final RaftClientRequestProto proto = ClientProtoUtils.toRaftClientRequestProto(request); b.setRaftClientRequest(proto); - rpcRequest = proto.getRpcRequest(); } - if (request instanceof GroupListRequest) { - return ClientProtoUtils.toGroupListReply( - proxy.send(rpcRequest, b.build()).getGroupListReply()); - } else if (request instanceof GroupInfoRequest) { - return ClientProtoUtils.toGroupInfoReply( - proxy.send(rpcRequest, b.build()).getGroupInfoReply()); + return b.build(); + } + + private RaftRpcRequestProto getRpcRequestProto(RaftNettyServerRequestProto serverRequestProto) { + if (serverRequestProto.hasGroupManagementRequest()) { + return serverRequestProto.getGroupManagementRequest().getRpcRequest(); + } else if (serverRequestProto.hasSetConfigurationRequest()) { + return serverRequestProto.getSetConfigurationRequest().getRpcRequest(); + } else if (serverRequestProto.hasGroupListRequest()) { + return serverRequestProto.getGroupListRequest().getRpcRequest(); + } else if (serverRequestProto.hasGroupInfoRequest()) { + return serverRequestProto.getGroupInfoRequest().getRpcRequest(); } else { - return ClientProtoUtils.toRaftClientReply( - proxy.send(rpcRequest, b.build()).getRaftClientReply()); + return serverRequestProto.getRaftClientRequest().getRpcRequest(); } } } diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientStreamRpc.java b/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientStreamRpc.java index 020acc2fdb..534fcc5818 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientStreamRpc.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/client/NettyClientStreamRpc.java @@ -21,6 +21,7 @@ import org.apache.ratis.client.DataStreamClientRpc; import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.datastream.impl.DataStreamRequestByteBuf; import org.apache.ratis.datastream.impl.DataStreamRequestByteBuffer; import org.apache.ratis.datastream.impl.DataStreamRequestFilePositionCount; import org.apache.ratis.io.StandardWriteOption; @@ -52,7 +53,6 @@ import org.apache.ratis.thirdparty.io.netty.handler.codec.ByteToMessageDecoder; import org.apache.ratis.thirdparty.io.netty.handler.codec.MessageToMessageEncoder; import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContext; -import org.apache.ratis.thirdparty.io.netty.util.concurrent.ScheduledFuture; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.NetUtils; @@ -370,6 +370,7 @@ public void initChannel(SocketChannel ch) { p.addLast(ENCODER); p.addLast(ENCODER_FILE_POSITION_COUNT); p.addLast(ENCODER_BYTE_BUFFER); + p.addLast(ENCODER_BYTE_BUF); p.addLast(newDecoder()); p.addLast(handler); } @@ -386,6 +387,16 @@ protected void encode(ChannelHandlerContext context, DataStreamRequestByteBuffer } } + static final MessageToMessageEncoder ENCODER_BYTE_BUF = new EncoderByteBuf(); + + @ChannelHandler.Sharable + static class EncoderByteBuf extends MessageToMessageEncoder { + @Override + protected void encode(ChannelHandlerContext context, DataStreamRequestByteBuf request, List out) { + NettyDataStreamUtils.encodeDataStreamRequestByteBuf(request, out::add, context.alloc()); + } + } + static final MessageToMessageEncoder ENCODER_FILE_POSITION_COUNT = new EncoderFilePositionCount(); @@ -454,15 +465,13 @@ public CompletableFuture streamAsync(DataStreamRequest request) LOG.debug("{}: write after {}", this, request); final TimeDuration timeout = isClose ? closeTimeout : requestTimeout; - // if reply success cancel this future - final ScheduledFuture timeoutFuture = channel.eventLoop().schedule(() -> { + replyEntry.scheduleTimeout(() -> channel.eventLoop().schedule(() -> { if (!f.isDone()) { f.completeExceptionally(new TimeoutIOException( - "Timeout " + timeout + ": Failed to send " + request + " channel: " + channel)); + "Timeout " + timeout + ": Failed to send " + request + " via channel " + channel)); replyMap.fail(requestEntry); } - }, timeout.toLong(timeout.getUnit()), timeout.getUnit()); - replyEntry.setTimeoutFuture(timeoutFuture); + }, timeout.getDuration(), timeout.getUnit())); } }); return f; diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/server/ChannelMap.java b/ratis-netty/src/main/java/org/apache/ratis/netty/server/ChannelMap.java new file mode 100644 index 0000000000..7b0d761840 --- /dev/null +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/server/ChannelMap.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.netty.server; + +import org.apache.ratis.protocol.ClientInvocationId; +import org.apache.ratis.thirdparty.io.netty.channel.ChannelId; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** Map: {@link ChannelId} -> {@link ClientInvocationId}s. */ +class ChannelMap { + private final Map> map = new ConcurrentHashMap<>(); + + void add(ChannelId channelId, ClientInvocationId clientInvocationId) { + map.computeIfAbsent(channelId, (e) -> new ConcurrentHashMap<>()) + .put(clientInvocationId, clientInvocationId); + } + + void remove(ChannelId channelId, ClientInvocationId clientInvocationId) { + Optional.ofNullable(map.get(channelId)) + .ifPresent((ids) -> ids.remove(clientInvocationId)); + } + + Set remove(ChannelId channelId) { + return Optional.ofNullable(map.remove(channelId)) + .map(Map::keySet) + .orElse(Collections.emptySet()); + } +} diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamManagement.java b/ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamManagement.java index 276a365cea..127ed09835 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamManagement.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/server/DataStreamManagement.java @@ -18,10 +18,12 @@ package org.apache.ratis.netty.server; -import org.apache.ratis.client.DataStreamOutputRpc; +import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.client.impl.ClientProtoUtils; +import org.apache.ratis.client.impl.DataStreamClientImpl.DataStreamOutputImpl; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.datastream.impl.DataStreamReplyByteBuffer; +import org.apache.ratis.datastream.impl.DataStreamRequestByteBuf; import org.apache.ratis.io.StandardWriteOption; import org.apache.ratis.io.WriteOption; import org.apache.ratis.metrics.Timekeeper; @@ -69,13 +71,10 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; @@ -111,12 +110,12 @@ void cleanUp() { } static class RemoteStream { - private final DataStreamOutputRpc out; + private final DataStreamOutputImpl out; private final AtomicReference> sendFuture = new AtomicReference<>(CompletableFuture.completedFuture(null)); private final RequestMetrics metrics; - RemoteStream(DataStreamOutputRpc out, RequestMetrics metrics) { + RemoteStream(DataStreamOutputImpl out, RequestMetrics metrics) { this.metrics = metrics; this.out = out; } @@ -132,7 +131,7 @@ static Iterable addFlush(List original) { CompletableFuture write(DataStreamRequestByteBuf request, Executor executor) { final Timekeeper.Context context = metrics.start(); return composeAsync(sendFuture, executor, - n -> out.writeAsync(request.slice().nioBuffer(), addFlush(request.getWriteOptionList())) + n -> out.writeAsync(request.slice().retain(), addFlush(request.getWriteOptionList())) .whenComplete((l, e) -> metrics.stop(context, e == null))); } } @@ -142,20 +141,20 @@ static class StreamInfo { private final boolean primary; private final LocalStream local; private final Set remotes; - private final RaftServer server; + private final Division division; private final AtomicReference> previous = new AtomicReference<>(CompletableFuture.completedFuture(null)); - StreamInfo(RaftClientRequest request, boolean primary, CompletableFuture stream, RaftServer server, - CheckedBiFunction, Set, IOException> getStreams, + StreamInfo(RaftClientRequest request, boolean primary, CompletableFuture stream, Division division, + CheckedBiFunction, Set, IOException> getStreams, Function metricsConstructor) throws IOException { this.request = request; this.primary = primary; this.local = new LocalStream(stream, metricsConstructor.apply(RequestType.LOCAL_WRITE)); - this.server = server; - final Set successors = getSuccessors(server.getId()); - final Set outs = getStreams.apply(request, successors); + this.division = division; + final Set successors = getSuccessors(division.getId()); + final Set outs = getStreams.apply(request, successors); this.remotes = outs.stream() .map(o -> new RemoteStream(o, metricsConstructor.apply(RequestType.REMOTE_WRITE))) .collect(Collectors.toSet()); @@ -169,16 +168,12 @@ RaftClientRequest getRequest() { return request; } - Division getDivision() throws IOException { - return server.getDivision(request.getRaftGroupId()); + Division getDivision() { + return division; } Collection getCommitInfos() { - try { - return getDivision().getCommitInfos(); - } catch (IOException e) { - throw new IllegalStateException(e); - } + return getDivision().getCommitInfos(); } boolean isPrimary() { @@ -198,7 +193,7 @@ public String toString() { return JavaUtils.getClassSimpleName(getClass()) + ":" + request; } - private Set getSuccessors(RaftPeerId peerId) throws IOException { + private Set getSuccessors(RaftPeerId peerId) { final RaftConfiguration conf = getDivision().getRaftConf(); final RoutingTable routingTable = request.getRoutingTable(); @@ -207,66 +202,31 @@ private Set getSuccessors(RaftPeerId peerId) throws IOException { } if (isPrimary()) { - // Default start topology + // Default star topology // get the other peers from the current configuration return conf.getCurrentPeers().stream() - .filter(p -> !p.getId().equals(server.getId())) + .filter(p -> !p.getId().equals(division.getId())) .collect(Collectors.toSet()); } return Collections.emptySet(); } - } - - static class StreamMap { - private final ConcurrentMap map = new ConcurrentHashMap<>(); - - StreamInfo computeIfAbsent(ClientInvocationId key, Function function) { - final StreamInfo info = map.computeIfAbsent(key, function); - LOG.debug("computeIfAbsent({}) returns {}", key, info); - return info; - } - StreamInfo get(ClientInvocationId key) { - final StreamInfo info = map.get(key); - LOG.debug("get({}) returns {}", key, info); - return info; - } - - StreamInfo remove(ClientInvocationId key) { - final StreamInfo info = map.remove(key); - LOG.debug("remove({}) returns {}", key, info); - return info; - } - } - - public static class ChannelMap { - private final Map> map = new ConcurrentHashMap<>(); - - public void add(ChannelId channelId, - ClientInvocationId clientInvocationId) { - map.computeIfAbsent(channelId, (e) -> new ConcurrentHashMap<>()).put(clientInvocationId, clientInvocationId); - } - - public void remove(ChannelId channelId, - ClientInvocationId clientInvocationId) { - Optional.ofNullable(map.get(channelId)).ifPresent((ids) -> ids.remove(clientInvocationId)); - } - - public Set remove(ChannelId channelId) { - return Optional.ofNullable(map.remove(channelId)) - .map(Map::keySet) - .orElse(Collections.emptySet()); + void cleanUp(ClientInvocationId invocationId) { + getDivision().getDataStreamMap().remove(invocationId); + getLocal().cleanUp(); + applyToRemotes(remote -> remote.out.closeAsync()); } } private final RaftServer server; private final String name; - private final StreamMap streams = new StreamMap(); + private final StreamMap streams = new StreamMap<>(); private final ChannelMap channels; private final ExecutorService requestExecutor; private final ExecutorService writeExecutor; + private final TimeDuration requestTimeout; private final NettyServerStreamRpcMetrics nettyServerStreamRpcMetrics; @@ -283,6 +243,7 @@ public Set remove(ChannelId channelId) { this.writeExecutor = ConcurrentUtils.newThreadPoolWithMax(useCachedThreadPool, RaftServerConfigKeys.DataStream.asyncWriteThreadPoolSize(properties), name + "-write-"); + this.requestTimeout = RaftClientConfigKeys.DataStream.requestTimeout(server.getProperties()); this.nettyServerStreamRpcMetrics = metrics; } @@ -315,12 +276,13 @@ private CompletableFuture computeDataStreamIfAbsent(RaftClientReques } private StreamInfo newStreamInfo(ByteBuf buf, - CheckedBiFunction, Set, IOException> getStreams) { + CheckedBiFunction, Set, IOException> getStreams) { try { final RaftClientRequest request = ClientProtoUtils.toRaftClientRequest( RaftClientRequestProto.parseFrom(buf.nioBuffer())); final boolean isPrimary = server.getId().equals(request.getServerId()); - return new StreamInfo(request, isPrimary, computeDataStreamIfAbsent(request), server, getStreams, + final Division division = server.getDivision(request.getRaftGroupId()); + return new StreamInfo(request, isPrimary, computeDataStreamIfAbsent(request), division, getStreams, getMetrics()::newRequestMetrics); } catch (Throwable e) { throw new CompletionException(e); @@ -345,6 +307,9 @@ static long writeTo(ByteBuf buf, Iterable options, final DataChannel channel = stream.getDataChannel(); long byteWritten = 0; for (ByteBuffer buffer : buf.nioBuffers()) { + if (buffer.remaining() == 0) { + continue; + } final ReferenceCountedObject wrapped = ReferenceCountedObject.wrap( buffer, buf::retain, ignored -> buf.release()); try(UncheckedAutoCloseable ignore = wrapped.retainAndReleaseOnClose()) { @@ -386,7 +351,7 @@ static DataStreamReplyByteBuffer newDataStreamReplyByteBuffer(DataStreamRequestB .build(); } - static void sendReply(List> remoteWrites, + private void sendReply(List> remoteWrites, DataStreamRequestByteBuf request, long bytesWritten, Collection commitInfos, ChannelHandlerContext ctx) { final boolean success = checkSuccessRemoteWrite(remoteWrites, bytesWritten, request); @@ -433,9 +398,7 @@ static void sendDataStreamException(Throwable throwable, DataStreamRequestByteBu void cleanUp(Set ids) { for (ClientInvocationId clientInvocationId : ids) { - Optional.ofNullable(streams.remove(clientInvocationId)) - .map(StreamInfo::getLocal) - .ifPresent(LocalStream::cleanUp); + removeDataStream(clientInvocationId); } } @@ -449,17 +412,26 @@ void cleanUpOnChannelInactive(ChannelId channelId, TimeDuration channelInactiveG } void read(DataStreamRequestByteBuf request, ChannelHandlerContext ctx, - CheckedBiFunction, Set, IOException> getStreams) { + CheckedBiFunction, Set, IOException> getStreams) { LOG.debug("{}: read {}", this, request); try { readImpl(request, ctx, getStreams); } catch (Throwable t) { replyDataStreamException(t, request, ctx); + removeDataStream(ClientInvocationId.valueOf(request.getClientId(), request.getStreamId())); + } + } + + private StreamInfo removeDataStream(ClientInvocationId invocationId) { + final StreamInfo removed = streams.remove(invocationId); + if (removed != null) { + removed.cleanUp(invocationId); } + return removed; } private void readImpl(DataStreamRequestByteBuf request, ChannelHandlerContext ctx, - CheckedBiFunction, Set, IOException> getStreams) { + CheckedBiFunction, Set, IOException> getStreams) { final boolean close = request.getWriteOptionList().contains(StandardWriteOption.CLOSE); ClientInvocationId key = ClientInvocationId.valueOf(request.getClientId(), request.getStreamId()); @@ -473,8 +445,6 @@ private void readImpl(DataStreamRequestByteBuf request, ChannelHandlerContext ct () -> newStreamInfo(request.slice(), getStreams)); info = streams.computeIfAbsent(key, id -> supplier.get()); if (!supplier.isInitialized()) { - final StreamInfo removed = streams.remove(key); - removed.getLocal().cleanUp(); throw new IllegalStateException("Failed to create a new stream for " + request + " since a stream already exists Key: " + key + " StreamInfo:" + info); } @@ -512,9 +482,15 @@ private void readImpl(DataStreamRequestByteBuf request, ChannelHandlerContext ct }, requestExecutor)).whenComplete((v, exception) -> { try { if (exception != null) { - final StreamInfo removed = streams.remove(key); replyDataStreamException(server, exception, info.getRequest(), request, ctx); - removed.getLocal().cleanUp(); + final StreamInfo removed = removeDataStream(key); + if (removed != null) { + Preconditions.assertSame(info, removed, "removed"); + } else { + info.cleanUp(key); + } + } else if (close) { + info.applyToRemotes(remote -> remote.out.closeAsync()); } } finally { request.release(); @@ -531,10 +507,15 @@ static void assertReplyCorrespondingToRequest( Preconditions.assertTrue(request.getStreamOffset() == reply.getStreamOffset()); } - static boolean checkSuccessRemoteWrite(List> replyFutures, long bytesWritten, + private boolean checkSuccessRemoteWrite(List> replyFutures, long bytesWritten, final DataStreamRequestByteBuf request) { for (CompletableFuture replyFuture : replyFutures) { - final DataStreamReply reply = replyFuture.join(); + final DataStreamReply reply; + try { + reply = replyFuture.get(requestTimeout.getDuration(), requestTimeout.getUnit()); + } catch (Exception e) { + throw new CompletionException("Failed to get reply for bytesWritten=" + bytesWritten + ", " + request, e); + } assertReplyCorrespondingToRequest(request, reply); if (!reply.isSuccess()) { LOG.warn("reply is not success, request: {}", request); diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java b/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java index c5f24b0587..451040bb62 100644 --- a/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/server/NettyServerStreamRpc.java @@ -20,9 +20,11 @@ import org.apache.ratis.client.DataStreamClient; import org.apache.ratis.client.DataStreamOutputRpc; +import org.apache.ratis.client.impl.DataStreamClientImpl.DataStreamOutputImpl; import org.apache.ratis.conf.Parameters; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.datastream.impl.DataStreamReplyByteBuffer; +import org.apache.ratis.datastream.impl.DataStreamRequestByteBuf; import org.apache.ratis.netty.NettyConfigKeys; import org.apache.ratis.netty.NettyDataStreamUtils; import org.apache.ratis.netty.NettyUtils; @@ -90,8 +92,8 @@ void addPeers(Collection newPeers) { map.addRaftPeers(newPeers); } - Set getDataStreamOutput(RaftClientRequest request, Set peers) throws IOException { - final Set outs = new HashSet<>(); + Set getDataStreamOutput(RaftClientRequest request, Set peers) throws IOException { + final Set outs = new HashSet<>(); try { getDataStreamOutput(request, peers, outs); } catch (IOException e) { @@ -101,11 +103,11 @@ Set getDataStreamOutput(RaftClientRequest request, Set peers, Set outs) + private void getDataStreamOutput(RaftClientRequest request, Set peers, Set outs) throws IOException { for (RaftPeer peer : peers) { try { - outs.add((DataStreamOutputRpc) map.computeIfAbsent(peer).get().stream(request)); + outs.add((DataStreamOutputImpl) map.computeIfAbsent(peer).get().stream(request)); } catch (IOException e) { map.handleException(peer.getId(), e, true); throw new IOException(map.getName() + ": Failed to getDataStreamOutput for " + peer, e); @@ -238,7 +240,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { } @Override - public void channelInactive(ChannelHandlerContext ctx) throws Exception { + public void channelInactive(ChannelHandlerContext ctx) { requests.cleanUpOnChannelInactive(ctx.channel().id(), channelInactiveGracePeriod); } diff --git a/ratis-netty/src/main/java/org/apache/ratis/netty/server/StreamMap.java b/ratis-netty/src/main/java/org/apache/ratis/netty/server/StreamMap.java new file mode 100644 index 0000000000..073698cb80 --- /dev/null +++ b/ratis-netty/src/main/java/org/apache/ratis/netty/server/StreamMap.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.netty.server; + +import org.apache.ratis.protocol.ClientInvocationId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; + +/** + * Map: {@link ClientInvocationId} -> {@link STREAM}. + * + * @param the stream type. + */ +class StreamMap { + public static final Logger LOG = LoggerFactory.getLogger(StreamMap.class); + + private final ConcurrentMap map = new ConcurrentHashMap<>(); + + STREAM computeIfAbsent(ClientInvocationId key, Function function) { + final STREAM info = map.computeIfAbsent(key, function); + LOG.debug("computeIfAbsent({}) returns {}", key, info); + return info; + } + + STREAM get(ClientInvocationId key) { + final STREAM info = map.get(key); + LOG.debug("get({}) returns {}", key, info); + return info; + } + + STREAM remove(ClientInvocationId key) { + final STREAM info = map.remove(key); + LOG.debug("remove({}) returns {}", key, info); + return info; + } +} diff --git a/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java b/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java index 990b63d90d..b912e14927 100644 --- a/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java +++ b/ratis-netty/src/test/java/org/apache/ratis/netty/MiniRaftClusterWithNetty.java @@ -48,7 +48,7 @@ default Factory getFactory() { } } - public static final DelayLocalExecutionInjection sendServerRequest + public static final DelayLocalExecutionInjection SEND_SERVER_REQUEST = new DelayLocalExecutionInjection(NettyRpcService.SEND_SERVER_REQUEST); protected MiniRaftClusterWithNetty(String[] ids, String[] listenerIds, RaftProperties properties) { @@ -64,7 +64,7 @@ protected Parameters setPropertiesAndInitParameters(RaftPeerId id, RaftGroup gro @Override protected void blockQueueAndSetDelay(String leaderId, int delayMs) throws InterruptedException { - RaftTestUtil.blockQueueAndSetDelay(getServers(), sendServerRequest, + RaftTestUtil.blockQueueAndSetDelay(getServers(), SEND_SERVER_REQUEST, leaderId, delayMs, getTimeoutMax()); } } diff --git a/ratis-proto/pom.xml b/ratis-proto/pom.xml index e0cf2c8587..e3342f84ba 100644 --- a/ratis-proto/pom.xml +++ b/ratis-proto/pom.xml @@ -17,7 +17,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-proto @@ -25,6 +25,8 @@ true + + true true diff --git a/ratis-proto/src/main/proto/Raft.proto b/ratis-proto/src/main/proto/Raft.proto index 586ec1b052..6dbfdb15a5 100644 --- a/ratis-proto/src/main/proto/Raft.proto +++ b/ratis-proto/src/main/proto/Raft.proto @@ -155,6 +155,7 @@ enum InstallSnapshotResult { CONF_MISMATCH = 4; SNAPSHOT_INSTALLED = 5; SNAPSHOT_UNAVAILABLE = 6; + SNAPSHOT_EXPIRED = 7; } message RequestVoteRequestProto { @@ -168,6 +169,7 @@ message RequestVoteReplyProto { RaftRpcReplyProto serverReply = 1; uint64 term = 2; bool shouldShutdown = 3; + TermIndexProto lastEntry = 4; // to determine if the voter log is empty. } message CommitInfoProto { @@ -470,7 +472,7 @@ message SnapshotManagementRequestProto { } message SnapshotCreateRequestProto { - + uint64 creationGap = 1; } message StartLeaderElectionRequestProto { @@ -556,4 +558,14 @@ message GroupInfoReplyProto { bool isRaftStorageHealthy = 4; repeated CommitInfoProto commitInfos = 5; RaftConfigurationProto conf = 6; + LogInfoProto logInfo = 7; +} + +/** Add new LogInfoProto for RATIS-2030, allow GroupInfoCommand to show each server's last committed log, + last applied log, last snapshot log, last entry log.*/ +message LogInfoProto { + TermIndexProto lastSnapshot = 1; + TermIndexProto applied = 2; + TermIndexProto committed = 3; + TermIndexProto lastEntry = 4; } diff --git a/ratis-replicated-map/pom.xml b/ratis-replicated-map/pom.xml index 5b4b8f21c1..e6144fd103 100644 --- a/ratis-replicated-map/pom.xml +++ b/ratis-replicated-map/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-replicated-map Apache Ratis Replicated Map + + + true + + org.apache.ratis diff --git a/ratis-resource-bundle/pom.xml b/ratis-resource-bundle/pom.xml index a879a74238..b0fd2b2208 100644 --- a/ratis-resource-bundle/pom.xml +++ b/ratis-resource-bundle/pom.xml @@ -23,7 +23,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT .. diff --git a/ratis-resource-bundle/src/main/resources/META-INF/LICENSE.vm b/ratis-resource-bundle/src/main/resources/META-INF/LICENSE.vm index 3d9f76a119..874d716c8b 100644 --- a/ratis-resource-bundle/src/main/resources/META-INF/LICENSE.vm +++ b/ratis-resource-bundle/src/main/resources/META-INF/LICENSE.vm @@ -221,10 +221,10 @@ under the License. limitations under the License. ## Special cases, for e.g. ASL2.0 licensed works that bundle additional third party works -#set($bundled-jquery = ${bundled-jquery.equalsIgnoreCase("true")}) -#set($bundled-logo = ${bundled-logo.equalsIgnoreCase("true")}) -#set($bundled-dependencies = ${bundled-dependencies.equalsIgnoreCase("true")}) -#if($bundled-jquery || $bundled-logo || $bundled-dependencies) +#set($bundled_jquery = ${bundled_jquery.equalsIgnoreCase("true")}) +#set($bundled_logo = ${bundled_logo.equalsIgnoreCase("true")}) +#set($bundled_dependencies = ${bundled_dependencies.equalsIgnoreCase("true")}) +#if($bundled_jquery || $bundled_logo || $bundled_dependencies) ==== ${project.name} contained works @@ -884,12 +884,12 @@ facade for Java, which can be obtained at: * HOMEPAGE: * http://www.slf4j.org/ #end -## Supplemental from commons-math +## Supplemental from commons_math #macro(commons_math_license) ---- APACHE COMMONS MATH DERIVATIVE WORKS: -The Apache commons-math library includes a number of subcomponents +The Apache commons_math library includes a number of subcomponents whose implementation is derived from original sources written in C or Fortran. License terms of the original sources are reproduced below. @@ -1178,7 +1178,7 @@ For the org.apache.hadoop.util.bloom.* classes: ## skip jquery ## skip backbone ## relocated jackson 2.4.0 is ASLv2 with no notice -## relocated commons-logging 1.1.1 is in NOTICE.vm +## relocated commons_logging 1.1.1 is in NOTICE.vm #end #macro (thrift_license) ## Thrift supplemental for libthrift is a no-op. @@ -1295,15 +1295,15 @@ You can redistribute it and/or modify it under either the terms of the PURPOSE. #end ## modules with bundled works in source -#if(${bundled-jquery}) +#if(${bundled_jquery}) #jquery_license() #end -#if(${bundled-logo}) +#if(${bundled_logo}) #orca_logo_license() #end ## when true, we're in a module that makes a binary dist with ## bundled works. -#if(${bundled-dependencies}) +#if(${bundled_dependencies}) ==== ## gather up CDDL licensed works #set($cddl_1_0 = []) @@ -1316,8 +1316,8 @@ You can redistribute it and/or modify it under either the terms of the #set($cpl = []) ## gather up EPL 1.0 works #set($epl = []) -## track commons-math -#set($commons-math = false) +## track commons_math +#set($commons_math = false) ## track if we need jruby additionals. #set($jruby = false) ## track hadoops @@ -1342,7 +1342,7 @@ g:${dep.groupId} AND a:${dep.artifactId} AND v:${dep.version} Until ratis-resource-bundle/src/main/resources/supplemental-models.xml is updated, the build should fail. #end -#if(${debug-print-included-work-info.equalsIgnoreCase("true")}) +#if(${debug_print_included_work_info.equalsIgnoreCase("true")}) ===== Check license for included work @@ -1364,8 +1364,8 @@ ${dep.scm.url} #end #set($aggregated=false) ## Check for our set of known dependencies that require manual LICENSE additions. -#if($dep.artifactId.equals("commons-math3")) -#set($commons-math=true) +#if($dep.artifactId.equals("commons_math3")) +#set($commons_math=true) #end #if($dep.artifactId.equals("jruby-complete")) #set($jruby=true) @@ -1465,7 +1465,7 @@ ${dep.scm.url} #if($hadoop) #hadoop_license() #end -#if($commons-math) +#if($commons_math) #commons_math_license() #end #if(!(${mit.isEmpty()})) diff --git a/ratis-resource-bundle/src/main/resources/META-INF/NOTICE.vm b/ratis-resource-bundle/src/main/resources/META-INF/NOTICE.vm index f3fca903b3..7d473b60ac 100644 --- a/ratis-resource-bundle/src/main/resources/META-INF/NOTICE.vm +++ b/ratis-resource-bundle/src/main/resources/META-INF/NOTICE.vm @@ -23,10 +23,10 @@ Copyright 2017-2019 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). ## Specially handled included deps. e.g. ASL2.0 works that have additional bundled works but -#set($bundled-boostrap = ${bundled-boostrap.equalsIgnoreCase("true")}) -#set($bundled-logo = ${bundled-logo.equalsIgnoreCase("true")}) -#set($bundled-dependencies = ${bundled-dependencies.equalsIgnoreCase("true")}) -#if($bundled-bootstrap || $bundled-logo || $bundled-dependencies) +#set($bundled_boostrap = ${bundled_boostrap.equalsIgnoreCase("true")}) +#set($bundled_logo = ${bundled_logo.equalsIgnoreCase("true")}) +#set($bundled_dependencies = ${bundled_dependencies.equalsIgnoreCase("true")}) +#if($bundled_bootstrap || $bundled_logo || $bundled_dependencies) ==== ${project.name} contained works @@ -102,13 +102,13 @@ noted in the LICENSE file. #end -#macro(jetty_notice $jetty-include-unixcrypt) +#macro(jetty_notice $jetty_include_unixcrypt) -- This product includes portions of 'The Jetty Web Container' Copyright 1995-2016 Mort Bay Consulting Pty Ltd. -#if(${jetty-include-unixcrypt}) +#if(${jetty_include_unixcrypt}) ## UnixCrypt.java paragraph only in server The UnixCrypt.java code ~Implements the one way cryptography used by Unix systems for simple password protection. Copyright 1996 Aki Yoshida, @@ -207,13 +207,13 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) #end ## first bundled source -#if(${bundled-logo}) +#if(${bundled_logo}) #orca_logo_notice() #end -#if(${bundled-bootstrap}) +#if(${bundled_bootstrap}) #bootstrap_notice() #end -#if(${bundled-dependencies}) +#if(${bundled_dependencies}) #** Note that this will fail the build if we don't have a license. update supplemental-models via setting '-Dlicense.debug.print.included' and looking in the generated LICENSE file @@ -229,7 +229,7 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) #set($mpl_1_1=[]) ## track jettys #set($jetty=false) -#set($jetty-with-crypt=false) +#set($jetty_with_crypt=false) ## track jruby #set($jruby=false) #foreach( ${dep} in ${projects} ) @@ -244,7 +244,7 @@ under the Apache License 2.0 (see: StringUtils.containsWhitespace()) #if(${dep.artifactId.startsWith("jetty")}) #set($jetty=true) #if(${dep.artifactId.equals("jetty")}) -#set($jetty-with-crypt=true) +#set($jetty_with_crypt=true) #end #end #if(${dep.artifactId.equals("log4j")}) @@ -317,7 +317,7 @@ For source see '${dep.url}'. #end ## Print out jetty #if(${jetty}) -#jetty_notice(${jetty-with-crypt}) +#jetty_notice(${jetty_with_crypt}) #end ## Now go through all the lists of Category-B licensed works and make sure we ## name them and give a URL for the project's home page. diff --git a/ratis-server-api/pom.xml b/ratis-server-api/pom.xml index 52f5eba361..e486d03225 100644 --- a/ratis-server-api/pom.xml +++ b/ratis-server-api/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-server-api Apache Ratis Server API + + + true + + org.apache.ratis @@ -47,12 +52,5 @@ org.slf4j slf4j-api - - - com.github.spotbugs - spotbugs-annotations - provided - true - diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java b/ratis-server-api/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java index 4e948c6f81..6316ef6073 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/DataStreamServerRpc.java @@ -20,18 +20,10 @@ import org.apache.ratis.protocol.RaftPeer; import java.io.Closeable; -import java.net.InetSocketAddress; /** * A server interface handling incoming streams * Relays those streams to other servers after persisting */ -public interface DataStreamServerRpc extends RaftPeer.Add, Closeable { - /** - * start server - */ - void start(); - - /** @return the address where this RPC server is listening to. */ - InetSocketAddress getInetSocketAddress(); +public interface DataStreamServerRpc extends ServerRpc, RaftPeer.Add, Closeable { } diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionInfo.java b/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionInfo.java index c07317bd69..78073a21c5 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionInfo.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/DivisionInfo.java @@ -80,4 +80,10 @@ default boolean isAlive() { * otherwise, return null. */ long[] getFollowerNextIndices(); + + /** + * @return an array of match indices of the followers if this server division is the leader; + * otherwise, return null. + */ + long[] getFollowerMatchIndices(); } diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java index 565e881269..849597433a 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerConfigKeys.java @@ -77,9 +77,14 @@ static TimeDuration sleepDeviationThreshold(RaftProperties properties) { return getTimeDuration(properties.getTimeDuration(SLEEP_DEVIATION_THRESHOLD_DEFAULT.getUnit()), SLEEP_DEVIATION_THRESHOLD_KEY, SLEEP_DEVIATION_THRESHOLD_DEFAULT, getDefaultLog()); } + /** @deprecated use {@link #setSleepDeviationThreshold(RaftProperties, TimeDuration)}. */ + @Deprecated static void setSleepDeviationThreshold(RaftProperties properties, int thresholdMs) { setInt(properties::setInt, SLEEP_DEVIATION_THRESHOLD_KEY, thresholdMs); } + static void setSleepDeviationThreshold(RaftProperties properties, TimeDuration threshold) { + setTimeDuration(properties::setTimeDuration, SLEEP_DEVIATION_THRESHOLD_KEY, threshold); + } String CLOSE_THRESHOLD_KEY = PREFIX + ".close.threshold"; TimeDuration CLOSE_THRESHOLD_DEFAULT = TimeDuration.valueOf(60, TimeUnit.SECONDS); @@ -87,8 +92,13 @@ static TimeDuration closeThreshold(RaftProperties properties) { return getTimeDuration(properties.getTimeDuration(CLOSE_THRESHOLD_DEFAULT.getUnit()), CLOSE_THRESHOLD_KEY, CLOSE_THRESHOLD_DEFAULT, getDefaultLog()); } - static void setCloseThreshold(RaftProperties properties, int thresholdMs) { - setInt(properties::setInt, CLOSE_THRESHOLD_KEY, thresholdMs); + /** @deprecated use {@link #setCloseThreshold(RaftProperties, TimeDuration)}. */ + @Deprecated + static void setCloseThreshold(RaftProperties properties, int thresholdSec) { + setInt(properties::setInt, CLOSE_THRESHOLD_KEY, thresholdSec); + } + static void setCloseThreshold(RaftProperties properties, TimeDuration threshold) { + setTimeDuration(properties::setTimeDuration, CLOSE_THRESHOLD_KEY, threshold); } /** @@ -106,6 +116,21 @@ static void setStagingCatchupGap(RaftProperties properties, int stagingCatchupGa setInt(properties::setInt, STAGING_CATCHUP_GAP_KEY, stagingCatchupGap); } + String STAGING_TIMEOUT_KEY = PREFIX + ".staging.timeout"; + + TimeDuration STAGING_TIMEOUT_DEFAULT = null; + + static TimeDuration stagingTimeout(RaftProperties properties) { + final TimeDuration fallbackStagingTimeout = Rpc.timeoutMax(properties, null).multiply(3); + return getTimeDuration(properties.getTimeDuration(fallbackStagingTimeout.getUnit()), + STAGING_TIMEOUT_KEY, STAGING_TIMEOUT_DEFAULT, + Rpc.TIMEOUT_MAX_KEY, fallbackStagingTimeout, getDefaultLog()); + } + static void setStagingTimeout(RaftProperties properties, TimeDuration stagingTimeout) { + setTimeDuration(properties::setTimeDuration, STAGING_TIMEOUT_KEY, stagingTimeout); + } + + interface ThreadPool { String PREFIX = RaftServerConfigKeys.PREFIX + ".threadpool"; @@ -465,6 +490,18 @@ static void setAsyncFlushEnabled(RaftProperties properties, boolean asyncFlush) setBoolean(properties::setBoolean, ASYNC_FLUSH_ENABLED_KEY, asyncFlush); } + /** Log metadata can guarantee that a server can recover commit index and state machine + * even if a majority of servers are dead by consuming a little extra space. */ + String LOG_METADATA_ENABLED_KEY = PREFIX + ".log-metadata.enabled"; + boolean LOG_METADATA_ENABLED_DEFAULT = true; + static boolean logMetadataEnabled(RaftProperties properties) { + return getBoolean(properties::getBoolean, + LOG_METADATA_ENABLED_KEY, LOG_METADATA_ENABLED_DEFAULT, getDefaultLog()); + } + static void setLogMetadataEnabled(RaftProperties properties, boolean logMetadata) { + setBoolean(properties::setBoolean, LOG_METADATA_ENABLED_KEY, logMetadata); + } + /** The policy to handle corrupted raft log. */ enum CorruptionPolicy { /** Rethrow the exception. */ @@ -647,6 +684,18 @@ static void setTriggerWhenStopEnabled(RaftProperties properties, boolean trigger setBoolean(properties::setBoolean, TRIGGER_WHEN_STOP_ENABLED_KEY, triggerWhenStopEnabled); } + /** whether trigger snapshot when remove raft server */ + String TRIGGER_WHEN_REMOVE_ENABLED_KEY = PREFIX + ".trigger-when-remove.enabled"; + /** by default let the state machine to trigger snapshot when remove */ + boolean TRIGGER_WHEN_REMOVE_ENABLED_DEFAULT = true; + static boolean triggerWhenRemoveEnabled(RaftProperties properties) { + return getBoolean(properties::getBoolean, + TRIGGER_WHEN_REMOVE_ENABLED_KEY, TRIGGER_WHEN_REMOVE_ENABLED_DEFAULT, getDefaultLog()); + } + static void setTriggerWhenRemoveEnabled(RaftProperties properties, boolean triggerWhenRemoveEnabled) { + setBoolean(properties::setBoolean, TRIGGER_WHEN_REMOVE_ENABLED_KEY, triggerWhenRemoveEnabled); + } + /** The log index gap between to two snapshot creations. */ String CREATION_GAP_KEY = PREFIX + ".creation.gap"; long CREATION_GAP_DEFAULT = 1024; diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerRpc.java b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerRpc.java index d81f9cc8bc..76bd817f5e 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerRpc.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/RaftServerRpc.java @@ -26,20 +26,13 @@ import org.apache.ratis.util.JavaUtils; import java.io.Closeable; -import java.io.IOException; import java.net.InetSocketAddress; /** * An server-side interface for supporting different RPC implementations * such as Netty, gRPC and Hadoop. */ -public interface RaftServerRpc extends RaftServerProtocol, RpcType.Get, RaftPeer.Add, Closeable { - /** Start the RPC service. */ - void start() throws IOException; - - /** @return the address where this RPC server is listening */ - InetSocketAddress getInetSocketAddress(); - +public interface RaftServerRpc extends RaftServerProtocol, ServerRpc, RpcType.Get, RaftPeer.Add, Closeable { /** @return the address where this RPC server is listening for client requests */ default InetSocketAddress getClientServerAddress() { return getInetSocketAddress(); diff --git a/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerCallListener.java b/ratis-server-api/src/main/java/org/apache/ratis/server/ServerRpc.java similarity index 51% rename from ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerCallListener.java rename to ratis-server-api/src/main/java/org/apache/ratis/server/ServerRpc.java index 77ae4c8f35..6ad5eacf14 100644 --- a/ratis-grpc/src/main/java/org/apache/ratis/grpc/metrics/intercept/server/MetricServerCallListener.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/ServerRpc.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,29 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.ratis.server; -package org.apache.ratis.grpc.metrics.intercept.server; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.apache.ratis.thirdparty.io.grpc.ForwardingServerCallListener; -import org.apache.ratis.thirdparty.io.grpc.ServerCall; -import org.apache.ratis.grpc.metrics.MessageMetrics; - -@SuppressFBWarnings("URF_UNREAD_FIELD") -public class MetricServerCallListener extends ForwardingServerCallListener { - private final ServerCall.Listener delegate; - private MessageMetrics metrics; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; - MetricServerCallListener( - ServerCall.Listener delegate, - MessageMetrics metrics - ){ - this.delegate = delegate; - this.metrics = metrics; - } +/** + * A general server interface. + */ +public interface ServerRpc extends Closeable { + /** Start the RPC service. */ + void start() throws IOException; - @Override - protected ServerCall.Listener delegate() { - return delegate; - } + /** @return the address where this RPC server is listening to. */ + InetSocketAddress getInetSocketAddress(); } diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java b/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java index 020a352c05..a333b8393a 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/leader/LogAppender.java @@ -87,10 +87,11 @@ default void stop() { default CompletableFuture stopAsync() { stop(); return CompletableFuture.supplyAsync(() -> { - for (; isRunning(); ) { + while (isRunning()) { try { Thread.sleep(10); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new CompletionException("stopAsync interrupted", e); } } @@ -124,7 +125,9 @@ default RaftPeerId getFollowerId() { * @param heartbeat the returned request must be a heartbeat. * * @return a new {@link AppendEntriesRequestProto} object. + * @deprecated this is no longer a public API. */ + @Deprecated AppendEntriesRequestProto newAppendEntriesRequest(long callId, boolean heartbeat) throws RaftLogIOException; /** @return a new {@link InstallSnapshotRequestProto} object. */ @@ -142,7 +145,7 @@ default SnapshotInfo shouldInstallSnapshot() { // we should install snapshot if the follower needs to catch up and: // 1. there is no local log entry but there is snapshot // 2. or the follower's next index is smaller than the log start index - // 3. or the follower is bootstrapping and has not installed any snapshot yet + // 3. or the follower is bootstrapping (i.e. not yet caught up) and has not installed any snapshot yet final FollowerInfo follower = getFollower(); final boolean isFollowerBootstrapping = getLeaderState().isFollowerBootstrapping(follower); final SnapshotInfo snapshot = getServer().getStateMachine().getLatestSnapshot(); diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/RaftServerAsynchronousProtocol.java b/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/RaftServerAsynchronousProtocol.java index 8a904069ba..035e0a815f 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/RaftServerAsynchronousProtocol.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/RaftServerAsynchronousProtocol.java @@ -22,14 +22,39 @@ import org.apache.ratis.proto.RaftProtos.ReadIndexReplyProto; import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto; import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto; +import org.apache.ratis.util.ReferenceCountedObject; import java.io.IOException; import java.util.concurrent.CompletableFuture; public interface RaftServerAsynchronousProtocol { - CompletableFuture appendEntriesAsync(AppendEntriesRequestProto request) - throws IOException; + /** + * It is recommended to override {@link #appendEntriesAsync(ReferenceCountedObject)} instead. + * Then, it does not have to override this method. + */ + default CompletableFuture appendEntriesAsync(AppendEntriesRequestProto request) + throws IOException { + throw new UnsupportedOperationException(); + } + + /** + * A referenced counted request is submitted from a client for processing. + * Implementations of this method should retain the request, process it and then release it. + * The request may be retained even after the future returned by this method has completed. + * + * @return a future of the reply + * @see ReferenceCountedObject + */ + default CompletableFuture appendEntriesAsync( + ReferenceCountedObject requestRef) throws IOException { + // Default implementation for backward compatibility. + try { + return appendEntriesAsync(requestRef.retain()); + } finally { + requestRef.release(); + } + } CompletableFuture readIndexAsync(ReadIndexRequestProto request) throws IOException; diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/TermIndex.java b/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/TermIndex.java index 7def686bb5..369aefc85f 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/TermIndex.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/protocol/TermIndex.java @@ -19,12 +19,29 @@ import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.proto.RaftProtos.TermIndexProto; +import org.apache.ratis.server.raftlog.RaftLog; +import org.apache.ratis.util.BiWeakValueCache; +import org.apache.ratis.util.MemoizedSupplier; import java.util.Comparator; import java.util.Optional; +import java.util.function.Supplier; /** The term and the log index defined in the Raft consensus algorithm. */ public interface TermIndex extends Comparable { + /** + * The initial value. + * When a new Raft group starts, + * all the servers has term 0 and index -1 (= {@link RaftLog#INVALID_LOG_INDEX}). + * Note that term is incremented during leader election + * and index is incremented when writing to the {@link RaftLog}. + * The least term and index possibly written to the {@link RaftLog} + * are respectively 1 and 0 (= {@link RaftLog#LEAST_VALID_LOG_INDEX}). + */ + TermIndex INITIAL_VALUE = valueOf(0, RaftLog.INVALID_LOG_INDEX); + TermIndex PROTO_DEFAULT = valueOf(TermIndexProto.getDefaultInstance()); + + /** An empty {@link TermIndex} array. */ TermIndex[] EMPTY_ARRAY = {}; /** @return the term. */ @@ -60,43 +77,74 @@ static TermIndex valueOf(LogEntryProto proto) { /** @return a {@link TermIndex} object. */ static TermIndex valueOf(long term, long index) { - return new TermIndex() { - @Override - public long getTerm() { - return term; - } - - @Override - public long getIndex() { - return index; - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } else if (!(obj instanceof TermIndex)) { - return false; + return Impl.getCache().getOrCreate(term, index); + } + + /** + * An implementation for private use. + * Note that this is not a public API, although this is public class. + */ + final class Impl { + private Impl() { } + + private static final BiWeakValueCache CACHE + = new BiWeakValueCache<>("term", "index", Impl::newTermIndex); + + static BiWeakValueCache getCache() { + return CACHE; + } + + private static TermIndex newTermIndex(long term, long index) { + return new TermIndex() { + private final Supplier protoSupplier = MemoizedSupplier.valueOf(TermIndex.super::toProto); + + @Override + public long getTerm() { + return term; + } + + @Override + public long getIndex() { + return index; } - final TermIndex that = (TermIndex) obj; - return this.getTerm() == that.getTerm() - && this.getIndex() == that.getIndex(); - } - - @Override - public int hashCode() { - return Long.hashCode(term) ^ Long.hashCode(index); - } - - private String longToString(long n) { - return n >= 0L? String.valueOf(n) : "~"; - } - - @Override - public String toString() { - return String.format("(t:%s, i:%s)", longToString(term), longToString(index)); - } - }; + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } else if (!(obj instanceof TermIndex)) { + return false; + } + + final TermIndex that = (TermIndex) obj; + return this.getTerm() == that.getTerm() + && this.getIndex() == that.getIndex(); + } + + @Override + public int hashCode() { + return Long.hashCode(term) ^ Long.hashCode(index); + } + + @Override + public TermIndexProto toProto() { + return protoSupplier.get(); + } + + private String longToString(long n) { + return n >= 0L ? String.valueOf(n) : "~"; + } + + @Override + public String toString() { + if (this.equals(INITIAL_VALUE)) { + return ""; + } else if (this.equals(PROTO_DEFAULT)) { + return ""; + } + return String.format("(t:%s, i:%s)", longToString(term), longToString(index)); + } + }; + } } } \ No newline at end of file diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLog.java b/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLog.java index e504462b80..ca785a4a6d 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLog.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLog.java @@ -21,6 +21,7 @@ import org.apache.ratis.server.metrics.RaftLogMetrics; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.storage.RaftStorageMetadata; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,16 +58,44 @@ default boolean contains(TermIndex ti) { /** * @return null if the log entry is not found in this log; - * otherwise, return the log entry corresponding to the given index. + * otherwise, return a copy of the log entry corresponding to the given index. + * @deprecated use {@link RaftLog#retainLog(long)} instead in order to avoid copying. */ + @Deprecated LogEntryProto get(long index) throws RaftLogIOException; + /** + * @return a retained {@link ReferenceCountedObject} to the log entry corresponding to the given index if it exists; + * otherwise, return null. + * Since the returned reference is retained, the caller must call {@link ReferenceCountedObject#release()}} + * after use. + */ + default ReferenceCountedObject retainLog(long index) throws RaftLogIOException { + ReferenceCountedObject wrap = ReferenceCountedObject.wrap(get(index)); + wrap.retain(); + return wrap; + } + /** * @return null if the log entry is not found in this log; * otherwise, return the {@link EntryWithData} corresponding to the given index. + * @deprecated use {@link #retainEntryWithData(long)}. */ + @Deprecated EntryWithData getEntryWithData(long index) throws RaftLogIOException; + /** + * @return null if the log entry is not found in this log; + * otherwise, return a retained reference of the {@link EntryWithData} corresponding to the given index. + * Since the returned reference is retained, the caller must call {@link ReferenceCountedObject#release()}} + * after use. + */ + default ReferenceCountedObject retainEntryWithData(long index) throws RaftLogIOException { + final ReferenceCountedObject wrap = ReferenceCountedObject.wrap(getEntryWithData(index)); + wrap.retain(); + return wrap; +} + /** * @param startIndex the starting log index (inclusive) * @param endIndex the ending log index (exclusive) @@ -157,6 +186,15 @@ default long getNextIndex() { * containing both the log entry and the state machine data. */ interface EntryWithData { + /** @return the index of this entry. */ + default long getIndex() { + try { + return getEntry(TimeDuration.ONE_MINUTE).getIndex(); + } catch (Exception e) { + throw new IllegalStateException("Failed to getIndex", e); + } + } + /** @return the serialized size including both log entry and state machine data. */ int getSerializedSize(); diff --git a/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLogSequentialOps.java b/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLogSequentialOps.java index 7b9f42b6bd..5a25728830 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLogSequentialOps.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/server/raftlog/RaftLogSequentialOps.java @@ -22,8 +22,10 @@ import org.apache.ratis.server.RaftConfiguration; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.StringUtils; import org.apache.ratis.util.function.CheckedSupplier; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import java.util.Arrays; import java.util.List; @@ -77,8 +79,10 @@ OUTPUT runSequentially( // The current thread is already the runner. return operation.get(); } else { + final Throwable cause = new Throwable("The thread already running: " + previous); + cause.setStackTrace(previous.getStackTrace()); throw new IllegalStateException( - name + ": Already running a method by " + previous + ", current=" + current); + name + ": Already running a method by " + previous + ", current=" + current, cause); } } } @@ -122,31 +126,56 @@ OUTPUT runSequentially( CompletableFuture appendEntry(LogEntryProto entry); /** - * Append asynchronously an entry. - * Used by the leader. + * @deprecated use {@link #appendEntry(ReferenceCountedObject, TransactionContext)}}. */ + @Deprecated default CompletableFuture appendEntry(LogEntryProto entry, TransactionContext context) { - return appendEntry(entry); + throw new UnsupportedOperationException(); + } + + /** + * Append asynchronously an entry. + * Used for scenarios that there is a ReferenceCountedObject context for resource cleanup when the given entry + * is no longer used/referenced by this log. + */ + default CompletableFuture appendEntry(ReferenceCountedObject entryRef, + TransactionContext context) { + return appendEntry(entryRef.get(), context); } /** * The same as append(Arrays.asList(entries)). * - * @deprecated use {@link #append(List)} + * @deprecated use {@link #append(ReferenceCountedObject)}. */ @Deprecated default List> append(LogEntryProto... entries) { return append(Arrays.asList(entries)); } + /** + * @deprecated use {@link #append(ReferenceCountedObject)}. + */ + @Deprecated + default List> append(List entries) { + throw new UnsupportedOperationException(); + } + /** * Append asynchronously all the given log entries. * Used by the followers. * * If an existing entry conflicts with a new one (same index but different terms), * delete the existing entry and all entries that follow it (§5.3). + * + * A reference counter is also submitted. + * For each entry, implementations of this method should retain the counter, process it and then release. */ - List> append(List entries); + default List> append(ReferenceCountedObject> entriesRef) { + try(UncheckedAutoCloseableSupplier> entries = entriesRef.retainAndReleaseOnClose()) { + return append(entries.get()); + } + } /** * Truncate asynchronously the log entries till the given index (inclusively). diff --git a/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java b/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java index b1fc5addae..3960ab8287 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/statemachine/StateMachine.java @@ -88,11 +88,35 @@ default CompletableFuture read(LogEntryProto entry, TransactionConte return read(entry); } + /** + * Read asynchronously the state machine data from this state machine. + * StateMachines implement this method when the read result contains retained resources that should be released + * after use. + * + * @return a future for the read task. The result of the future is a {@link ReferenceCountedObject} wrapping the + * read result. Client code of this method must call {@link ReferenceCountedObject#release()} after + * use. + */ + default CompletableFuture> retainRead(LogEntryProto entry, + TransactionContext context) { + return read(entry, context).thenApply(r -> { + if (r == null) { + return null; + } + ReferenceCountedObject ref = ReferenceCountedObject.wrap(r); + ref.retain(); + return ref; + + }); + } + /** * Write asynchronously the state machine data in the given log entry to this state machine. * * @return a future for the write task + * @deprecated Applications should implement {@link #write(ReferenceCountedObject, TransactionContext)} instead. */ + @Deprecated default CompletableFuture write(LogEntryProto entry) { return CompletableFuture.completedFuture(null); } @@ -101,11 +125,36 @@ default CompletableFuture write(LogEntryProto entry) { * Write asynchronously the state machine data in the given log entry to this state machine. * * @return a future for the write task + * @deprecated Applications should implement {@link #write(ReferenceCountedObject, TransactionContext)} instead. */ + @Deprecated default CompletableFuture write(LogEntryProto entry, TransactionContext context) { return write(entry); } + /** + * Write asynchronously the state machine data in the given log entry to this state machine. + * + * @param entryRef Reference to a log entry. + * Implementations of this method may call {@link ReferenceCountedObject#get()} + * to access the log entry before this method returns. + * If the log entry is needed after this method returns, + * e.g. for asynchronous computation or caching, + * the implementation must invoke {@link ReferenceCountedObject#retain()} + * and {@link ReferenceCountedObject#release()}. + * @return a future for the write task + */ + default CompletableFuture write(ReferenceCountedObject entryRef, TransactionContext context) { + final LogEntryProto entry = entryRef.get(); + try { + final LogEntryProto copy = LogEntryProto.parseFrom(entry.toByteString()); + return write(copy, context); + } catch (InvalidProtocolBufferException e) { + return JavaUtils.completeExceptionally(new IllegalStateException( + "Failed to copy log entry " + TermIndex.valueOf(entry), e)); + } + } + /** * Create asynchronously a {@link DataStream} to stream state machine data. * The state machine may use the first message (i.e. request.getMessage()) as the header to create the stream. @@ -214,9 +263,20 @@ default void notifyLogFailed(Throwable cause, LogEntryProto failedEntry) {} default void notifySnapshotInstalled(InstallSnapshotResult result, long snapshotIndex, RaftPeer peer) {} /** - * Notify the {@link StateMachine} that a raft server has step down. + * Notify the {@link StateMachine} that the server for this division has been shut down. + * @Deprecated please use/override {@link #notifyServerShutdown(RoleInfoProto, boolean)} instead + */ + @Deprecated + default void notifyServerShutdown(RoleInfoProto roleInfo) { + notifyServerShutdown(roleInfo, false); + } + + /** + * Notify the {@link StateMachine} that either the server for this division or all the servers have been shut down. + * @param roleInfo roleInfo this server + * @param allServer whether all raft servers will be shutdown at this time */ - default void notifyServerShutdown(RoleInfoProto roleInfo) {} + default void notifyServerShutdown(RoleInfoProto roleInfo, boolean allServer) {} } /** diff --git a/ratis-server-api/src/main/java/org/apache/ratis/statemachine/TransactionContext.java b/ratis-server-api/src/main/java/org/apache/ratis/statemachine/TransactionContext.java index e0190747fc..d6ff9a0833 100644 --- a/ratis-server-api/src/main/java/org/apache/ratis/statemachine/TransactionContext.java +++ b/ratis-server-api/src/main/java/org/apache/ratis/statemachine/TransactionContext.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.Objects; +import java.util.Optional; /** * Context for a transaction. @@ -59,7 +60,10 @@ public interface TransactionContext { /** * Returns the data from the {@link StateMachine} * @return the data from the {@link StateMachine} + * @deprecated access StateMachineLogEntry via {@link TransactionContext#getLogEntryRef()} or + * {@link TransactionContext#getLogEntryUnsafe()} */ + @Deprecated StateMachineLogEntryProto getStateMachineLogEntry(); /** Set exception in case of failure. */ @@ -94,11 +98,40 @@ public interface TransactionContext { LogEntryProto initLogEntry(long term, long index); /** - * Returns the committed log entry - * @return the committed log entry + * @return a copy of the committed log entry if it exists; otherwise, returns null + * + * @deprecated Use {@link #getLogEntryRef()} or {@link #getLogEntryUnsafe()} to avoid copying. */ + @Deprecated LogEntryProto getLogEntry(); + /** + * @return the committed log entry if it exists; otherwise, returns null. + * The returned value is safe to use only before {@link StateMachine#applyTransaction} returns. + * Once {@link StateMachine#applyTransaction} has returned, it is unsafe to use the log entry + * since the underlying buffers can possiby be released. + */ + default LogEntryProto getLogEntryUnsafe() { + return getLogEntryRef().get(); + } + + /** + * Get a {@link ReferenceCountedObject} to the committed log entry. + * + * It is safe to access the log entry by calling {@link ReferenceCountedObject#get()} + * (without {@link ReferenceCountedObject#retain()}) + * inside the scope of {@link StateMachine#applyTransaction}. + * + * If the log entry is needed after {@link StateMachine#applyTransaction} returns, + * e.g. for asynchronous computation or caching, + * the caller must invoke {@link ReferenceCountedObject#retain()} and {@link ReferenceCountedObject#release()}. + * + * @return a reference to the committed log entry if it exists; otherwise, returns null. + */ + default ReferenceCountedObject getLogEntryRef() { + return Optional.ofNullable(getLogEntryUnsafe()).map(this::wrap).orElse(null); + } + /** Wrap the given log entry as a {@link ReferenceCountedObject} for retaining it for later use. */ default ReferenceCountedObject wrap(LogEntryProto entry) { Preconditions.assertSame(getLogEntry().getTerm(), entry.getTerm(), "entry.term"); @@ -202,7 +235,7 @@ public TransactionContext build() { return newTransactionContext(stateMachine, clientRequest, stateMachineLogEntry, logData, stateMachineData, stateMachineContext); } else { - Objects.requireNonNull(logEntry, "logEntry MUST NOT be null since clientRequest == null"); + Objects.requireNonNull(logEntry, "logEntry must not be null since clientRequest == null"); Preconditions.assertTrue(logEntry.hasStateMachineLogEntry(), () -> "Unexpected logEntry: stateMachineLogEntry not found, logEntry=" + logEntry); return newTransactionContext(serverRole, stateMachine, logEntry); diff --git a/ratis-server/pom.xml b/ratis-server/pom.xml index eb2920323d..8ce84148df 100644 --- a/ratis-server/pom.xml +++ b/ratis-server/pom.xml @@ -17,7 +17,7 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-server @@ -70,6 +70,31 @@ junit test + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.junit.platform + junit-platform-launcher + test + + + org.junit.jupiter + junit-jupiter-params + test + org.mockito mockito-core @@ -84,13 +109,5 @@ ratis-metrics-default test - - - - com.github.spotbugs - spotbugs-annotations - provided - true - diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ConfigurationManager.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ConfigurationManager.java index 0e020b7e36..2ba8107837 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ConfigurationManager.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ConfigurationManager.java @@ -25,7 +25,10 @@ import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.StringUtils; -import java.util.*; +import java.util.NavigableMap; +import java.util.SortedMap; +import java.util.TreeMap; + /** * Maintain the mappings between log index and corresponding raft configuration. @@ -41,9 +44,9 @@ public class ConfigurationManager { * The current raft configuration. If configurations is not empty, should be * the last entry of the map. Otherwise is initialConf. */ - private volatile RaftConfigurationImpl currentConf; + private RaftConfigurationImpl currentConf; /** Cache the peer corresponding to {@link #id}. */ - private volatile RaftPeer currentPeer; + private RaftPeer currentPeer; ConfigurationManager(RaftPeerId id, RaftConfigurationImpl initialConf) { this.id = id; @@ -76,11 +79,11 @@ private void addRaftConfigurationImpl(long logIndex, RaftConfigurationImpl conf) } } - RaftConfigurationImpl getCurrent() { + synchronized RaftConfigurationImpl getCurrent() { return currentConf; } - RaftPeer getCurrentPeer() { + synchronized RaftPeer getCurrentPeer() { return currentPeer; } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/FollowerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/FollowerState.java index 3911e39a5c..b01abcddc0 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/FollowerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/FollowerState.java @@ -19,13 +19,14 @@ import org.apache.ratis.server.DivisionInfo; import org.apache.ratis.server.leader.LeaderState; +import org.apache.ratis.server.util.ServerStringUtils; import org.apache.ratis.util.Daemon; -import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.ToIntFunction; @@ -59,13 +60,15 @@ int update(AtomicInteger outstanding) { private final RaftServerImpl server; private final Timestamp creationTime = Timestamp.currentTime(); + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile Timestamp lastRpcTime = creationTime; private volatile boolean isRunning = true; + private final CompletableFuture stopped = new CompletableFuture<>(); private final AtomicInteger outstandingOp = new AtomicInteger(); FollowerState(RaftServerImpl server, Object reason) { super(newBuilder() - .setName(server.getMemberId() + "-" + JavaUtils.getClassSimpleName(FollowerState.class)) + .setName(ServerStringUtils.generateUnifiedName(server.getMemberId(), FollowerState.class)) .setThreadGroup(server.getThreadGroup())); this.server = server; this.reason = reason; @@ -92,8 +95,10 @@ boolean isCurrentLeaderValid() { return lastRpcTime.elapsedTime().compareTo(server.properties().minRpcTimeout()) < 0; } - void stopRunning() { + CompletableFuture stopRunning() { this.isRunning = false; + interrupt(); + return stopped; } boolean lostMajorityHeartbeatsRecently() { @@ -121,6 +126,22 @@ private boolean shouldRun() { @Override public void run() { + try { + runImpl(); + } finally { + stopped.complete(null); + } + } + + private boolean roleChangeChecking(TimeDuration electionTimeout) { + return outstandingOp.get() == 0 + && isRunning && server.getInfo().isFollower() + && lastRpcTime.elapsedTime().compareTo(electionTimeout) >= 0 + && !lostMajorityHeartbeatsRecently() + && server.isRunning(); + } + + private void runImpl() { final TimeDuration sleepDeviationThreshold = server.getSleepDeviationThreshold(); while (shouldRun()) { final TimeDuration electionTimeout = server.getRandomElectionTimeout(); @@ -136,10 +157,7 @@ public void run() { break; } synchronized (server) { - if (outstandingOp.get() == 0 - && isRunning && server.getInfo().isFollower() - && lastRpcTime.elapsedTime().compareTo(electionTimeout) >= 0 - && !lostMajorityHeartbeatsRecently()) { + if (roleChangeChecking(electionTimeout)) { LOG.info("{}: change to CANDIDATE, lastRpcElapsedTime:{}, electionTimeout:{}", this, lastRpcTime.elapsedTime(), electionTimeout); server.getLeaderElectionMetrics().onLeaderElectionTimeout(); // Update timeout metric counters. diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java index 53e83d866e..9953e12aff 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderElection.java @@ -20,16 +20,17 @@ import org.apache.ratis.metrics.Timekeeper; import org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto; import org.apache.ratis.proto.RaftProtos.RequestVoteRequestProto; +import org.apache.ratis.proto.RaftProtos.TermIndexProto; +import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.server.DivisionInfo; import org.apache.ratis.server.RaftConfiguration; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.protocol.TermIndex; +import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.server.util.ServerStringUtils; import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.ratis.util.Daemon; -import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.LogUtils; import org.apache.ratis.util.Preconditions; @@ -46,6 +47,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; @@ -78,6 +80,121 @@ class LeaderElection implements Runnable { public static final Logger LOG = LoggerFactory.getLogger(LeaderElection.class); + interface ServerInterface { + default RaftPeerId getId() { + return getMemberId().getPeerId(); + } + + RaftGroupMemberId getMemberId(); + boolean isAlive(); + boolean isCandidate(); + + long getCurrentTerm(); + long getLastCommittedIndex(); + TermIndex getLastEntry(); + + boolean isPreVoteEnabled(); + ConfAndTerm initElection(Phase phase) throws IOException; + RequestVoteReplyProto requestVote(RequestVoteRequestProto r) throws IOException; + + void changeToLeader(); + void rejected(long term, ResultAndTerm result) throws IOException; + void shutdown(); + + Timekeeper getLeaderElectionTimer(); + void onNewLeaderElectionCompletion(); + + TimeDuration getRandomElectionTimeout(); + ThreadGroup getThreadGroup(); + + static ServerInterface get(RaftServerImpl server) { + final boolean preVote = RaftServerConfigKeys.LeaderElection.preVote(server.getRaftServer().getProperties()); + + return new ServerInterface() { + @Override + public RaftGroupMemberId getMemberId() { + return server.getMemberId(); + } + + @Override + public boolean isAlive() { + return server.getInfo().isAlive(); + } + + @Override + public boolean isCandidate() { + return server.getInfo().isCandidate(); + } + + @Override + public long getCurrentTerm() { + return server.getState().getCurrentTerm(); + } + + @Override + public long getLastCommittedIndex() { + return server.getRaftLog().getLastCommittedIndex(); + } + + @Override + public TermIndex getLastEntry() { + return server.getState().getLastEntry(); + } + + @Override + public boolean isPreVoteEnabled() { + return preVote; + } + + @Override + public ConfAndTerm initElection(Phase phase) throws IOException { + return server.getState().initElection(phase); + } + + @Override + public RequestVoteReplyProto requestVote(RequestVoteRequestProto r) throws IOException { + return server.getServerRpc().requestVote(r); + } + + @Override + public void changeToLeader() { + server.changeToLeader(); + } + + @Override + public void rejected(long term, ResultAndTerm result) throws IOException { + server.changeToFollowerAndPersistMetadata(term, false, result); + } + + @Override + public void shutdown() { + server.close(); + server.getStateMachine().event().notifyServerShutdown(server.getRoleInfoProto(), false); + } + + @Override + public Timekeeper getLeaderElectionTimer() { + return server.getLeaderElectionMetrics().getLeaderElectionTimer(); + } + + @Override + public void onNewLeaderElectionCompletion() { + server.getLeaderElectionMetrics().onNewLeaderElectionCompletion(); + } + + @Override + public TimeDuration getRandomElectionTimeout() { + return server.getRandomElectionTimeout(); + } + + @Override + public ThreadGroup getThreadGroup() { + return server.getThreadGroup(); + } + }; + } + } + private ResultAndTerm logAndReturn(Phase phase, Result result, Map responses, List exceptions) { return logAndReturn(phase, result, responses, exceptions, null); @@ -106,7 +223,7 @@ enum Phase { enum Result {PASSED, SINGLE_MODE_PASSED, REJECTED, TIMEOUT, DISCOVERED_A_NEW_TERM, SHUTDOWN, NOT_IN_CONF} - private static class ResultAndTerm { + static class ResultAndTerm { private final Result result; private final Long term; @@ -143,7 +260,7 @@ static class Executor { } void shutdown() { - executor.shutdown(); + executor.shutdownNow(); } void submit(Callable task) { @@ -183,23 +300,26 @@ public String toString() { private final String name; private final LifeCycle lifeCycle; private final Daemon daemon; + private final CompletableFuture stopped = new CompletableFuture<>(); - private final RaftServerImpl server; + private final ServerInterface server; private final boolean skipPreVote; private final ConfAndTerm round0; LeaderElection(RaftServerImpl server, boolean force) { - this.name = server.getMemberId() + "-" + JavaUtils.getClassSimpleName(getClass()) + COUNT.incrementAndGet(); + this(ServerInterface.get(server), force); + } + + LeaderElection(ServerInterface server, boolean force) { + this.name = ServerStringUtils.generateUnifiedName(server.getMemberId(), getClass()) + COUNT.incrementAndGet(); this.lifeCycle = new LifeCycle(this); this.daemon = Daemon.newBuilder().setName(name).setRunnable(this) .setThreadGroup(server.getThreadGroup()).build(); this.server = server; - this.skipPreVote = force || - !RaftServerConfigKeys.LeaderElection.preVote( - server.getRaftServer().getProperties()); + this.skipPreVote = force || !server.isPreVoteEnabled(); try { // increase term of the candidate in advance if it's forced to election - this.round0 = force ? server.getState().initElection(Phase.ELECTION) : null; + this.round0 = force ? server.initElection(Phase.ELECTION) : null; } catch (IOException e) { throw new IllegalStateException(name + ": Failed to initialize election", e); } @@ -223,8 +343,9 @@ private void startIfNew(Runnable starter) { } } - void shutdown() { + CompletableFuture shutdown() { lifeCycle.checkStateAndClose(); + return stopped; } @VisibleForTesting @@ -234,13 +355,21 @@ LifeCycle.State getCurrentState() { @Override public void run() { + try { + runImpl(); + } finally { + stopped.complete(null); + } + } + + private void runImpl() { if (!lifeCycle.compareAndTransition(STARTING, RUNNING)) { final LifeCycle.State state = lifeCycle.getCurrentState(); LOG.info("{}: skip running since this is already {}", this, state); return; } - try (AutoCloseable ignored = Timekeeper.start(server.getLeaderElectionMetrics().getLeaderElectionTimer())) { + try (AutoCloseable ignored = Timekeeper.start(server.getLeaderElectionTimer())) { for (int round = 0; shouldRun(); round++) { if (skipPreVote || askForVotes(Phase.PRE_VOTE, round)) { if (askForVotes(Phase.ELECTION, round)) { @@ -254,10 +383,10 @@ public void run() { } final LifeCycle.State state = lifeCycle.getCurrentState(); if (state.isClosingOrClosed()) { - LOG.info(this + ": since this is already " + state + ", safely ignore " + e); + LOG.info("{}: since this is already {}, safely ignore {}", this, state, e.toString()); } else { - if (!server.getInfo().isAlive()) { - LOG.info(this + ": since the server is not alive, safely ignore " + e); + if (!server.isAlive()) { + LOG.info("{}: since the server is not alive, safely ignore {}", this, e.toString()); } else { LOG.error("{}: Failed, state={}", this, state, e); } @@ -265,18 +394,17 @@ public void run() { } } finally { // Update leader election completion metric(s). - server.getLeaderElectionMetrics().onNewLeaderElectionCompletion(); + server.onNewLeaderElectionCompletion(); lifeCycle.checkStateAndClose(() -> {}); } } private boolean shouldRun() { - final DivisionInfo info = server.getInfo(); - return lifeCycle.getCurrentState().isRunning() && info.isCandidate() && info.isAlive(); + return lifeCycle.getCurrentState().isRunning() && server.isCandidate() && server.isAlive(); } private boolean shouldRun(long electionTerm) { - return shouldRun() && server.getState().getCurrentTerm() == electionTerm; + return shouldRun() && server.getCurrentTerm() == electionTerm; } private ResultAndTerm submitRequestAndWaitResult(Phase phase, RaftConfigurationImpl conf, long electionTerm) @@ -289,7 +417,7 @@ private ResultAndTerm submitRequestAndWaitResult(Phase phase, RaftConfigurationI if (others.isEmpty()) { r = new ResultAndTerm(Result.PASSED, electionTerm); } else { - final TermIndex lastEntry = server.getState().getLastEntry(); + final TermIndex lastEntry = server.getLastEntry(); final Executor voteExecutor = new Executor(this, others.size()); try { final int submitted = submitRequests(phase, electionTerm, lastEntry, others, voteExecutor); @@ -312,8 +440,7 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException, } // If round0 is non-null, we have already called initElection in the constructor, // reuse round0 to avoid initElection again for the first round - final ConfAndTerm confAndTerm = (round == 0 && round0 != null) ? - round0 : server.getState().initElection(phase); + final ConfAndTerm confAndTerm = (round == 0 && round0 != null) ? round0 : server.initElection(phase); electionTerm = confAndTerm.getTerm(); conf = confAndTerm.getConf(); } @@ -333,15 +460,14 @@ private boolean askForVotes(Phase phase, int round) throws InterruptedException, return true; case NOT_IN_CONF: case SHUTDOWN: - server.getRaftServer().close(); - server.getStateMachine().event().notifyServerShutdown(server.getRoleInfoProto()); + server.shutdown(); return false; case TIMEOUT: return false; // should retry case REJECTED: case DISCOVERED_A_NEW_TERM: - final long term = r.maxTerm(server.getState().getCurrentTerm()); - server.changeToFollowerAndPersistMetadata(term, false, r); + final long term = r.maxTerm(server.getCurrentTerm()); + server.rejected(term, r); return false; default: throw new IllegalArgumentException("Unable to process result " + r.result); } @@ -354,7 +480,7 @@ private int submitRequests(Phase phase, long electionTerm, TermIndex lastEntry, for (final RaftPeer peer : others) { final RequestVoteRequestProto r = ServerProtoUtils.toRequestVoteRequestProto( server.getMemberId(), peer.getId(), electionTerm, lastEntry, phase == Phase.PRE_VOTE); - voteExecutor.submit(() -> server.getServerRpc().requestVote(r)); + voteExecutor.submit(() -> server.requestVote(r)); submitted++; } return submitted; @@ -380,6 +506,9 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt Set higherPriorityPeers = getHigherPriorityPeers(conf); final boolean singleMode = conf.isSingleMode(server.getId()); + // true iff this server does not have any commits + final boolean emptyCommit = server.getLastCommittedIndex() < RaftLog.LEAST_VALID_LOG_INDEX; + while (waitForNum > 0 && shouldRun(electionTerm)) { final TimeDuration waitTime = timeout.elapsedTime().apply(n -> -n); if (waitTime.isNonPositive()) { @@ -429,7 +558,10 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt // all higher priority peers have replied higherPriorityPeers.remove(replierId); - if (r.getServerReply().getSuccess()) { + final boolean acceptVote = r.getServerReply().getSuccess() + // When the commits are non-empty, do not accept votes from empty log voters. + && (emptyCommit || nonEmptyLog(r)); + if (acceptVote) { votedPeers.add(replierId); // If majority and all peers with higher priority have voted, candidate pass vote if (higherPriorityPeers.isEmpty() && conf.hasMajority(votedPeers, server.getId())) { @@ -438,6 +570,7 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt } else { rejectedPeers.add(replierId); if (conf.majorityRejectVotes(rejectedPeers)) { + LOG.info("rejectedPeers: {}, emptyCommit? {}", rejectedPeers, emptyCommit); return logAndReturn(phase, Result.REJECTED, responses, exceptions); } } @@ -457,6 +590,26 @@ private ResultAndTerm waitForResults(Phase phase, long electionTerm, int submitt } } + /** + * @return true if the given reply indicates that the voter has a non-empty raft log. + * Note that a voter running with an old version may not include the lastEntry in the reply. + * For compatibility, this method returns true for such case. + */ + static boolean nonEmptyLog(RequestVoteReplyProto reply) { + final TermIndexProto lastEntry = reply.getLastEntry(); + // valid term >= 1 and valid index >= 0; therefore, (0, 0) can only be the proto default + if (lastEntry.equals(TermIndexProto.getDefaultInstance())) { // default: (0,0) + LOG.info("Reply missing lastEntry: {} ", ServerStringUtils.toRequestVoteReplyString(reply)); + return true; // accept voters with an older version + } + if (lastEntry.getTerm() > 0) { // when log is empty, lastEntry is (0,-1). + return true; // accept voters with a non-empty log + } + + LOG.info("Replier log is empty: {} ", ServerStringUtils.toRequestVoteReplyString(reply)); + return false; // reject voters with an empty log + } + @Override public String toString() { return name; diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderLease.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderLease.java index 315cc9f143..3b8f53da14 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderLease.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderLease.java @@ -92,7 +92,7 @@ private Timestamp getMaxTimestampWithMajorityAck(List followers) { return Timestamp.currentTime(); } - final int mid = followers.size() / 2; + final long mid = followers.size() / 2; return followers.stream() .map(FollowerInfo::getLastRespondedAppendEntriesSendTime) .sorted() diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java index b2788918d5..7141888968 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java @@ -49,6 +49,7 @@ import org.apache.ratis.server.raftlog.LogEntryHeader; import org.apache.ratis.server.raftlog.LogProtoUtils; import org.apache.ratis.server.raftlog.RaftLog; +import org.apache.ratis.server.util.ServerStringUtils; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.util.CodeInjectionForTesting; import org.apache.ratis.util.CollectionUtils; @@ -56,6 +57,7 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; @@ -110,12 +112,10 @@ private enum Type { } private final Type type; - private final long newTerm; private final Runnable handler; - StateUpdateEvent(Type type, long newTerm, Runnable handler) { + StateUpdateEvent(Type type, Runnable handler) { this.type = type; - this.newTerm = newTerm; this.handler = handler; } @@ -131,25 +131,30 @@ public boolean equals(Object obj) { return false; } final StateUpdateEvent that = (StateUpdateEvent)obj; - return this.type == that.type && this.newTerm == that.newTerm; + return this.type == that.type; } @Override public int hashCode() { - return Objects.hash(type, newTerm); + return type.hashCode(); } @Override public String toString() { - return type + (newTerm >= 0? ":" + newTerm: ""); + return type.name(); } } private class EventQueue { - private final String name = server.getMemberId() + "-" + JavaUtils.getClassSimpleName(getClass()); - private final BlockingQueue queue = new ArrayBlockingQueue<>(4096); + private final String name = ServerStringUtils.generateUnifiedName(server.getMemberId(), getClass()); + private final BlockingQueue queue = new ArrayBlockingQueue<>( + StateUpdateEvent.Type.values().length);; - void submit(StateUpdateEvent event) { + // submit can be invoked by different threads -- need to be synchronized + synchronized void submit(StateUpdateEvent event) { + if (queue.contains(event)) { // avoid duplicated events + return; + } try { queue.put(event); } catch (InterruptedException e) { @@ -158,10 +163,10 @@ void submit(StateUpdateEvent event) { } } + // poll is invoked only by the EventProcessor thread -- synchronized is not needed StateUpdateEvent poll() { - final StateUpdateEvent e; try { - e = queue.poll(server.getMaxTimeoutMs(), TimeUnit.MILLISECONDS); + return queue.poll(server.getMaxTimeoutMs(), TimeUnit.MILLISECONDS); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); String s = this + ": poll() is interrupted"; @@ -172,14 +177,6 @@ StateUpdateEvent poll() { throw new IllegalStateException(s + " UNEXPECTEDLY", ie); } } - - if (e != null) { - // remove duplicated events from the head. - while(e.equals(queue.peek())) { - queue.poll(); - } - } - return e; } @Override @@ -223,11 +220,8 @@ boolean removeAll(Collection c) { } CompletableFuture stopAll() { - final CompletableFuture[] futures = new CompletableFuture[senders.size()]; - for(int i = 0; i < futures.length; i++) { - futures[i] = senders.get(i).stopAsync(); - } - return CompletableFuture.allOf(futures); + return CompletableFuture.allOf(senders.stream(). + map(LogAppender::stopAsync).toArray(CompletableFuture[]::new)); } } @@ -268,7 +262,7 @@ static boolean isSameConf(CurrentOldFollowerInfos cached, RaftConfigurationImpl static class FollowerInfoMap { private final Map map = new ConcurrentHashMap<>(); - + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile CurrentOldFollowerInfos followerInfos; void put(RaftPeerId id, FollowerInfo info) { @@ -310,31 +304,29 @@ CompletableFuture getAppliedIndexFuture() { return appliedIndexFuture; } - boolean isApplied(LogEntryProto logEntry) { - if (appliedIndexFuture.isDone()) { - return true; - } - final long appliedIndex = logEntry != null? logEntry.getIndex(): server.getState().getLastAppliedIndex(); - if (appliedIndex >= startIndex) { - appliedIndexFuture.complete(appliedIndex); - LOG.info("leader is ready since appliedIndex == {} >= startIndex == {}", - appliedIndex, startIndex); - return true; - } else { - return false; + boolean checkStartIndex(LogEntryProto logEntry) { + final boolean completed = logEntry.getIndex() == startIndex && appliedIndexFuture.complete(startIndex); + if (completed) { + LOG.info("Leader {} is ready since appliedIndex == startIndex == {}", LeaderStateImpl.this, startIndex); } + return completed; + } + + boolean isApplied() { + return JavaUtils.isCompletedNormally(appliedIndexFuture); } } private final StateUpdateEvent updateCommitEvent = - new StateUpdateEvent(StateUpdateEvent.Type.UPDATE_COMMIT, -1, this::updateCommit); + new StateUpdateEvent(StateUpdateEvent.Type.UPDATE_COMMIT, this::updateCommit); private final StateUpdateEvent checkStagingEvent = - new StateUpdateEvent(StateUpdateEvent.Type.CHECK_STAGING, -1, this::checkStaging); + new StateUpdateEvent(StateUpdateEvent.Type.CHECK_STAGING, this::checkStaging); private final String name; private final RaftServerImpl server; private final RaftLog raftLog; private final long currentTerm; + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile ConfigurationStagingState stagingState; private final FollowerInfoMap followerInfoMap = new FollowerInfoMap(); @@ -353,7 +345,9 @@ boolean isApplied(LogEntryProto logEntry) { private final MemoizedSupplier startupLogEntry = MemoizedSupplier.valueOf(StartupLogEntry::new); private final AtomicBoolean isStopped = new AtomicBoolean(); + private final boolean logMetadataEnabled; private final int stagingCatchupGap; + private final TimeDuration stagingTimeout; private final RaftServerMetricsImpl raftServerMetrics; private final LogAppenderMetrics logAppenderMetrics; private final long followerMaxGapThreshold; @@ -363,11 +357,12 @@ boolean isApplied(LogEntryProto logEntry) { private final LeaderLease lease; LeaderStateImpl(RaftServerImpl server) { - this.name = server.getMemberId() + "-" + JavaUtils.getClassSimpleName(getClass()); + this.name = ServerStringUtils.generateUnifiedName(server.getMemberId(), getClass()); this.server = server; final RaftProperties properties = server.getRaftServer().getProperties(); stagingCatchupGap = RaftServerConfigKeys.stagingCatchupGap(properties); + stagingTimeout = RaftServerConfigKeys.stagingTimeout(properties); final ServerState state = server.getState(); this.raftLog = state.getLog(); @@ -378,11 +373,12 @@ boolean isApplied(LogEntryProto logEntry) { raftServerMetrics = server.getRaftServerMetrics(); logAppenderMetrics = new LogAppenderMetrics(server.getMemberId()); this.pendingRequests = new PendingRequests(server.getMemberId(), properties, raftServerMetrics); - this.watchRequests = new WatchRequests(server.getMemberId(), properties); + this.watchRequests = new WatchRequests(server.getMemberId(), properties, raftServerMetrics); this.messageStreamRequests = new MessageStreamRequests(server.getMemberId()); this.pendingStepDown = new PendingStepDown(this); this.readIndexHeartbeats = new ReadIndexHeartbeats(); this.lease = new LeaderLease(properties); + this.logMetadataEnabled = RaftServerConfigKeys.Log.logMetadataEnabled(properties); long maxPendingRequests = RaftServerConfigKeys.Write.elementLimit(properties); double followerGapRatioMax = RaftServerConfigKeys.Write.followerGapRatioMax(properties); @@ -421,12 +417,11 @@ void start() { } boolean isReady() { - return startupLogEntry.isInitialized() && startupLogEntry.get().isApplied(null); + return startupLogEntry.isInitialized() && startupLogEntry.get().isApplied(); } void checkReady(LogEntryProto entry) { - Preconditions.assertTrue(startupLogEntry.isInitialized()); - if (entry.getTerm() == getCurrentTerm() && startupLogEntry.get().isApplied(entry)) { + if (entry.getTerm() == server.getState().getCurrentTerm() && startupLogEntry.get().checkStartIndex(entry)) { server.getStateMachine().leaderEvent().notifyLeaderReady(); } } @@ -456,6 +451,7 @@ CompletableFuture stop() { logAppenderMetrics.unregister(); raftServerMetrics.unregister(); pendingRequests.close(); + watchRequests.close(); return f; } @@ -529,20 +525,21 @@ PendingRequests.Permit tryAcquirePendingRequest(Message message) { PendingRequest addPendingRequest(PendingRequests.Permit permit, RaftClientRequest request, TransactionContext entry) { if (LOG.isDebugEnabled()) { LOG.debug("{}: addPendingRequest at {}, entry={}", this, request, - LogProtoUtils.toLogEntryString(entry.getLogEntry())); + LogProtoUtils.toLogEntryString(entry.getLogEntryUnsafe())); } return pendingRequests.add(permit, request, entry); } - CompletableFuture streamAsync(RaftClientRequest request) { - return messageStreamRequests.streamAsync(request) + CompletableFuture streamAsync(ReferenceCountedObject requestRef) { + RaftClientRequest request = requestRef.get(); + return messageStreamRequests.streamAsync(requestRef) .thenApply(dummy -> server.newSuccessReply(request)) .exceptionally(e -> exception2RaftClientReply(request, e)); } - CompletableFuture streamEndOfRequestAsync(RaftClientRequest request) { - return messageStreamRequests.streamEndOfRequestAsync(request) - .thenApply(bytes -> RaftClientRequest.toWriteRequest(request, Message.valueOf(bytes))); + CompletableFuture> streamEndOfRequestAsync( + ReferenceCountedObject requestRef) { + return messageStreamRequests.streamEndOfRequestAsync(requestRef); } CompletableFuture addWatchRequest(RaftClientRequest request) { @@ -618,7 +615,7 @@ public AppendEntriesRequestProto newAppendEntriesRequestProto(FollowerInfo follo List entries, TermIndex previous, long callId) { final boolean initializing = !isCaughtUp(follower); final RaftPeerId targetId = follower.getId(); - return ServerProtoUtils.toAppendEntriesRequestProto(server.getMemberId(), targetId, currentTerm, entries, + return ServerProtoUtils.toAppendEntriesRequestProto(server.getMemberId(), targetId, getCurrentTerm(), entries, ServerImplUtils.effectiveCommitIndex(raftLog.getLastCommittedIndex(), previous, entries.size()), initializing, previous, server.getCommitInfos(), callId); } @@ -640,6 +637,10 @@ private RaftPeer getPeer(RaftPeerId id) { return server.getRaftConf().getPeer(id, RaftPeerRole.FOLLOWER, RaftPeerRole.LISTENER); } + private LogAppender newLogAppender(FollowerInfo f) { + return server.getRaftServer().getFactory().newLogAppender(server, this, f); + } + private Collection addSenders(Collection newPeers, long nextIndex, boolean caughtUp) { final Timestamp t = Timestamp.currentTime().addTimeMs(-server.getMaxTimeoutMs()); final List newAppenders = newPeers.stream().map(peer -> { @@ -647,7 +648,7 @@ private Collection addSenders(Collection newPeers, long n followerInfoMap.put(peer.getId(), f); raftServerMetrics.addFollower(peer.getId()); logAppenderMetrics.addFollowerGauges(peer.getId(), f::getNextIndex, f::getMatchIndex, f::getLastRpcTime); - return server.newLogAppender(this, f); + return newLogAppender(f); }).collect(Collectors.toList()); senders.addAll(newAppenders); return newAppenders; @@ -694,17 +695,17 @@ private void updateSenders(RaftConfigurationImpl conf) { } void submitStepDownEvent(StepDownReason reason) { - submitStepDownEvent(getCurrentTerm(), reason); + submitStepDownEvent(currentTerm, reason); } void submitStepDownEvent(long term, StepDownReason reason) { - eventQueue.submit(new StateUpdateEvent(StateUpdateEvent.Type.STEP_DOWN, term, () -> stepDown(term, reason))); + eventQueue.submit(new StateUpdateEvent(StateUpdateEvent.Type.STEP_DOWN, () -> stepDown(term, reason))); } private void stepDown(long term, StepDownReason reason) { try { lease.getAndSetEnabled(false); - server.changeToFollowerAndPersistMetadata(term, false, reason); + server.changeToFollowerAndPersistMetadata(term, false, reason).join(); pendingStepDown.complete(server::newSuccessReply); } catch(IOException e) { final String s = this + ": Failed to persist metadata for term " + term; @@ -788,12 +789,13 @@ public void run() { private BootStrapProgress checkProgress(FollowerInfo follower, long committed) { Preconditions.assertTrue(!isCaughtUp(follower)); final Timestamp progressTime = Timestamp.currentTime().addTimeMs(-server.getMaxTimeoutMs()); - final Timestamp timeoutTime = Timestamp.currentTime().addTimeMs(-3L * server.getMaxTimeoutMs()); + final Timestamp timeoutTime = Timestamp.currentTime().addTimeMs(-stagingTimeout.toLong(TimeUnit.MILLISECONDS)); if (follower.getLastRpcResponseTime().compareTo(timeoutTime) < 0) { LOG.debug("{} detects a follower {} timeout ({}ms) for bootstrapping", this, follower, follower.getLastRpcResponseTime().elapsedTimeMs()); return BootStrapProgress.NOPROGRESS; } else if (follower.getMatchIndex() + stagingCatchupGap > committed + && follower.getMatchIndex() >= server.getRaftConf().getLogEntryIndex() && follower.getLastRpcResponseTime().compareTo(progressTime) > 0 && follower.hasAttemptedToInstallSnapshot()) { return BootStrapProgress.CAUGHTUP; @@ -814,7 +816,9 @@ public void onFollowerSuccessAppendEntries(FollowerInfo follower) { @Override public boolean isFollowerBootstrapping(FollowerInfo follower) { - return isBootStrappingPeer(follower.getId()); + // It is better to check caught up than staging state + // since a follower may have already caught up but still in the staging state. + return !isCaughtUp(follower); } private void checkStaging() { @@ -846,7 +850,12 @@ private void checkStaging() { } boolean isBootStrappingPeer(RaftPeerId peerId) { - return Optional.ofNullable(stagingState).map(s -> s.contains(peerId)).orElse(false); + final Optional info = getLogAppender(peerId); + if (info.isPresent()) { + return !isCaughtUp(info.get().getFollower()); + } + final ConfigurationStagingState staging = stagingState; + return staging != null && staging.contains(peerId); } void submitUpdateCommitEvent() { @@ -942,8 +951,7 @@ private boolean hasMajority(Predicate isAcked) { private void updateCommit(LogEntryHeader[] entriesToCommit) { final long newCommitIndex = raftLog.getLastCommittedIndex(); - logMetadata(newCommitIndex); - commitIndexChanged(); + long lastCommitIndex = RaftLog.INVALID_LOG_INDEX; boolean hasConfiguration = false; for (LogEntryHeader entry : entriesToCommit) { @@ -952,7 +960,14 @@ private void updateCommit(LogEntryHeader[] entriesToCommit) { } hasConfiguration |= entry.getLogEntryBodyCase() == LogEntryBodyCase.CONFIGURATIONENTRY; raftLog.getRaftLogMetrics().onLogEntryCommitted(entry); + if (entry.getLogEntryBodyCase() != LogEntryBodyCase.METADATAENTRY) { + lastCommitIndex = entry.getIndex(); + } } + if (logMetadataEnabled && lastCommitIndex != RaftLog.INVALID_LOG_INDEX) { + logMetadata(lastCommitIndex); + } + commitIndexChanged(); if (hasConfiguration) { checkAndUpdateConfiguration(); } @@ -972,8 +987,9 @@ private void updateCommit(long majority, long min) { } private void logMetadata(long commitIndex) { - raftLog.appendMetadata(currentTerm, commitIndex); - notifySenders(); + if (raftLog.appendMetadata(currentTerm, commitIndex) != RaftLog.INVALID_LOG_INDEX) { + notifySenders(); + } } private void checkAndUpdateConfiguration() { @@ -1190,18 +1206,22 @@ private boolean checkLeaderLease() { && (server.getRaftConf().isSingleton() || lease.isValid()); } - void replyPendingRequest(long logIndex, RaftClientReply reply) { - pendingRequests.replyPendingRequest(logIndex, reply); + void replyPendingRequest(TermIndex termIndex, RaftClientReply reply) { + pendingRequests.replyPendingRequest(termIndex, reply); } - TransactionContext getTransactionContext(long index) { - return pendingRequests.getTransactionContext(index); + TransactionContext getTransactionContext(TermIndex termIndex) { + return pendingRequests.getTransactionContext(termIndex); } long[] getFollowerNextIndices() { return getLogAppenders().mapToLong(s -> s.getFollower().getNextIndex()).toArray(); } + long[] getFollowerMatchIndices() { + return getLogAppenders().mapToLong(s -> s.getFollower().getMatchIndex()).toArray(); + } + static Map newMap(Collection peers, String str) { Objects.requireNonNull(peers, () -> str + " == null"); final Map map = new HashMap<>(); @@ -1212,7 +1232,7 @@ static Map newMap(Collection peers, String str) } private class ConfigurationStagingState { - private final String name = server.getMemberId() + "-" + JavaUtils.getClassSimpleName(getClass()); + private final String name = ServerStringUtils.generateUnifiedName(server.getMemberId(), getClass()); private final Map newPeers; private final Map newListeners; private final PeerConfiguration newConf; diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/MessageStreamRequests.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/MessageStreamRequests.java index ac81b348bb..c00c57b364 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/MessageStreamRequests.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/MessageStreamRequests.java @@ -25,12 +25,15 @@ import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; class MessageStreamRequests { public static final Logger LOG = LoggerFactory.getLogger(MessageStreamRequests.class); @@ -39,12 +42,14 @@ private static class PendingStream { private final ClientInvocationId key; private long nextId = -1; private ByteString bytes = ByteString.EMPTY; + private final List> pendingRefs = new LinkedList<>(); PendingStream(ClientInvocationId key) { this.key = key; } - synchronized CompletableFuture append(long messageId, Message message) { + synchronized CompletableFuture append(long messageId, + ReferenceCountedObject requestRef) { if (nextId == -1) { nextId = messageId; } else if (messageId != nextId) { @@ -52,27 +57,38 @@ synchronized CompletableFuture append(long messageId, Message messag "Unexpected message id in " + key + ": messageId = " + messageId + " != nextId = " + nextId)); } nextId++; + final Message message = requestRef.retain().getMessage(); + pendingRefs.add(requestRef); bytes = bytes.concat(message.getContent()); return CompletableFuture.completedFuture(bytes); } - synchronized CompletableFuture getBytes(long messageId, Message message) { - return append(messageId, message); + synchronized CompletableFuture> getWriteRequest(long messageId, + ReferenceCountedObject requestRef) { + return append(messageId, requestRef) + .thenApply(appended -> RaftClientRequest.toWriteRequest(requestRef.get(), () -> appended)) + .thenApply(request -> ReferenceCountedObject.delegateFrom(pendingRefs, request)); + } + + synchronized void clear() { + pendingRefs.forEach(ReferenceCountedObject::release); + pendingRefs.clear(); } } static class StreamMap { - private final ConcurrentMap map = new ConcurrentHashMap<>(); + private final Map map = new HashMap<>(); - PendingStream computeIfAbsent(ClientInvocationId key) { + synchronized PendingStream computeIfAbsent(ClientInvocationId key) { return map.computeIfAbsent(key, PendingStream::new); } - PendingStream remove(ClientInvocationId key) { + synchronized PendingStream remove(ClientInvocationId key) { return map.remove(key); } - void clear() { + synchronized void clear() { + map.values().forEach(PendingStream::clear); map.clear(); } } @@ -84,15 +100,18 @@ void clear() { this.name = name + "-" + JavaUtils.getClassSimpleName(getClass()); } - CompletableFuture streamAsync(RaftClientRequest request) { + CompletableFuture streamAsync(ReferenceCountedObject requestRef) { + final RaftClientRequest request = requestRef.get(); final MessageStreamRequestTypeProto stream = request.getType().getMessageStream(); Preconditions.assertTrue(!stream.getEndOfRequest()); final ClientInvocationId key = ClientInvocationId.valueOf(request.getClientId(), stream.getStreamId()); final PendingStream pending = streams.computeIfAbsent(key); - return pending.append(stream.getMessageId(), request.getMessage()); + return pending.append(stream.getMessageId(), requestRef); } - CompletableFuture streamEndOfRequestAsync(RaftClientRequest request) { + CompletableFuture> streamEndOfRequestAsync( + ReferenceCountedObject requestRef) { + final RaftClientRequest request = requestRef.get(); final MessageStreamRequestTypeProto stream = request.getType().getMessageStream(); Preconditions.assertTrue(stream.getEndOfRequest()); final ClientInvocationId key = ClientInvocationId.valueOf(request.getClientId(), stream.getStreamId()); @@ -101,7 +120,7 @@ CompletableFuture streamEndOfRequestAsync(RaftClientRequest request) if (pending == null) { return JavaUtils.completeExceptionally(new StreamException(name + ": " + key + " not found")); } - return pending.getBytes(stream.getMessageId(), request.getMessage()); + return pending.getWriteRequest(stream.getMessageId(), requestRef); } void clear() { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/PendingRequest.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/PendingRequest.java index a0b96cc3d2..d72fcde90b 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/PendingRequest.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/PendingRequest.java @@ -17,28 +17,28 @@ */ package org.apache.ratis.server.impl; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.ratis.proto.RaftProtos.RaftClientRequestProto.TypeCase; import org.apache.ratis.proto.RaftProtos.CommitInfoProto; import org.apache.ratis.protocol.*; import org.apache.ratis.protocol.exceptions.NotLeaderException; -import org.apache.ratis.server.raftlog.RaftLog; +import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; import java.util.Collection; +import java.util.Objects; import java.util.concurrent.CompletableFuture; -public class PendingRequest implements Comparable { - private final long index; +class PendingRequest { + private final TermIndex termIndex; private final RaftClientRequest request; private final TransactionContext entry; private final CompletableFuture futureToComplete = new CompletableFuture<>(); private final CompletableFuture futureToReturn; - PendingRequest(long index, RaftClientRequest request, TransactionContext entry) { - this.index = index; + PendingRequest(RaftClientRequest request, TransactionContext entry) { + this.termIndex = entry == null? null: TermIndex.valueOf(entry.getLogEntryUnsafe()); this.request = request; this.entry = entry; if (request.is(TypeCase.FORWARD)) { @@ -49,7 +49,7 @@ public class PendingRequest implements Comparable { } PendingRequest(SetConfigurationRequest request) { - this(RaftLog.INVALID_LOG_INDEX, request, null); + this(request, null); } RaftClientReply convert(RaftClientRequest q, RaftClientReply p) { @@ -63,8 +63,8 @@ RaftClientReply convert(RaftClientRequest q, RaftClientReply p) { .build(); } - long getIndex() { - return index; + TermIndex getTermIndex() { + return Objects.requireNonNull(termIndex, "termIndex == null"); } RaftClientRequest getRequest() { @@ -101,14 +101,8 @@ TransactionContext setNotLeaderException(NotLeaderException nle, Collection {}", name, index, p); + synchronized PendingRequest put(Permit permit, PendingRequest p) { + LOG.debug("{}: PendingRequests.put {}", name, p); final Permit removed = permits.remove(permit); if (removed == null) { return null; } Preconditions.assertTrue(removed == permit); - final PendingRequest previous = map.put(index, p); + final PendingRequest previous = map.put(p.getTermIndex(), p); Preconditions.assertTrue(previous == null); return p; } - PendingRequest get(long index) { - final PendingRequest r = map.get(index); - LOG.debug("{}: PendingRequests.get {} returns {}", name, index, r); + PendingRequest get(TermIndex termIndex) { + final PendingRequest r = map.get(termIndex); + LOG.debug("{}: PendingRequests.get {} returns {}", name, termIndex, r); return r; } - PendingRequest remove(long index) { - final PendingRequest r = map.remove(index); - LOG.debug("{}: PendingRequests.remove {} returns {}", name, index, r); + PendingRequest remove(TermIndex termIndex) { + final PendingRequest r = map.remove(termIndex); + LOG.debug("{}: PendingRequests.remove {} returns {}", name, termIndex, r); if (r == null) { return null; } @@ -193,7 +194,7 @@ Collection setNotLeaderException(NotLeaderException nle, LOG.debug("{}: PendingRequests.setNotLeaderException", name); final List transactions = new ArrayList<>(map.size()); for(;;) { - final Iterator i = map.keySet().iterator(); + final Iterator i = map.keySet().iterator(); if (!i.hasNext()) { // the map is empty return transactions; } @@ -232,11 +233,8 @@ Permit tryAcquire(Message message) { } PendingRequest add(Permit permit, RaftClientRequest request, TransactionContext entry) { - // externally synced for now - final long index = entry.getLogEntry().getIndex(); - LOG.debug("{}: addPendingRequest at index={}, request={}", name, index, request); - final PendingRequest pending = new PendingRequest(index, request, entry); - return pendingRequests.put(permit, index, pending); + final PendingRequest pending = new PendingRequest(request, entry); + return pendingRequests.put(permit, pending); } PendingRequest addConfRequest(SetConfigurationRequest request) { @@ -265,17 +263,17 @@ void failSetConfiguration(RaftException e) { pendingSetConf = null; } - TransactionContext getTransactionContext(long index) { - PendingRequest pendingRequest = pendingRequests.get(index); + TransactionContext getTransactionContext(TermIndex termIndex) { + final PendingRequest pendingRequest = pendingRequests.get(termIndex); // it is possible that the pendingRequest is null if this peer just becomes // the new leader and commits transactions received by the previous leader return pendingRequest != null ? pendingRequest.getEntry() : null; } - void replyPendingRequest(long index, RaftClientReply reply) { - final PendingRequest pending = pendingRequests.remove(index); + void replyPendingRequest(TermIndex termIndex, RaftClientReply reply) { + final PendingRequest pending = pendingRequests.remove(termIndex); if (pending != null) { - Preconditions.assertTrue(pending.getIndex() == index); + Preconditions.assertEquals(termIndex, pending.getTermIndex(), "termIndex"); pending.setReply(reply); } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java index d609264af5..2e5cd58120 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftConfigurationImpl.java @@ -58,7 +58,7 @@ static final class Builder { private Builder() {} Builder setConf(PeerConfiguration conf) { - Objects.requireNonNull(conf); + Objects.requireNonNull(conf, "PeerConfiguration == null"); Preconditions.assertTrue(this.conf == null, "conf is already set."); this.conf = conf; return this; @@ -73,7 +73,7 @@ Builder setConf(Iterable peers, Iterable listeners) { } Builder setConf(RaftConfigurationImpl transitionalConf) { - Objects.requireNonNull(transitionalConf); + Objects.requireNonNull(transitionalConf, "transitionalConf == null"); Preconditions.assertTrue(transitionalConf.isTransitional()); Preconditions.assertTrue(!forceTransitional); @@ -83,7 +83,7 @@ Builder setConf(RaftConfigurationImpl transitionalConf) { Builder setOldConf(PeerConfiguration oldConf) { - Objects.requireNonNull(oldConf); + Objects.requireNonNull(oldConf, "oldConf == null"); Preconditions.assertTrue(this.oldConf == null, "oldConf is already set."); this.oldConf = oldConf; return this; @@ -94,7 +94,7 @@ Builder setOldConf(Iterable oldPeers, Iterable oldListeners) } Builder setOldConf(RaftConfigurationImpl stableConf) { - Objects.requireNonNull(stableConf); + Objects.requireNonNull(stableConf, "stableConf == null"); Preconditions.assertTrue(stableConf.isStable()); Preconditions.assertTrue(!forceStable); @@ -133,7 +133,7 @@ RaftConfigurationImpl build() { private RaftConfigurationImpl(PeerConfiguration conf, PeerConfiguration oldConf, long logEntryIndex) { - this.conf = Objects.requireNonNull(conf); + this.conf = Objects.requireNonNull(conf, "PeerConfiguration == null"); this.oldConf = oldConf; this.logEntryIndex = logEntryIndex; } @@ -148,6 +148,7 @@ boolean isStable() { return oldConf == null; } + @SuppressWarnings({"squid:S6466"}) // Suppress ArrayIndexOutOfBoundsException warning boolean containsInConf(RaftPeerId peerId, RaftPeerRole... roles) { if (roles == null || roles.length == 0) { return conf.contains(peerId); @@ -298,7 +299,7 @@ boolean isSingleton() { @Override public String toString() { - return logEntryIndex + ": " + conf + ", old=" + oldConf; + return "conf: {index: " + logEntryIndex + ", cur=" + conf + ", old=" + oldConf + "}"; } boolean hasNoChange(Collection newMembers, Collection newListeners) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java index 51067a87a1..043ba1ee71 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerImpl.java @@ -17,37 +17,31 @@ */ package org.apache.ratis.server.impl; -import java.io.File; -import java.io.IOException; -import java.nio.file.NoSuchFileException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.management.ObjectName; - import org.apache.ratis.client.impl.ClientProtoUtils; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.metrics.Timekeeper; -import org.apache.ratis.proto.RaftProtos.*; +import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto; +import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto.AppendResult; +import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto; +import org.apache.ratis.proto.RaftProtos.CommitInfoProto; +import org.apache.ratis.proto.RaftProtos.InstallSnapshotReplyProto; +import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto; +import org.apache.ratis.proto.RaftProtos.InstallSnapshotResult; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.proto.RaftProtos.LogInfoProto; +import org.apache.ratis.proto.RaftProtos.RaftClientRequestProto; import org.apache.ratis.proto.RaftProtos.RaftClientRequestProto.TypeCase; +import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto; +import org.apache.ratis.proto.RaftProtos.RaftPeerRole; +import org.apache.ratis.proto.RaftProtos.RaftRpcRequestProto; +import org.apache.ratis.proto.RaftProtos.ReadIndexReplyProto; +import org.apache.ratis.proto.RaftProtos.ReadIndexRequestProto; +import org.apache.ratis.proto.RaftProtos.ReplicationLevel; +import org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto; +import org.apache.ratis.proto.RaftProtos.RequestVoteRequestProto; +import org.apache.ratis.proto.RaftProtos.RoleInfoProto; +import org.apache.ratis.proto.RaftProtos.StartLeaderElectionReplyProto; +import org.apache.ratis.proto.RaftProtos.StartLeaderElectionRequestProto; import org.apache.ratis.protocol.ClientInvocationId; import org.apache.ratis.protocol.GroupInfoReply; import org.apache.ratis.protocol.GroupInfoRequest; @@ -65,19 +59,31 @@ import org.apache.ratis.protocol.SetConfigurationRequest; import org.apache.ratis.protocol.SnapshotManagementRequest; import org.apache.ratis.protocol.TransferLeadershipRequest; -import org.apache.ratis.protocol.exceptions.*; +import org.apache.ratis.protocol.exceptions.GroupMismatchException; +import org.apache.ratis.protocol.exceptions.LeaderNotReadyException; +import org.apache.ratis.protocol.exceptions.LeaderSteppingDownException; +import org.apache.ratis.protocol.exceptions.NotLeaderException; +import org.apache.ratis.protocol.exceptions.RaftException; +import org.apache.ratis.protocol.exceptions.ReadException; +import org.apache.ratis.protocol.exceptions.ReadIndexException; +import org.apache.ratis.protocol.exceptions.ReconfigurationInProgressException; +import org.apache.ratis.protocol.exceptions.ResourceUnavailableException; +import org.apache.ratis.protocol.exceptions.ServerNotReadyException; +import org.apache.ratis.protocol.exceptions.SetConfigurationException; +import org.apache.ratis.protocol.exceptions.StaleReadException; +import org.apache.ratis.protocol.exceptions.StateMachineException; +import org.apache.ratis.protocol.exceptions.TransferLeadershipException; import org.apache.ratis.server.DataStreamMap; import org.apache.ratis.server.DivisionInfo; import org.apache.ratis.server.DivisionProperties; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; -import org.apache.ratis.server.RaftServerMXBean; import org.apache.ratis.server.RaftServerRpc; import org.apache.ratis.server.impl.LeaderElection.Phase; import org.apache.ratis.server.impl.RetryCacheImpl.CacheEntry; -import org.apache.ratis.server.leader.FollowerInfo; +import org.apache.ratis.server.impl.ServerImplUtils.ConsecutiveIndices; +import org.apache.ratis.server.impl.ServerImplUtils.NavigableIndices; import org.apache.ratis.server.leader.LeaderState; -import org.apache.ratis.server.leader.LogAppender; import org.apache.ratis.server.metrics.LeaderElectionMetrics; import org.apache.ratis.server.metrics.RaftServerMetricsImpl; import org.apache.ratis.server.protocol.RaftServerAsynchronousProtocol; @@ -88,11 +94,11 @@ import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.server.storage.RaftStorageDirectory; -import org.apache.ratis.server.util.ServerStringUtils; import org.apache.ratis.statemachine.SnapshotInfo; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.statemachine.impl.TransactionContextImpl; +import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.apache.ratis.util.CodeInjectionForTesting; import org.apache.ratis.util.CollectionUtils; @@ -100,25 +106,53 @@ import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.JavaUtils; -import org.apache.ratis.util.JmxRegister; import org.apache.ratis.util.LifeCycle; +import org.apache.ratis.util.LifeCycle.State; import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.ProtoUtils; import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.TimeDuration; -import org.apache.ratis.util.Timestamp; import org.apache.ratis.util.function.CheckedSupplier; -import static org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto.AppendResult.INCONSISTENCY; -import static org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto.AppendResult.NOT_LEADER; -import static org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto.AppendResult.SUCCESS; -import static org.apache.ratis.server.raftlog.RaftLog.INVALID_LOG_INDEX; -import static org.apache.ratis.util.LifeCycle.State.EXCEPTION; -import static org.apache.ratis.util.LifeCycle.State.NEW; -import static org.apache.ratis.util.LifeCycle.State.PAUSED; -import static org.apache.ratis.util.LifeCycle.State.PAUSING; -import static org.apache.ratis.util.LifeCycle.State.RUNNING; -import static org.apache.ratis.util.LifeCycle.State.STARTING; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; + +import java.io.File; +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.ratis.server.impl.ServerImplUtils.assertEntries; +import static org.apache.ratis.server.impl.ServerImplUtils.assertGroup; +import static org.apache.ratis.server.impl.ServerImplUtils.effectiveCommitIndex; +import static org.apache.ratis.server.impl.ServerProtoUtils.toAppendEntriesReplyProto; +import static org.apache.ratis.server.impl.ServerProtoUtils.toReadIndexReplyProto; +import static org.apache.ratis.server.impl.ServerProtoUtils.toReadIndexRequestProto; +import static org.apache.ratis.server.impl.ServerProtoUtils.toRequestVoteReplyProto; +import static org.apache.ratis.server.impl.ServerProtoUtils.toStartLeaderElectionReplyProto; +import static org.apache.ratis.server.util.ServerStringUtils.toAppendEntriesReplyString; +import static org.apache.ratis.server.util.ServerStringUtils.toAppendEntriesRequestString; +import static org.apache.ratis.server.util.ServerStringUtils.toRequestVoteReplyString; class RaftServerImpl implements RaftServer.Division, RaftServerProtocol, RaftServerAsynchronousProtocol, @@ -130,6 +164,7 @@ class RaftServerImpl implements RaftServer.Division, static final String APPEND_TRANSACTION = CLASS_NAME + ".appendTransaction"; static final String LOG_SYNC = APPEND_ENTRIES + ".logComplete"; static final String START_LEADER_ELECTION = CLASS_NAME + ".startLeaderElection"; + static final String START_COMPLETE = CLASS_NAME + ".startComplete"; class Info implements DivisionInfo { @Override @@ -174,6 +209,14 @@ public long[] getFollowerNextIndices() { .map(LeaderStateImpl::getFollowerNextIndices) .orElse(null); } + + @Override + public long[] getFollowerMatchIndices() { + return role.getLeaderState() + .filter(leader -> isLeader()) + .map(LeaderStateImpl::getFollowerMatchIndices) + .orElse(null); + } } private final RaftServerProxy proxy; @@ -192,14 +235,15 @@ public long[] getFollowerNextIndices() { private final DataStreamMap dataStreamMap; private final RaftServerConfigKeys.Read.Option readOption; - private final TransactionManager transactionManager = new TransactionManager(); + private final TransactionManager transactionManager; private final RetryCacheImpl retryCache; private final CommitInfoCache commitInfoCache = new CommitInfoCache(); private final WriteIndexCache writeIndexCache; - private final RaftServerJmxAdapter jmxAdapter; + private final RaftServerJmxAdapter jmxAdapter = new RaftServerJmxAdapter(this); private final LeaderElectionMetrics leaderElectionMetrics; private final RaftServerMetricsImpl raftServerMetrics; + private final CountDownLatch closeFinishedLatch = new CountDownLatch(1); // To avoid append entry before complete start() method // For example, if thread1 start(), but before thread1 startAsFollower(), thread2 receive append entry @@ -218,6 +262,9 @@ public long[] getFollowerNextIndices() { private final AtomicBoolean firstElectionSinceStartup = new AtomicBoolean(true); private final ThreadGroup threadGroup; + private final AtomicReference> appendLogFuture; + private final NavigableIndices appendLogTermIndices = new NavigableIndices(); + RaftServerImpl(RaftGroup group, StateMachine stateMachine, RaftServerProxy proxy, RaftStorage.StartupOption option) throws IOException { final RaftPeerId id = proxy.getId(); @@ -238,8 +285,8 @@ public long[] getFollowerNextIndices() { this.dataStreamMap = new DataStreamMapImpl(id); this.readOption = RaftServerConfigKeys.Read.option(properties); this.writeIndexCache = new WriteIndexCache(properties); + this.transactionManager = new TransactionManager(id); - this.jmxAdapter = new RaftServerJmxAdapter(); this.leaderElectionMetrics = LeaderElectionMetrics.getLeaderElectionMetrics( getMemberId(), state::getLastLeaderElapsedTimeMs); this.raftServerMetrics = RaftServerMetricsImpl.computeIfAbsentRaftServerMetrics( @@ -251,6 +298,7 @@ public long[] getFollowerNextIndices() { this.transferLeadership = new TransferLeadership(this, properties); this.snapshotRequestHandler = new SnapshotManagementRequestHandler(this); this.snapshotInstallationHandler = new SnapshotInstallationHandler(this, properties); + this.appendLogFuture = new AtomicReference<>(CompletableFuture.completedFuture(null)); this.serverExecutor = ConcurrentUtils.newThreadPoolWithMax( RaftServerConfigKeys.ThreadPool.serverCached(properties), @@ -271,10 +319,6 @@ public DivisionProperties properties() { return divisionProperties; } - LogAppender newLogAppender(LeaderState leaderState, FollowerInfo f) { - return getRaftServer().getFactory().newLogAppender(this, leaderState, f); - } - int getMaxTimeoutMs() { return properties().maxRpcTimeoutMs(); } @@ -354,7 +398,7 @@ private void setRole(RaftPeerRole newRole, Object reason) { } boolean start() throws IOException { - if (!lifeCycle.compareAndTransition(NEW, STARTING)) { + if (!lifeCycle.compareAndTransition(State.NEW, State.STARTING)) { return false; } state.initialize(stateMachine); @@ -368,24 +412,18 @@ boolean start() throws IOException { startAsPeer(RaftPeerRole.LISTENER); } else { LOG.info("{}: start with initializing state, conf={}", getMemberId(), conf); - startInitializing(); + setRole(RaftPeerRole.FOLLOWER, "start"); } - registerMBean(getId(), getMemberId().getGroupId(), jmxAdapter, jmxAdapter); + jmxAdapter.registerMBean(); state.start(); - startComplete.compareAndSet(false, true); + CodeInjectionForTesting.execute(START_COMPLETE, getId(), null, role); + if (startComplete.compareAndSet(false, true)) { + LOG.info("{}: Successfully started.", getMemberId()); + } return true; } - static boolean registerMBean( - RaftPeerId id, RaftGroupId groupId, RaftServerMXBean mBean, JmxRegister jmx) { - final String prefix = "Ratis:service=RaftServer,group=" + groupId + ",id="; - final String registered = jmx.register(mBean, Arrays.asList( - () -> prefix + id, - () -> prefix + ObjectName.quote(id.toString()))); - return registered != null; - } - /** * The peer belongs to the current configuration, should start as a follower or listener */ @@ -402,17 +440,7 @@ private void startAsPeer(RaftPeerRole newRole) { } role.startFollowerState(this, reason); - lifeCycle.transition(RUNNING); - } - - /** - * The peer does not have any configuration (maybe it will later be included - * in some configuration). Start still as a follower but will not vote or - * start election. - */ - private void startInitializing() { - setRole(RaftPeerRole.FOLLOWER, "startInitializing"); - // do not start FollowerState + lifeCycle.transition(State.RUNNING); } ServerState getState() { @@ -455,7 +483,15 @@ void groupRemove(boolean deleteDirectory, boolean renameDirectory) { final RaftStorageDirectory dir = state.getStorage().getStorageDir(); /* Shutdown is triggered here inorder to avoid any locked files. */ + state.getStateMachineUpdater().setRemoving(); close(); + try { + closeFinishedLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.warn("{}: Waiting closing interrupted, will not continue to remove group locally", getMemberId()); + return; + } getStateMachine().event().notifyGroupRemove(); if (deleteDirectory) { for (int i = 0; i < FileUtils.NUM_ATTEMPTS; i ++) { @@ -534,6 +570,7 @@ public void close() { } catch (Exception e) { LOG.warn(getMemberId() + ": Failed to shutdown serverExecutor", e); } + closeFinishedLatch.countDown(); }); } @@ -549,21 +586,17 @@ void setFirstElection(Object reason) { * @param force Force to start a new {@link FollowerState} even if this server is already a follower. * @return if the term/votedFor should be updated to the new term */ - private synchronized boolean changeToFollower( - long newTerm, - boolean force, - boolean allowListener, - Object reason) { + private synchronized CompletableFuture changeToFollower( + long newTerm, boolean force, boolean allowListener, Object reason, AtomicBoolean metadataUpdated) { final RaftPeerRole old = role.getCurrentRole(); - final boolean metadataUpdated = state.updateCurrentTerm(newTerm); if (old == RaftPeerRole.LISTENER && !allowListener) { throw new IllegalStateException("Unexpected role " + old); } - + CompletableFuture future = CompletableFuture.completedFuture(null); if ((old != RaftPeerRole.FOLLOWER || force) && old != RaftPeerRole.LISTENER) { setRole(RaftPeerRole.FOLLOWER, reason); if (old == RaftPeerRole.LEADER) { - role.shutdownLeaderState(false) + future = role.shutdownLeaderState(false) .exceptionally(e -> { if (e != null) { if (!getInfo().isAlive()) { @@ -572,27 +605,33 @@ private synchronized boolean changeToFollower( } } throw new CompletionException("Failed to shutdownLeaderState: " + this, e); - }) - .join(); + }); state.setLeader(null, reason); } else if (old == RaftPeerRole.CANDIDATE) { - role.shutdownLeaderElection(); + future = role.shutdownLeaderElection(); } else if (old == RaftPeerRole.FOLLOWER) { - role.shutdownFollowerState(); + future = role.shutdownFollowerState(); } + + metadataUpdated.set(state.updateCurrentTerm(newTerm)); role.startFollowerState(this, reason); setFirstElection(reason); + } else { + metadataUpdated.set(state.updateCurrentTerm(newTerm)); } - return metadataUpdated; + return future; } - synchronized void changeToFollowerAndPersistMetadata( + synchronized CompletableFuture changeToFollowerAndPersistMetadata( long newTerm, boolean allowListener, Object reason) throws IOException { - if (changeToFollower(newTerm, false, allowListener, reason)) { + final AtomicBoolean metadataUpdated = new AtomicBoolean(); + final CompletableFuture future = changeToFollower(newTerm, false, allowListener, reason, metadataUpdated); + if (metadataUpdated.get()) { state.persistMetadata(); } + return future; } synchronized void changeToLeader() { @@ -608,6 +647,15 @@ synchronized void changeToLeader() { @Override public Collection getCommitInfos() { + try { + return getCommitInfosImpl(); + } catch (Throwable t) { + LOG.warn("{} Failed to getCommitInfos", getMemberId(), t); + return Collections.emptyList(); + } + } + + private Collection getCommitInfosImpl() { final List infos = new ArrayList<>(); // add the commit info of this server final long commitIndex = updateCommitInfoCache(); @@ -637,50 +685,33 @@ GroupInfoReply getGroupInfo(GroupInfoRequest request) { final RaftConfigurationProto conf = LogProtoUtils.toRaftConfigurationProtoBuilder(getRaftConf()).build(); return new GroupInfoReply(request, getCommitInfos(), getGroup(), getRoleInfoProto(), - dir.isHealthy(), conf); + dir.isHealthy(), conf, getLogInfo()); } - RoleInfoProto getRoleInfoProto() { - RaftPeerRole currentRole = role.getCurrentRole(); - RoleInfoProto.Builder roleInfo = RoleInfoProto.newBuilder() - .setSelf(getPeer().getRaftPeerProto()) - .setRole(currentRole) - .setRoleElapsedTimeMs(role.getRoleElapsedTimeMs()); - switch (currentRole) { - case CANDIDATE: - CandidateInfoProto.Builder candidate = CandidateInfoProto.newBuilder() - .setLastLeaderElapsedTimeMs(state.getLastLeaderElapsedTimeMs()); - roleInfo.setCandidateInfo(candidate); - break; - - case LISTENER: - case FOLLOWER: - final Optional fs = role.getFollowerState(); - final ServerRpcProto leaderInfo = ServerProtoUtils.toServerRpcProto( - getRaftConf().getPeer(state.getLeaderId()), - fs.map(FollowerState::getLastRpcTime).map(Timestamp::elapsedTimeMs).orElse(0L)); - // FollowerState can be null while adding a new peer as it is not - // a voting member yet - roleInfo.setFollowerInfo(FollowerInfoProto.newBuilder() - .setLeaderInfo(leaderInfo) - .setOutstandingOp(fs.map(FollowerState::getOutstandingOp).orElse(0))); - break; - - case LEADER: - role.getLeaderState().ifPresent(ls -> { - final LeaderInfoProto.Builder leader = LeaderInfoProto.newBuilder(); - ls.getLogAppenders().map(LogAppender::getFollower).forEach(f -> - leader.addFollowerInfo(ServerProtoUtils.toServerRpcProto( - f.getPeer(), f.getLastRpcResponseTime().elapsedTimeMs()))); - leader.setTerm(ls.getCurrentTerm()); - roleInfo.setLeaderInfo(leader); - }); - break; - - default: - throw new IllegalStateException("incorrect role of server " + currentRole); + LogInfoProto getLogInfo(){ + final RaftLog log = getRaftLog(); + LogInfoProto.Builder logInfoBuilder = LogInfoProto.newBuilder(); + final TermIndex applied = getStateMachine().getLastAppliedTermIndex(); + if (applied != null) { + logInfoBuilder.setApplied(applied.toProto()); + } + final TermIndex committed = log.getTermIndex(log.getLastCommittedIndex()); + if (committed != null) { + logInfoBuilder.setCommitted(committed.toProto()); } - return roleInfo.build(); + final TermIndex entry = log.getLastEntryTermIndex(); + if (entry != null) { + logInfoBuilder.setLastEntry(entry.toProto()); + } + final SnapshotInfo snapshot = getStateMachine().getLatestSnapshot(); + if (snapshot != null) { + logInfoBuilder.setLastSnapshot(snapshot.getTermIndex().toProto()); + } + return logInfoBuilder.build(); + } + + RoleInfoProto getRoleInfoProto() { + return role.buildRoleInfoProto(this); } synchronized void changeToCandidate(boolean forceStartLeaderElection) { @@ -696,7 +727,7 @@ synchronized void changeToCandidate(boolean forceStartLeaderElection) { @Override public String toString() { - return role + " " + state + " " + lifeCycle.getCurrentState(); + return role + " (" + lifeCycle.getCurrentState() + "): " + state; } RaftClientReply.Builder newReplyBuilder(RaftClientRequest request) { @@ -741,7 +772,7 @@ private CompletableFuture checkLeaderState(RaftClientRequest re */ private CompletableFuture checkLeaderState(RaftClientRequest request, CacheEntry entry) { try { - assertGroup(request.getRequestorId(), request.getRaftGroupId()); + assertGroup(getMemberId(), request); } catch (GroupMismatchException e) { return RetryCacheImpl.failWithException(e, entry); } @@ -771,7 +802,7 @@ private CompletableFuture checkLeaderState(RaftClientRequest re } NotLeaderException generateNotLeaderException() { - if (lifeCycle.getCurrentState() != RUNNING) { + if (!lifeCycle.getCurrentState().isRunning()) { return new NotLeaderException(getMemberId(), null, null); } RaftPeerId leaderId = state.getLeaderId(); @@ -790,15 +821,6 @@ void assertLifeCycleState(Set expected) throws ServerNotReadyEx getMemberId() + " is not in " + expected + ": current state is " + c), expected); } - void assertGroup(Object requestorId, RaftGroupId requestorGroupId) throws GroupMismatchException { - final RaftGroupId groupId = getMemberId().getGroupId(); - if (!groupId.equals(requestorGroupId)) { - throw new GroupMismatchException(getMemberId() - + ": The group (" + requestorGroupId + ") of " + requestorId - + " does not match the group (" + groupId + ") of the server " + getId()); - } - } - /** * Append a transaction to the log for processing a client request. * Note that the given request could be different from {@link TransactionContext#getClientRequest()} @@ -860,20 +882,14 @@ private CompletableFuture appendTransaction( leaderState.notifySenders(); } - final CompletableFuture future = pending.getFuture(); - if (request.is(TypeCase.WRITE)) { - // check replication - final ReplicationLevel replication = request.getType().getWrite().getReplication(); - if (replication != ReplicationLevel.MAJORITY) { - return future.thenCompose(reply -> waitForReplication(reply, replication)); - } - } - - return future; + return pending.getFuture(); } /** Wait until the given replication requirement is satisfied. */ private CompletableFuture waitForReplication(RaftClientReply reply, ReplicationLevel replication) { + if (!reply.isSuccess()) { + return CompletableFuture.completedFuture(reply); + } final RaftClientRequest.Type type = RaftClientRequest.watchRequestType(reply.getLogIndex(), replication); final RaftClientRequest watch = RaftClientRequest.newBuilder() .setServerId(reply.getServerId()) @@ -882,7 +898,24 @@ private CompletableFuture waitForReplication(RaftClientReply re .setCallId(reply.getCallId()) .setType(type) .build(); - return watchAsync(watch).thenApply(r -> reply); + return watchAsync(watch).thenApply(watchReply -> combineReplies(reply, watchReply)); + } + + private RaftClientReply combineReplies(RaftClientReply reply, RaftClientReply watchReply) { + final RaftClientReply combinedReply = RaftClientReply.newBuilder() + .setServerId(getMemberId()) + // from write reply + .setClientId(reply.getClientId()) + .setCallId(reply.getCallId()) + .setMessage(reply.getMessage()) + .setLogIndex(reply.getLogIndex()) + // from watchReply + .setSuccess(watchReply.isSuccess()) + .setException(watchReply.getException()) + .setCommitInfos(watchReply.getCommitInfos()) + .build(); + LOG.debug("combinedReply={}", combinedReply); + return combinedReply; } void stepDownOnJvmPause() { @@ -914,25 +947,27 @@ CompletableFuture executeSubmitClientRequestAsync( public CompletableFuture submitClientRequestAsync( ReferenceCountedObject requestRef) { final RaftClientRequest request = requestRef.retain(); - LOG.debug("{}: receive client request({})", getMemberId(), request); - try { + LOG.debug("{}: receive client request({})", getMemberId(), request); assertLifeCycleState(LifeCycle.States.RUNNING); - } catch (ServerNotReadyException e) { - final RaftClientReply reply = newExceptionReply(request, e); - requestRef.release(); - return CompletableFuture.completedFuture(reply); - } - final Timekeeper timer = raftServerMetrics.getClientRequestTimer(request.getType()); - final Optional timerContext = Optional.ofNullable(timer).map(Timekeeper::time); - return replyFuture(requestRef).whenComplete((clientReply, exception) -> { + RaftClientRequest.Type type = request.getType(); + final Timekeeper timer = raftServerMetrics.getClientRequestTimer(type); + final Optional timerContext = Optional.ofNullable(timer).map(Timekeeper::time); + return replyFuture(requestRef).whenComplete((clientReply, exception) -> { + timerContext.ifPresent(Timekeeper.Context::stop); + if (exception != null || clientReply.getException() != null) { + raftServerMetrics.incFailedRequestCount(type); + } + }); + } catch (RaftException e) { + return CompletableFuture.completedFuture(newExceptionReply(request, e)); + } catch (Throwable t) { + LOG.error("{} Failed to submitClientRequestAsync for {}", getMemberId(), request, t); + return CompletableFuture.completedFuture(newExceptionReply(request, new RaftException(t))); + } finally { requestRef.release(); - timerContext.ifPresent(Timekeeper.Context::stop); - if (exception != null || clientReply.getException() != null) { - raftServerMetrics.incFailedRequestCount(request.getType()); - } - }); + } } private CompletableFuture replyFuture(ReferenceCountedObject requestRef) { @@ -958,6 +993,19 @@ private CompletableFuture replyFuture(ReferenceCountedObject writeAsync(ReferenceCountedObject requestRef) { + final RaftClientRequest request = requestRef.get(); + final CompletableFuture future = writeAsyncImpl(requestRef); + if (request.is(TypeCase.WRITE)) { + // check replication + final ReplicationLevel replication = request.getType().getWrite().getReplication(); + if (replication != ReplicationLevel.MAJORITY) { + return future.thenCompose(r -> waitForReplication(r, replication)); + } + } + return future; + } + + private CompletableFuture writeAsyncImpl(ReferenceCountedObject requestRef) { final RaftClientRequest request = requestRef.get(); final CompletableFuture reply = checkLeaderState(request); if (reply != null) { @@ -1028,8 +1076,7 @@ private CompletableFuture sendReadIndexAsync(RaftClientRequ if (leaderId == null) { return JavaUtils.completeExceptionally(new ReadIndexException(getMemberId() + ": Leader is unknown.")); } - final ReadIndexRequestProto request = - ServerProtoUtils.toReadIndexRequestProto(clientRequest, getMemberId(), leaderId); + final ReadIndexRequestProto request = toReadIndexRequestProto(clientRequest, getMemberId(), leaderId); try { return getServerRpc().async().readIndexAsync(request); } catch (IOException e) { @@ -1102,21 +1149,28 @@ private CompletableFuture messageStreamAsync(ReferenceCountedOb } if (request.getType().getMessageStream().getEndOfRequest()) { - final CompletableFuture f = streamEndOfRequestAsync(request); + final CompletableFuture> f = streamEndOfRequestAsync(requestRef); if (f.isCompletedExceptionally()) { return f.thenApply(r -> null); } // the message stream has ended and the request become a WRITE request - return replyFuture(requestRef.delegate(f.join())); + ReferenceCountedObject joinedRequest = f.join(); + try { + return replyFuture(joinedRequest); + } finally { + // Released pending streaming requests. + joinedRequest.release(); + } } return role.getLeaderState() - .map(ls -> ls.streamAsync(request)) + .map(ls -> ls.streamAsync(requestRef)) .orElseGet(() -> CompletableFuture.completedFuture( newExceptionReply(request, generateNotLeaderException()))); } - private CompletableFuture streamEndOfRequestAsync(RaftClientRequest request) { + private CompletableFuture> streamEndOfRequestAsync( + ReferenceCountedObject request) { return role.getLeaderState() .map(ls -> ls.streamEndOfRequestAsync(request)) .orElse(null); @@ -1199,7 +1253,7 @@ CompletableFuture transferLeadershipAsync(TransferLeadershipReq LOG.info("{}: receive transferLeadership {}", getMemberId(), request); assertLifeCycleState(LifeCycle.States.RUNNING); - assertGroup(request.getRequestorId(), request.getRaftGroupId()); + assertGroup(getMemberId(), request); synchronized (this) { CompletableFuture reply = checkLeaderState(request); @@ -1229,7 +1283,7 @@ CompletableFuture transferLeadershipAsync(TransferLeadershipReq if (!conf.isHighestPriority(request.getNewLeader())) { String msg = getMemberId() + " refused to transfer leadership to peer " + request.getNewLeader() + - " as it does not has highest priority " + conf; + " as it does not has highest priority in " + conf; return logAndReturnTransferLeadershipFail(request, msg); } @@ -1240,10 +1294,11 @@ CompletableFuture transferLeadershipAsync(TransferLeadershipReq CompletableFuture takeSnapshotAsync(SnapshotManagementRequest request) throws IOException { LOG.info("{}: takeSnapshotAsync {}", getMemberId(), request); assertLifeCycleState(LifeCycle.States.RUNNING); - assertGroup(request.getRequestorId(), request.getRaftGroupId()); + assertGroup(getMemberId(), request); + Objects.requireNonNull(request.getCreate(), "create == null"); - //TODO(liuyaolong): get the gap value from shell command - long minGapValue = RaftServerConfigKeys.Snapshot.creationGap(proxy.getProperties()); + final long creationGap = request.getCreate().getCreationGap(); + long minGapValue = creationGap > 0? creationGap : RaftServerConfigKeys.Snapshot.creationGap(proxy.getProperties()); final long lastSnapshotIndex = Optional.ofNullable(stateMachine.getLatestSnapshot()) .map(SnapshotInfo::getIndex) .orElse(0L); @@ -1254,7 +1309,7 @@ CompletableFuture takeSnapshotAsync(SnapshotManagementRequest r synchronized (this) { final long installSnapshot = snapshotInstallationHandler.getInProgressInstallSnapshotIndex(); // check snapshot install/load - if (installSnapshot != INVALID_LOG_INDEX) { + if (installSnapshot != RaftLog.INVALID_LOG_INDEX) { String msg = String.format("%s: Failed do snapshot as snapshot (%s) installation is in progress", getMemberId(), installSnapshot); LOG.warn(msg); @@ -1272,7 +1327,7 @@ CompletableFuture leaderElectionManagementAsync(LeaderElectionM throws IOException { LOG.info("{} receive leaderElectionManagement request {}", getMemberId(), request); assertLifeCycleState(LifeCycle.States.RUNNING); - assertGroup(request.getRequestorId(), request.getRaftGroupId()); + assertGroup(getMemberId(), request); final LeaderElectionManagementRequest.Pause pause = request.getPause(); if (pause != null) { @@ -1291,7 +1346,7 @@ CompletableFuture leaderElectionManagementAsync(LeaderElectionM CompletableFuture stepDownLeaderAsync(TransferLeadershipRequest request) throws IOException { LOG.info("{} receive stepDown leader request {}", getMemberId(), request); assertLifeCycleState(LifeCycle.States.RUNNING); - assertGroup(request.getRequestorId(), request.getRaftGroupId()); + assertGroup(getMemberId(), request); return role.getLeaderState().map(leader -> leader.submitStepDownRequestAsync(request)) .orElseGet(() -> CompletableFuture.completedFuture( @@ -1308,7 +1363,7 @@ public RaftClientReply setConfiguration(SetConfigurationRequest request) throws public CompletableFuture setConfigurationAsync(SetConfigurationRequest request) throws IOException { LOG.info("{}: receive setConfiguration {}", getMemberId(), request); assertLifeCycleState(LifeCycle.States.RUNNING); - assertGroup(request.getRequestorId(), request.getRaftGroupId()); + assertGroup(getMemberId(), request); CompletableFuture reply = checkLeaderState(request); if (reply != null) { @@ -1387,15 +1442,13 @@ static List add(RaftPeerRole role, RaftConfigurationImpl conf, SetConf } /** - * check if the remote peer is not included in the current conf - * and should shutdown. should shutdown if all the following stands: - * 1. this is a leader + * The remote peer should shut down if all the following are true. + * 1. this is the current leader * 2. current conf is stable and has been committed - * 3. candidate id is not included in conf - * 4. candidate's last entry's index < conf's index + * 3. candidate is not in the current conf + * 4. candidate last entry index < conf index (the candidate was removed) */ - private boolean shouldSendShutdown(RaftPeerId candidateId, - TermIndex candidateLastEntry) { + private boolean shouldSendShutdown(RaftPeerId candidateId, TermIndex candidateLastEntry) { return getInfo().isLeader() && getRaftConf().isStable() && getState().isConfCommitted() @@ -1422,10 +1475,11 @@ private RequestVoteReplyProto requestVote(Phase phase, LOG.info("{}: receive requestVote({}, {}, {}, {}, {})", getMemberId(), phase, candidateId, candidateGroupId, candidateTerm, candidateLastEntry); assertLifeCycleState(LifeCycle.States.RUNNING); - assertGroup(candidateId, candidateGroupId); + assertGroup(getMemberId(), candidateId, candidateGroupId); boolean shouldShutdown = false; final RequestVoteReplyProto reply; + CompletableFuture future = null; synchronized (this) { // Check life cycle state again to avoid the PAUSING/PAUSED state. assertLifeCycleState(LifeCycle.States.RUNNING); @@ -1435,12 +1489,12 @@ private RequestVoteReplyProto requestVote(Phase phase, final boolean voteGranted = context.decideVote(candidate, candidateLastEntry); if (candidate != null && phase == Phase.ELECTION) { // change server state in the ELECTION phase - final boolean termUpdated = - changeToFollower(candidateTerm, true, false, "candidate:" + candidateId); + final AtomicBoolean termUpdated = new AtomicBoolean(); + future = changeToFollower(candidateTerm, true, false, "candidate:" + candidateId, termUpdated); if (voteGranted) { state.grantVote(candidate.getId()); } - if (termUpdated || voteGranted) { + if (termUpdated.get() || voteGranted) { state.persistMetadata(); // sync metafile } } @@ -1449,75 +1503,55 @@ private RequestVoteReplyProto requestVote(Phase phase, } else if(shouldSendShutdown(candidateId, candidateLastEntry)) { shouldShutdown = true; } - reply = ServerProtoUtils.toRequestVoteReplyProto(candidateId, getMemberId(), - voteGranted, state.getCurrentTerm(), shouldShutdown); + reply = toRequestVoteReplyProto(candidateId, getMemberId(), + voteGranted, state.getCurrentTerm(), shouldShutdown, state.getLastEntry()); if (LOG.isInfoEnabled()) { LOG.info("{} replies to {} vote request: {}. Peer's state: {}", - getMemberId(), phase, ServerStringUtils.toRequestVoteReplyString(reply), state); + getMemberId(), phase, toRequestVoteReplyString(reply), state); } } - return reply; - } - - private void validateEntries(long expectedTerm, TermIndex previous, - List entries) { - if (entries != null && !entries.isEmpty()) { - final long index0 = entries.get(0).getIndex(); - // Check if next entry's index is 1 greater than the snapshotIndex. If yes, then - // we do not have to check for the existence of previous. - if (index0 != state.getSnapshotIndex() + 1) { - if (previous == null || previous.getTerm() == 0) { - Preconditions.assertTrue(index0 == 0, - "Unexpected Index: previous is null but entries[%s].getIndex()=%s", - 0, index0); - } else { - Preconditions.assertTrue(previous.getIndex() == index0 - 1, - "Unexpected Index: previous is %s but entries[%s].getIndex()=%s", - previous, 0, index0); - } - } - - for (int i = 0; i < entries.size(); i++) { - LogEntryProto entry = entries.get(i); - final long t = entry.getTerm(); - Preconditions.assertTrue(expectedTerm >= t, - "Unexpected Term: entries[%s].getTerm()=%s but expectedTerm=%s", - i, t, expectedTerm); - - final long indexi = entry.getIndex(); - Preconditions.assertTrue(indexi == index0 + i, - "Unexpected Index: entries[%s].getIndex()=%s but entries[0].getIndex()=%s", - i, indexi, index0); - } + if (future != null) { + future.join(); } + return reply; } @Override public AppendEntriesReplyProto appendEntries(AppendEntriesRequestProto r) throws IOException { try { - return appendEntriesAsync(r).join(); + return appendEntriesAsync(ReferenceCountedObject.wrap(r)).join(); } catch (CompletionException e) { throw IOUtils.asIOException(JavaUtils.unwrapCompletionException(e)); } } @Override - public CompletableFuture appendEntriesAsync(AppendEntriesRequestProto r) - throws IOException { + public CompletableFuture appendEntriesAsync( + ReferenceCountedObject requestRef) throws IOException { + final AppendEntriesRequestProto r = requestRef.retain(); final RaftRpcRequestProto request = r.getServerRequest(); - final List entries = r.getEntriesList(); final TermIndex previous = r.hasPreviousLog()? TermIndex.valueOf(r.getPreviousLog()) : null; - final RaftPeerId requestorId = RaftPeerId.valueOf(request.getRequestorId()); - - preAppendEntriesAsync(requestorId, ProtoUtils.toRaftGroupId(request.getRaftGroupId()), r.getLeaderTerm(), - previous, r.getLeaderCommit(), r.getInitializing(), entries); try { - return appendEntriesAsync(requestorId, r.getLeaderTerm(), previous, r.getLeaderCommit(), - request.getCallId(), r.getInitializing(), r.getCommitInfosList(), entries); + final RaftPeerId leaderId = RaftPeerId.valueOf(request.getRequestorId()); + final RaftGroupId leaderGroupId = ProtoUtils.toRaftGroupId(request.getRaftGroupId()); + + CodeInjectionForTesting.execute(APPEND_ENTRIES, getId(), leaderId, previous, r); + + assertLifeCycleState(LifeCycle.States.STARTING_OR_RUNNING); + if (!startComplete.get()) { + throw new ServerNotReadyException(getMemberId() + ": The server role is not yet initialized."); + } + assertGroup(getMemberId(), leaderId, leaderGroupId); + assertEntries(r, previous, state); + + return appendEntriesAsync(leaderId, request.getCallId(), previous, requestRef); } catch(Exception t) { - LOG.error("{}: Failed appendEntriesAsync {}", getMemberId(), r, t); - throw t; + LOG.error("{}: Failed appendEntries* {}", getMemberId(), + toAppendEntriesRequestString(r, stateMachine::toStateMachineLogEntryString), t); + throw IOUtils.asIOException(t); + } finally { + requestRef.release(); } } @@ -1529,14 +1563,12 @@ public CompletableFuture readIndexAsync(ReadIndexRequestPro final LeaderStateImpl leader = role.getLeaderState().orElse(null); if (leader == null) { - return CompletableFuture.completedFuture( - ServerProtoUtils.toReadIndexReplyProto(peerId, getMemberId(), false, INVALID_LOG_INDEX)); + return CompletableFuture.completedFuture(toReadIndexReplyProto(peerId, getMemberId())); } return getReadIndex(ClientProtoUtils.toRaftClientRequest(request.getClientRequest()), leader) - .thenApply(index -> ServerProtoUtils.toReadIndexReplyProto(peerId, getMemberId(), true, index)) - .exceptionally(throwable -> - ServerProtoUtils.toReadIndexReplyProto(peerId, getMemberId(), false, INVALID_LOG_INDEX)); + .thenApply(index -> toReadIndexReplyProto(peerId, getMemberId(), true, index)) + .exceptionally(throwable -> toReadIndexReplyProto(peerId, getMemberId())); } static void logAppendEntries(boolean isHeartbeat, Supplier message) { @@ -1553,7 +1585,7 @@ static void logAppendEntries(boolean isHeartbeat, Supplier message) { Optional updateLastRpcTime(FollowerState.UpdateType updateType) { final Optional fs = role.getFollowerState(); - if (fs.isPresent() && lifeCycle.getCurrentState() == RUNNING) { + if (fs.isPresent() && lifeCycle.getCurrentState().isRunning()) { fs.get().updateLastRpcTime(updateType); return fs; } else { @@ -1561,24 +1593,6 @@ Optional updateLastRpcTime(FollowerState.UpdateType updateType) { } } - private void preAppendEntriesAsync(RaftPeerId leaderId, RaftGroupId leaderGroupId, long leaderTerm, - TermIndex previous, long leaderCommit, boolean initializing, List entries) throws IOException { - CodeInjectionForTesting.execute(APPEND_ENTRIES, getId(), - leaderId, leaderTerm, previous, leaderCommit, initializing, entries); - - assertLifeCycleState(LifeCycle.States.STARTING_OR_RUNNING); - if (!startComplete.get()) { - throw new ServerNotReadyException(getMemberId() + ": The server role is not yet initialized."); - } - assertGroup(leaderId, leaderGroupId); - - try { - validateEntries(leaderTerm, previous, entries); - } catch (IllegalArgumentException e) { - throw new IOException(e); - } - } - private long updateCommitInfoCache() { return commitInfoCache.update(getId(), state.getLog().getLastCommittedIndex()); } @@ -1587,45 +1601,38 @@ ExecutorService getServerExecutor() { return serverExecutor; } - @SuppressWarnings("checkstyle:parameternumber") - private CompletableFuture appendEntriesAsync( - RaftPeerId leaderId, long leaderTerm, TermIndex previous, long leaderCommit, long callId, boolean initializing, - List commitInfos, List entries) throws IOException { + private CompletableFuture appendEntriesAsync(RaftPeerId leaderId, long callId, + TermIndex previous, ReferenceCountedObject requestRef) throws IOException { + final AppendEntriesRequestProto proto = requestRef.get(); + final List entries = proto.getEntriesList(); final boolean isHeartbeat = entries.isEmpty(); - logAppendEntries(isHeartbeat, - () -> getMemberId() + ": receive appendEntries(" + leaderId + ", " + leaderTerm + ", " - + previous + ", " + leaderCommit + ", " + initializing - + ", commits:" + ProtoUtils.toString(commitInfos) - + ", cId:" + callId - + ", entries: " + LogProtoUtils.toLogEntriesString(entries)); + logAppendEntries(isHeartbeat, () -> getMemberId() + ": appendEntries* " + + toAppendEntriesRequestString(proto, stateMachine::toStateMachineLogEntryString)); + final long leaderTerm = proto.getLeaderTerm(); final long currentTerm; final long followerCommit = state.getLog().getLastCommittedIndex(); final Optional followerState; final Timekeeper.Context timer = raftServerMetrics.getFollowerAppendEntryTimer(isHeartbeat).time(); + final CompletableFuture future; synchronized (this) { // Check life cycle state again to avoid the PAUSING/PAUSED state. assertLifeCycleState(LifeCycle.States.STARTING_OR_RUNNING); - final boolean recognized = state.recognizeLeader(leaderId, leaderTerm); currentTerm = state.getCurrentTerm(); + final boolean recognized = state.recognizeLeader(Op.APPEND_ENTRIES, leaderId, leaderTerm); if (!recognized) { - final AppendEntriesReplyProto reply = ServerProtoUtils.toAppendEntriesReplyProto( - leaderId, getMemberId(), currentTerm, followerCommit, state.getNextIndex(), NOT_LEADER, callId, - INVALID_LOG_INDEX, isHeartbeat); - if (LOG.isDebugEnabled()) { - LOG.debug("{}: Not recognize {} (term={}) as leader, state: {} reply: {}", - getMemberId(), leaderId, leaderTerm, state, ServerStringUtils.toAppendEntriesReplyString(reply)); - } - return CompletableFuture.completedFuture(reply); + return CompletableFuture.completedFuture(toAppendEntriesReplyProto( + leaderId, getMemberId(), currentTerm, followerCommit, state.getNextIndex(), + AppendResult.NOT_LEADER, callId, RaftLog.INVALID_LOG_INDEX, isHeartbeat)); } try { - changeToFollowerAndPersistMetadata(leaderTerm, true, "appendEntries"); + future = changeToFollowerAndPersistMetadata(leaderTerm, true, "appendEntries"); } catch (IOException e) { return JavaUtils.completeExceptionally(e); } state.setLeader(leaderId, "appendEntries"); - if (!initializing && lifeCycle.compareAndTransition(STARTING, RUNNING)) { + if (!proto.getInitializing() && lifeCycle.compareAndTransition(State.STARTING, State.RUNNING)) { role.startFollowerState(this, Op.APPEND_ENTRIES); } followerState = updateLastRpcTime(FollowerState.UpdateType.APPEND_START); @@ -1637,20 +1644,23 @@ leaderId, getMemberId(), currentTerm, followerCommit, state.getNextIndex(), NOT_ // 3. There is a gap between the local log and the entries // In any of these scenarios, we should return an INCONSISTENCY reply // back to leader so that the leader can update this follower's next index. - - AppendEntriesReplyProto inconsistencyReply = checkInconsistentAppendEntries( - leaderId, currentTerm, followerCommit, previous, callId, isHeartbeat, entries); - if (inconsistencyReply != null) { + final long inconsistencyReplyNextIndex = checkInconsistentAppendEntries(previous, entries); + if (inconsistencyReplyNextIndex > RaftLog.INVALID_LOG_INDEX) { + final AppendEntriesReplyProto reply = toAppendEntriesReplyProto( + leaderId, getMemberId(), currentTerm, followerCommit, inconsistencyReplyNextIndex, + AppendResult.INCONSISTENCY, callId, RaftLog.INVALID_LOG_INDEX, isHeartbeat); + LOG.info("{}: appendEntries* reply {}", getMemberId(), toAppendEntriesReplyString(reply)); followerState.ifPresent(fs -> fs.updateLastRpcTime(FollowerState.UpdateType.APPEND_COMPLETE)); - return CompletableFuture.completedFuture(inconsistencyReply); + return future.thenApply(dummy -> reply); } state.updateConfiguration(entries); } + future.join(); + final CompletableFuture appendLog = entries.isEmpty()? CompletableFuture.completedFuture(null) + : appendLog(requestRef.delegate(entries)); - final List> futures = entries.isEmpty() ? Collections.emptyList() - : state.getLog().append(entries); - commitInfos.forEach(commitInfoCache::update); + proto.getCommitInfosList().forEach(commitInfoCache::update); CodeInjectionForTesting.execute(LOG_SYNC, getId(), null); if (!isHeartbeat) { @@ -1660,54 +1670,49 @@ leaderId, getMemberId(), currentTerm, followerCommit, state.getNextIndex(), NOT_ stateMachine.event().notifySnapshotInstalled(InstallSnapshotResult.SUCCESS, installedIndex, getPeer()); } } - return JavaUtils.allOf(futures).whenCompleteAsync( - (r, t) -> followerState.ifPresent(fs -> fs.updateLastRpcTime(FollowerState.UpdateType.APPEND_COMPLETE)), - serverExecutor - ).thenApply(v -> { - final AppendEntriesReplyProto reply; - synchronized(this) { - final long commitIndex = ServerImplUtils.effectiveCommitIndex(leaderCommit, previous, entries.size()); - state.updateCommitIndex(commitIndex, currentTerm, false); + + final long commitIndex = effectiveCommitIndex(proto.getLeaderCommit(), previous, entries.size()); + final long matchIndex = isHeartbeat? RaftLog.INVALID_LOG_INDEX: entries.get(entries.size() - 1).getIndex(); + return appendLog.whenCompleteAsync((r, t) -> { + followerState.ifPresent(fs -> fs.updateLastRpcTime(FollowerState.UpdateType.APPEND_COMPLETE)); + timer.stop(); + }, getServerExecutor()).thenApply(v -> { + final boolean updated = state.updateCommitIndex(commitIndex, currentTerm, false); + if (updated) { updateCommitInfoCache(); - final long n; - final long matchIndex; - if (!isHeartbeat) { - LogEntryProto requestLastEntry = entries.get(entries.size() - 1); - n = requestLastEntry.getIndex() + 1; - matchIndex = requestLastEntry.getIndex(); - } else { - n = state.getLog().getNextIndex(); - matchIndex = INVALID_LOG_INDEX; - } - reply = ServerProtoUtils.toAppendEntriesReplyProto(leaderId, getMemberId(), currentTerm, - state.getLog().getLastCommittedIndex(), n, SUCCESS, callId, matchIndex, - isHeartbeat); } - logAppendEntries(isHeartbeat, () -> getMemberId() + ": succeeded to handle AppendEntries. Reply: " - + ServerStringUtils.toAppendEntriesReplyString(reply)); - timer.stop(); // TODO: future never completes exceptionally? + final long nextIndex = isHeartbeat? state.getNextIndex(): matchIndex + 1; + final AppendEntriesReplyProto reply = toAppendEntriesReplyProto(leaderId, getMemberId(), + currentTerm, updated? commitIndex : state.getLog().getLastCommittedIndex(), + nextIndex, AppendResult.SUCCESS, callId, matchIndex, isHeartbeat); + logAppendEntries(isHeartbeat, () -> getMemberId() + + ": appendEntries* reply " + toAppendEntriesReplyString(reply)); return reply; }); } - - private AppendEntriesReplyProto checkInconsistentAppendEntries(RaftPeerId leaderId, long currentTerm, - long followerCommit, TermIndex previous, long callId, boolean isHeartbeat, List entries) { - final long replyNextIndex = checkInconsistentAppendEntries(previous, entries); - if (replyNextIndex == -1) { - return null; + private CompletableFuture appendLog(ReferenceCountedObject> entriesRef) { + final List entriesTermIndices; + try(UncheckedAutoCloseableSupplier> entries = entriesRef.retainAndReleaseOnClose()) { + entriesTermIndices = ConsecutiveIndices.convert(entries.get()); + if (!appendLogTermIndices.append(entriesTermIndices)) { + // index already exists, return the last future + return appendLogFuture.get(); + } } - final AppendEntriesReplyProto reply = ServerProtoUtils.toAppendEntriesReplyProto( - leaderId, getMemberId(), currentTerm, followerCommit, replyNextIndex, INCONSISTENCY, callId, - INVALID_LOG_INDEX, isHeartbeat); - LOG.info("{}: inconsistency entries. Reply:{}", getMemberId(), ServerStringUtils.toAppendEntriesReplyString(reply)); - return reply; + entriesRef.retain(); + return appendLogFuture.updateAndGet(f -> f.thenCompose( + ignored -> JavaUtils.allOf(state.getLog().append(entriesRef)))) + .whenComplete((v, e) -> { + entriesRef.release(); + appendLogTermIndices.removeExisting(entriesTermIndices); + }); } private long checkInconsistentAppendEntries(TermIndex previous, List entries) { // Check if a snapshot installation through state machine is in progress. final long installSnapshot = snapshotInstallationHandler.getInProgressInstallSnapshotIndex(); - if (installSnapshot != INVALID_LOG_INDEX) { + if (installSnapshot != RaftLog.INVALID_LOG_INDEX) { LOG.info("{}: Failed appendEntries as snapshot ({}) installation is in progress", getMemberId(), installSnapshot); return state.getNextIndex(); } @@ -1719,7 +1724,7 @@ private long checkInconsistentAppendEntries(TermIndex previous, List INVALID_LOG_INDEX && nextIndex >= firstEntryIndex) { + if (nextIndex > RaftLog.INVALID_LOG_INDEX && nextIndex >= firstEntryIndex) { LOG.info("{}: Failed appendEntries as the first entry (index {})" + " already exists (snapshotIndex: {}, commitIndex: {})", getMemberId(), firstEntryIndex, snapshotIndex, commitIndex); @@ -1728,13 +1733,13 @@ private long checkInconsistentAppendEntries(TermIndex previous, List replyPendingRequest( - ClientInvocationId invocationId, long logIndex, CompletableFuture stateMachineFuture) { + ClientInvocationId invocationId, TermIndex termIndex, CompletableFuture stateMachineFuture) { // update the retry cache final CacheEntry cacheEntry = retryCache.getOrCreateEntry(invocationId); Objects.requireNonNull(cacheEntry , "cacheEntry == null"); @@ -1849,8 +1849,8 @@ private CompletableFuture replyPendingRequest( } return stateMachineFuture.whenComplete((reply, exception) -> { - transactionManager.remove(logIndex); - final RaftClientReply.Builder b = newReplyBuilder(invocationId, logIndex); + getTransactionManager().remove(termIndex); + final RaftClientReply.Builder b = newReplyBuilder(invocationId, termIndex.getIndex()); final RaftClientReply r; if (exception == null) { r = b.setSuccess().setMessage(reply).build(); @@ -1862,59 +1862,82 @@ private CompletableFuture replyPendingRequest( } // update pending request - role.getLeaderState().ifPresent(leader -> leader.replyPendingRequest(logIndex, r)); + role.getLeaderState().ifPresent(leader -> leader.replyPendingRequest(termIndex, r)); cacheEntry.updateResult(r); }); } + TransactionManager getTransactionManager() { + return transactionManager; + } + + @VisibleForTesting + Map> getTransactionContextMapForTesting() { + return getTransactionManager().getMap(); + } + TransactionContext getTransactionContext(LogEntryProto entry, Boolean createNew) { if (!entry.hasStateMachineLogEntry()) { return null; } + final TermIndex termIndex = TermIndex.valueOf(entry); final Optional leader = getRole().getLeaderState(); if (leader.isPresent()) { - final TransactionContext context = leader.get().getTransactionContext(entry.getIndex()); + final TransactionContext context = leader.get().getTransactionContext(termIndex); if (context != null) { return context; } } if (!createNew) { - return transactionManager.get(entry.getIndex()); + return getTransactionManager().get(termIndex); } - return transactionManager.computeIfAbsent(entry.getIndex(), + return getTransactionManager().computeIfAbsent(termIndex, // call startTransaction only once MemoizedSupplier.valueOf(() -> stateMachine.startTransaction(entry, getInfo().getCurrentRole()))); } - CompletableFuture applyLogToStateMachine(LogEntryProto next) throws RaftLogIOException { - if (!next.hasStateMachineLogEntry()) { - stateMachine.event().notifyTermIndexUpdated(next.getTerm(), next.getIndex()); - } + CompletableFuture applyLogToStateMachine(ReferenceCountedObject nextRef) + throws RaftLogIOException { + LogEntryProto next = nextRef.get(); + CompletableFuture messageFuture = null; - if (next.hasConfigurationEntry()) { + switch (next.getLogEntryBodyCase()) { + case CONFIGURATIONENTRY: // the reply should have already been set. only need to record // the new conf in the metadata file and notify the StateMachine. state.writeRaftConfiguration(next); - stateMachine.event().notifyConfigurationChanged(next.getTerm(), next.getIndex(), next.getConfigurationEntry()); + stateMachine.event().notifyConfigurationChanged(next.getTerm(), next.getIndex(), + next.getConfigurationEntry()); role.getLeaderState().ifPresent(leader -> leader.checkReady(next)); - } else if (next.hasStateMachineLogEntry()) { + break; + case STATEMACHINELOGENTRY: TransactionContext trx = getTransactionContext(next, true); + Objects.requireNonNull(trx, "trx == null"); final ClientInvocationId invocationId = ClientInvocationId.valueOf(next.getStateMachineLogEntry()); writeIndexCache.add(invocationId.getClientId(), ((TransactionContextImpl) trx).getLogIndexFuture()); - + ((TransactionContextImpl) trx).setDelegatedRef(nextRef); try { // Let the StateMachine inject logic for committed transactions in sequential order. trx = stateMachine.applyTransactionSerial(trx); final CompletableFuture stateMachineFuture = stateMachine.applyTransaction(trx); - return replyPendingRequest(invocationId, next.getIndex(), stateMachineFuture); + messageFuture = replyPendingRequest(invocationId, TermIndex.valueOf(next), stateMachineFuture); } catch (Exception e) { throw new RaftLogIOException(e); } + break; + case METADATAENTRY: + break; + default: + throw new IllegalStateException("Unexpected LogEntryBodyCase " + next.getLogEntryBodyCase() + ", next=" + next); } - return null; + + if (next.getLogEntryBodyCase() != LogEntryProto.LogEntryBodyCase.STATEMACHINELOGENTRY) { + stateMachine.event().notifyTermIndexUpdated(next.getTerm(), next.getIndex()); + } + return messageFuture; } /** @@ -1924,7 +1947,10 @@ CompletableFuture applyLogToStateMachine(LogEntryProto next) throws Raf * @param logEntry the log entry being truncated */ void notifyTruncatedLogEntry(LogEntryProto logEntry) { + Optional.ofNullable(getState()).ifPresent(s -> s.truncate(logEntry.getIndex())); if (logEntry.hasStateMachineLogEntry()) { + getTransactionManager().remove(TermIndex.valueOf(logEntry)); + final ClientInvocationId invocationId = ClientInvocationId.valueOf(logEntry.getStateMachineLogEntry()); final CacheEntry cacheEntry = getRetryCache().getIfPresent(invocationId); if (cacheEntry != null) { @@ -1944,51 +1970,11 @@ public RaftServerMetricsImpl getRaftServerMetrics() { return raftServerMetrics; } - private class RaftServerJmxAdapter extends JmxRegister implements RaftServerMXBean { - @Override - public String getId() { - return getMemberId().getPeerId().toString(); - } - - @Override - public String getLeaderId() { - RaftPeerId leaderId = getState().getLeaderId(); - if (leaderId != null) { - return leaderId.toString(); - } else { - return null; - } - } - - @Override - public long getCurrentTerm() { - return getState().getCurrentTerm(); - } - - @Override - public String getGroupId() { - return getMemberId().getGroupId().toString(); - } - - @Override - public String getRole() { - return role.toString(); - } - - @Override - public List getFollowers() { - return role.getLeaderState().map(LeaderStateImpl::getFollowers).orElseGet(Stream::empty) - .map(RaftPeer::toString).collect(Collectors.toList()); - } - - @Override - public List getGroups() { - return proxy.getGroupIds().stream().map(RaftGroupId::toString) - .collect(Collectors.toList()); - } - } - void onGroupLeaderElected() { transferLeadership.complete(TransferLeadership.Result.SUCCESS); } + + boolean isRunning() { + return startComplete.get() && lifeCycle.getCurrentState() == State.RUNNING; + } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerJmxAdapter.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerJmxAdapter.java new file mode 100644 index 0000000000..54fa80150d --- /dev/null +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerJmxAdapter.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.server.impl; + +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.server.RaftServerMXBean; +import org.apache.ratis.util.JmxRegister; + +import javax.management.ObjectName; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** JMX for {@link RaftServerImpl}. */ +class RaftServerJmxAdapter extends JmxRegister implements RaftServerMXBean { + static boolean registerMBean(String id, String groupId, RaftServerMXBean mBean, JmxRegister jmx) { + final String prefix = "Ratis:service=RaftServer,group=" + groupId + ",id="; + final String registered = jmx.register(mBean, Arrays.asList( + () -> prefix + id, + () -> prefix + ObjectName.quote(id))); + return registered != null; + } + + private final RaftServerImpl server; + + RaftServerJmxAdapter(RaftServerImpl server) { + this.server = server; + } + + boolean registerMBean() { + return registerMBean(getId(), getGroupId(), this, this); + } + + @Override + public String getId() { + return server.getId().toString(); + } + + @Override + public String getLeaderId() { + RaftPeerId leaderId = server.getState().getLeaderId(); + if (leaderId != null) { + return leaderId.toString(); + } else { + return null; + } + } + + @Override + public long getCurrentTerm() { + return server.getState().getCurrentTerm(); + } + + @Override + public String getGroupId() { + return server.getMemberId().getGroupId().toString(); + } + + @Override + public String getRole() { + return server.getRole().toString(); + } + + @Override + public List getFollowers() { + return server.getRole().getLeaderState() + .map(LeaderStateImpl::getFollowers) + .orElseGet(Stream::empty) + .map(RaftPeer::toString) + .collect(Collectors.toList()); + } + + @Override + public List getGroups() { + return server.getRaftServer().getGroupIds().stream() + .map(RaftGroupId::toString) + .collect(Collectors.toList()); + } +} diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java index cb7918e51d..2914c434f1 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RaftServerProxy.java @@ -67,7 +67,12 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -138,6 +143,7 @@ private void close(RaftGroupId groupId, CompletableFuture future } catch (Throwable t) { LOG.warn("{}: Failed to close the division for {}", getId(), groupId, t); } + impl.getStateMachine().event().notifyServerShutdown(impl.getRoleInfoProto(), true); } synchronized List getGroupIds() { @@ -415,6 +421,13 @@ private void startImpl() throws IOException { public void close() { lifeCycle.checkStateAndClose(() -> { LOG.info("{}: close", getId()); + + try { + ConcurrentUtils.shutdownAndWait(implExecutor.get()); + } catch (Exception ignored) { + LOG.warn(getId() + ": Failed to shutdown implExecutor", ignored); + } + impls.close(); try { @@ -429,12 +442,6 @@ public void close() { LOG.warn(getId() + ": Failed to close " + SupportedDataStreamType.NETTY + " server", ignored); } - try { - ConcurrentUtils.shutdownAndWait(implExecutor.get()); - } catch (Exception ignored) { - LOG.warn(getId() + ": Failed to shutdown implExecutor", ignored); - } - try { ConcurrentUtils.shutdownAndWait(executor.get()); } catch (Exception ignored) { @@ -645,10 +652,17 @@ public StartLeaderElectionReplyProto startLeaderElection(StartLeaderElectionRequ } @Override - public CompletableFuture appendEntriesAsync(AppendEntriesRequestProto request) { - final RaftGroupId groupId = ProtoUtils.toRaftGroupId(request.getServerRequest().getRaftGroupId()); - return getImplFuture(groupId) - .thenCompose(impl -> impl.executeSubmitServerRequestAsync(() -> impl.appendEntriesAsync(request))); + public CompletableFuture appendEntriesAsync( + ReferenceCountedObject requestRef) { + AppendEntriesRequestProto request = requestRef.retain(); + try { + final RaftGroupId groupId = ProtoUtils.toRaftGroupId(request.getServerRequest().getRaftGroupId()); + return getImplFuture(groupId) + .thenCompose(impl -> JavaUtils.callAsUnchecked( + () -> impl.appendEntriesAsync(requestRef), CompletionException::new)); + } finally { + requestRef.release(); + } } @Override diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java index 3f31a25308..4ff1460d7d 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ReadIndexHeartbeats.java @@ -22,6 +22,8 @@ import org.apache.ratis.server.leader.LogAppender; import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.server.raftlog.RaftLogIndex; +import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,7 +101,7 @@ CompletableFuture getFuture() { boolean receive(LogAppender logAppender, AppendEntriesReplyProto proto, Predicate> hasMajority) { - if (isCompletedNormally()) { + if (JavaUtils.isCompletedNormally(future)) { return true; } @@ -112,22 +114,25 @@ boolean receive(LogAppender logAppender, AppendEntriesReplyProto proto, } } - return isCompletedNormally(); + return JavaUtils.isCompletedNormally(future); } boolean isAcknowledged(RaftPeerId id) { return Optional.ofNullable(replies.get(id)).filter(HeartbeatAck::isAcknowledged).isPresent(); } - - boolean isCompletedNormally() { - return future.isDone() && !future.isCancelled() && !future.isCompletedExceptionally(); - } } class AppendEntriesListeners { private final NavigableMap sorted = new TreeMap<>(); + private Exception exception = null; synchronized AppendEntriesListener add(long commitIndex, Function constructor) { + if (exception != null) { + Preconditions.assertTrue(sorted.isEmpty()); + final AppendEntriesListener listener = constructor.apply(commitIndex); + listener.getFuture().completeExceptionally(exception); + return listener; + } return sorted.computeIfAbsent(commitIndex, constructor); } @@ -155,6 +160,10 @@ synchronized void onAppendEntriesReply(LogAppender appender, AppendEntriesReplyP } synchronized void failAll(Exception e) { + if (exception != null) { + return; + } + exception = e; sorted.forEach((index, listener) -> listener.getFuture().completeExceptionally(e)); sorted.clear(); } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java index a8bac4e5e8..50d238b07a 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RetryCacheImpl.java @@ -67,7 +67,7 @@ boolean isDone() { } boolean isCompletedNormally() { - return !failed && replyFuture.isDone() && !replyFuture.isCompletedExceptionally() && !replyFuture.isCancelled(); + return !failed && JavaUtils.isCompletedNormally(replyFuture); } void updateResult(RaftClientReply reply) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/RoleInfo.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/RoleInfo.java index fe2bc963b1..a5cd7da665 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/RoleInfo.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/RoleInfo.java @@ -18,8 +18,14 @@ package org.apache.ratis.server.impl; +import org.apache.ratis.proto.RaftProtos.CandidateInfoProto; +import org.apache.ratis.proto.RaftProtos.FollowerInfoProto; +import org.apache.ratis.proto.RaftProtos.LeaderInfoProto; import org.apache.ratis.proto.RaftProtos.RaftPeerRole; +import org.apache.ratis.proto.RaftProtos.RoleInfoProto; +import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.server.leader.LogAppender; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.Timestamp; @@ -32,6 +38,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static org.apache.ratis.server.impl.ServerProtoUtils.toServerRpcProto; + /** * Maintain the Role of a Raft Peer. */ @@ -39,7 +47,7 @@ class RoleInfo { public static final Logger LOG = LoggerFactory.getLogger(RoleInfo.class); private final RaftPeerId id; - private volatile RaftPeerRole role; + private final AtomicReference role = new AtomicReference<>(); /** Used when the peer is leader */ private final AtomicReference leaderState = new AtomicReference<>(); /** Used when the peer is follower, to monitor election timeout */ @@ -56,7 +64,7 @@ class RoleInfo { } void transitionRole(RaftPeerRole newRole) { - this.role = newRole; + this.role.set(newRole); this.transitionTime.set(Timestamp.currentTime()); } @@ -65,7 +73,7 @@ long getRoleElapsedTimeMs() { } RaftPeerRole getCurrentRole() { - return role; + return role.get(); } boolean isLeaderReady() { @@ -105,13 +113,13 @@ void startFollowerState(RaftServerImpl server, Object reason) { updateAndGet(followerState, new FollowerState(server, reason)).start(); } - void shutdownFollowerState() { + CompletableFuture shutdownFollowerState() { final FollowerState follower = followerState.getAndSet(null); - if (follower != null) { - LOG.info("{}: shutdown {}", id, follower); - follower.stopRunning(); - follower.interrupt(); + if (follower == null) { + return CompletableFuture.completedFuture(null); } + LOG.info("{}: shutdown {}", id, follower); + return follower.stopRunning(); } void startLeaderElection(RaftServerImpl server, boolean force) { @@ -125,13 +133,13 @@ void setLeaderElectionPause(boolean pause) { pauseLeaderElection.set(pause); } - void shutdownLeaderElection() { + CompletableFuture shutdownLeaderElection() { final LeaderElection election = leaderElection.getAndSet(null); - if (election != null) { - LOG.info("{}: shutdown {}", id, election); - election.shutdown(); - // no need to interrupt the election thread + if (election == null) { + return CompletableFuture.completedFuture(null); } + LOG.info("{}: shutdown {}", id, election); + return election.shutdown(); } private T updateAndGet(AtomicReference ref, T current) { @@ -141,6 +149,55 @@ private T updateAndGet(AtomicReference ref, T current) { return updated; } + RoleInfoProto buildRoleInfoProto(RaftServerImpl server) { + final RaftPeerRole currentRole = getCurrentRole(); + final RoleInfoProto.Builder proto = RoleInfoProto.newBuilder() + .setSelf(server.getPeer().getRaftPeerProto()) + .setRole(currentRole) + .setRoleElapsedTimeMs(getRoleElapsedTimeMs()); + + switch (currentRole) { + case LEADER: + getLeaderState().ifPresent(leader -> { + final LeaderInfoProto.Builder b = LeaderInfoProto.newBuilder() + .setTerm(leader.getCurrentTerm()); + leader.getLogAppenders() + .map(LogAppender::getFollower) + .map(f -> toServerRpcProto(f.getPeer(), f.getLastRpcResponseTime().elapsedTimeMs())) + .forEach(b::addFollowerInfo); + proto.setLeaderInfo(b); + }); + return proto.build(); + + case CANDIDATE: + return proto.setCandidateInfo(CandidateInfoProto.newBuilder() + .setLastLeaderElapsedTimeMs(server.getState().getLastLeaderElapsedTimeMs())) + .build(); + + case LISTENER: + case FOLLOWER: + // FollowerState can be null while adding a new peer as it is not a voting member yet + final FollowerState follower = getFollowerState().orElse(null); + final long rpcElapsed; + final int outstandingOp; + if (follower != null) { + rpcElapsed = follower.getLastRpcTime().elapsedTimeMs(); + outstandingOp = follower.getOutstandingOp(); + } else { + rpcElapsed = 0; + outstandingOp = 0; + } + final RaftPeer leader = server.getRaftConf().getPeer(server.getState().getLeaderId()); + return proto.setFollowerInfo(FollowerInfoProto.newBuilder() + .setLeaderInfo(toServerRpcProto(leader, rpcElapsed)) + .setOutstandingOp(outstandingOp)) + .build(); + + default: + throw new IllegalStateException("Unexpected role " + currentRole); + } + } + @Override public String toString() { return String.format("%9s", role); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java index e4fe8f232f..864b402a23 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerImplUtils.java @@ -19,9 +19,15 @@ import org.apache.ratis.conf.Parameters; import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.exceptions.GroupMismatchException; import org.apache.ratis.server.RaftConfiguration; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.protocol.TermIndex; @@ -34,12 +40,127 @@ import org.apache.ratis.util.TimeDuration; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.Objects; import java.util.concurrent.TimeUnit; /** Server utilities for internal use. */ public final class ServerImplUtils { + /** The consecutive indices within the same term. */ + static class ConsecutiveIndices { + /** Convert the given entries to a list of {@link ConsecutiveIndices} */ + static List convert(List entries) { + if (entries == null || entries.isEmpty()) { + return Collections.emptyList(); + } + + List indices = null; + + LogEntryProto previous = entries.get(0); + long startIndex = previous.getIndex(); + int count = 1; + + for (int i = 1; i < entries.size(); i++) { + final LogEntryProto current = entries.get(i); + // validate if the indices are consecutive + Preconditions.assertSame(previous.getIndex() + 1, current.getIndex(), "index"); + + if (current.getTerm() == previous.getTerm()) { + count++; + } else { + // validate if the terms are increasing + Preconditions.assertTrue(previous.getTerm() < current.getTerm(), "term"); + if (indices == null) { + indices = new ArrayList<>(); + } + indices.add(new ConsecutiveIndices(previous.getTerm(), startIndex, count)); + + startIndex = current.getIndex(); + count = 1; + } + previous = current; + } + + final ConsecutiveIndices last = new ConsecutiveIndices(previous.getTerm(), startIndex, count); + if (indices == null) { + return Collections.singletonList(last); + } else { + indices.add(last); + return indices; + } + } + + private final long term; + private final long startIndex; + private final int count; + + ConsecutiveIndices(long term, long startIndex, int count) { + Preconditions.assertTrue(count > 0, () -> "count = " + count + " <= 0 "); + this.term = term; + this.startIndex = startIndex; + this.count = count; + } + + long getNextIndex() { + return startIndex + count; + } + + Long getTerm(long index) { + final long diff = index - startIndex; + return diff < 0 || diff >= count ? null: term; + } + } + + /** A data structure to support the {@link #contains(TermIndex)} method. */ + static class NavigableIndices { + private final NavigableMap map = new TreeMap<>(); + + boolean contains(TermIndex ti) { + final Long term = getTerm(ti.getIndex()); + return term != null && term == ti.getTerm(); + } + + synchronized Long getTerm(long index) { + if (map.isEmpty()) { + return null; + } + + final Map.Entry floorEntry = map.floorEntry(index); + if (floorEntry == null) { + return null; + } + return floorEntry.getValue().getTerm(index); + } + + synchronized boolean append(List entriesTermIndices) { + for(int i = 0; i < entriesTermIndices.size(); i++) { + final ConsecutiveIndices indices = entriesTermIndices.get(i); + final ConsecutiveIndices previous = map.put(indices.startIndex, indices); + if (previous != null) { + // index already exists, revert this append + map.put(previous.startIndex, previous); + for(int j = 0; j < i; j++) { + map.remove(entriesTermIndices.get(j).startIndex); + } + return false; + } + } + return true; + } + + synchronized void removeExisting(List entriesTermIndices) { + for(ConsecutiveIndices indices : entriesTermIndices) { + final ConsecutiveIndices removed = map.remove(indices.startIndex); + Preconditions.assertSame(indices, removed, "removed"); + } + } + } + private ServerImplUtils() { //Never constructed } @@ -50,8 +171,8 @@ public static RaftServerProxy newRaftServer( ThreadGroup threadGroup, RaftProperties properties, Parameters parameters) throws IOException { RaftServer.LOG.debug("newRaftServer: {}, {}", id, group); if (group != null && !group.getPeers().isEmpty()) { - Preconditions.assertNotNull(id, "RaftPeerId %s is not in RaftGroup %s", id, group); - Preconditions.assertNotNull(group.getPeer(id), "RaftPeerId %s is not in RaftGroup %s", id, group); + Objects.requireNonNull(id, () -> "RaftPeerId " + id + " is not in RaftGroup " + group); + Objects.requireNonNull(group.getPeer(id), () -> "RaftPeerId " + id + " is not in RaftGroup " + group); } final RaftServerProxy proxy = newRaftServer(id, stateMachineRegistry, threadGroup, properties, parameters); proxy.initGroups(group, option); @@ -88,7 +209,51 @@ public static RaftConfiguration newRaftConfiguration(List conf, List entries = proto.getEntriesList(); + if (entries != null && !entries.isEmpty()) { + final long index0 = entries.get(0).getIndex(); + // Check if next entry's index is 1 greater than the snapshotIndex. If yes, then + // we do not have to check for the existence of previous. + if (index0 != state.getSnapshotIndex() + 1) { + final long expected = previous == null || previous.getTerm() == 0 ? 0 : previous.getIndex() + 1; + Preconditions.assertTrue(index0 == expected, + "Unexpected Index: previous is %s but entries[%s].getIndex() == %s != %s", + previous, 0, index0, expected); + } + + final long leaderTerm = proto.getLeaderTerm(); + for (int i = 0; i < entries.size(); i++) { + final LogEntryProto entry = entries.get(i); + final long entryTerm = entry.getTerm(); + Preconditions.assertTrue(entryTerm <= leaderTerm , + "Unexpected Term: entries[%s].getTerm() == %s > leaderTerm == %s", + i, entryTerm, leaderTerm); + + final long indexI = entry.getIndex(); + final long expected = index0 + i; + Preconditions.assertTrue(indexI == expected, + "Unexpected Index: entries[0].getIndex() == %s but entries[%s].getIndex() == %s != %s", + index0, i, indexI, expected); + } + } } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerProtoUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerProtoUtils.java index f2be8c61c6..e6a29189a8 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerProtoUtils.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerProtoUtils.java @@ -43,11 +43,13 @@ private static RaftRpcReplyProto.Builder toRaftRpcReplyProtoBuilder( } static RequestVoteReplyProto toRequestVoteReplyProto( - RaftPeerId requestorId, RaftGroupMemberId replyId, boolean success, long term, boolean shouldShutdown) { + RaftPeerId requestorId, RaftGroupMemberId replyId, boolean success, long term, boolean shouldShutdown, + TermIndex lastEntry) { return RequestVoteReplyProto.newBuilder() .setServerReply(toRaftRpcReplyProtoBuilder(requestorId, replyId, success)) .setTerm(term) .setShouldShutdown(shouldShutdown) + .setLastEntry((lastEntry != null? lastEntry : TermIndex.INITIAL_VALUE).toProto()) .build(); } @@ -114,7 +116,7 @@ static ReadIndexRequestProto toReadIndexRequestProto( RaftClientRequest clientRequest, RaftGroupMemberId requestorId, RaftPeerId replyId) { return ReadIndexRequestProto.newBuilder() .setServerRequest(ClientProtoUtils.toRaftRpcRequestProtoBuilder(requestorId, replyId)) - .setClientRequest(ClientProtoUtils.toRaftClientRequestProto(clientRequest)) + .setClientRequest(ClientProtoUtils.toRaftClientRequestProto(clientRequest, false)) .build(); } @@ -126,6 +128,10 @@ static ReadIndexReplyProto toReadIndexReplyProto( .build(); } + static ReadIndexReplyProto toReadIndexReplyProto(RaftPeerId requestorId, RaftGroupMemberId replyId) { + return toReadIndexReplyProto(requestorId, replyId, false, RaftLog.INVALID_LOG_INDEX); + } + @SuppressWarnings("parameternumber") static AppendEntriesReplyProto toAppendEntriesReplyProto( RaftPeerId requestorId, RaftGroupMemberId replyId, long term, diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java index e21f63caa6..c49e9554f0 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/ServerState.java @@ -87,6 +87,7 @@ class ServerState { /** * Candidate that this peer granted vote for in current term (or null if none) */ + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile RaftPeerId votedFor; /** @@ -158,6 +159,7 @@ void writeRaftConfiguration(LogEntryProto conf) { } void start() { + // initialize stateMachineUpdater stateMachineUpdater.get().start(); } @@ -170,6 +172,7 @@ private RaftLog initRaftLog(LongSupplier getSnapshotIndexFromStateMachine, RaftP } } + @SuppressWarnings({"squid:S2095"}) // Suppress closeable warning private static RaftLog initRaftLog(RaftGroupMemberId memberId, RaftServerImpl server, RaftStorage storage, Consumer logConsumer, LongSupplier getSnapshotIndexFromStateMachine, RaftProperties prop) throws IOException { @@ -318,27 +321,24 @@ TermIndex getLastEntry() { void appendLog(TransactionContext operation) throws StateMachineException { getLog().append(currentTerm.get(), operation); - Objects.requireNonNull(operation.getLogEntry()); + Objects.requireNonNull(operation.getLogEntryUnsafe(), "transaction-logEntry"); } - /** - * Check if accept the leader selfId and term from the incoming AppendEntries rpc. - * If accept, update the current state. - * @return true if the check passes - */ - boolean recognizeLeader(RaftPeerId peerLeaderId, long leaderTerm) { + /** @return true iff the given peer id is recognized as the leader. */ + boolean recognizeLeader(Object op, RaftPeerId peerId, long peerTerm) { final long current = currentTerm.get(); - if (leaderTerm < current) { + if (peerTerm < current) { + LOG.warn("{}: Failed to recognize {} as leader for {} since peerTerm = {} < currentTerm = {}", + getMemberId(), peerId, op, peerTerm, current); return false; } final RaftPeerId curLeaderId = getLeaderId(); - if (leaderTerm > current || curLeaderId == null) { - // If the request indicates a term that is greater than the current term - // or no leader has been set for the current term, make sure to update - // leader and term later - return true; + if (peerTerm == current && curLeaderId != null && !curLeaderId.equals(peerId)) { + LOG.warn("{}: Failed to recognize {} as leader for {} since current leader is {} (peerTerm = currentTerm = {})", + getMemberId(), peerId, op, curLeaderId, current); + return false; } - return curLeaderId.equals(peerLeaderId); + return true; } static int compareLog(TermIndex lastEntry, TermIndex candidateLastEntry) { @@ -387,6 +387,10 @@ void setRaftConf(RaftConfiguration conf) { LOG.trace("{}: {}", getMemberId(), configurationManager); } + void truncate(long logIndex) { + configurationManager.removeConfigurations(logIndex); + } + void updateConfiguration(List entries) { if (entries != null && !entries.isEmpty()) { configurationManager.removeConfigurations(entries.get(0).getIndex()); @@ -424,7 +428,6 @@ void close() { } LOG.warn(getMemberId() + ": Failed to join " + getStateMachineUpdater(), e); } - LOG.info("{}: applyIndex: {}", getMemberId(), getLastAppliedIndex()); try { if (log.isInitialized()) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/SnapshotInstallationHandler.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/SnapshotInstallationHandler.java index 9794314b83..4f1ac4177f 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/SnapshotInstallationHandler.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/SnapshotInstallationHandler.java @@ -32,9 +32,11 @@ import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.protocol.RaftServerProtocol; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.LogProtoUtils; import org.apache.ratis.server.util.ServerStringUtils; +import org.apache.ratis.util.BatchLogger; import org.apache.ratis.util.CodeInjectionForTesting; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.Preconditions; @@ -45,15 +47,24 @@ import java.io.IOException; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import static org.apache.ratis.server.impl.ServerProtoUtils.toInstallSnapshotReplyProto; +import static org.apache.ratis.server.impl.ServerProtoUtils.toServerRpcProto; import static org.apache.ratis.server.raftlog.RaftLog.INVALID_LOG_INDEX; class SnapshotInstallationHandler { static final Logger LOG = LoggerFactory.getLogger(SnapshotInstallationHandler.class); + private enum BatchLogKey implements BatchLogger.Key { + INSTALL_SNAPSHOT_REQUEST, + INSTALL_SNAPSHOT_REPLY + } + static final TermIndex INVALID_TERM_INDEX = TermIndex.valueOf(0, INVALID_LOG_INDEX); private final RaftServerImpl server; @@ -65,6 +76,9 @@ class SnapshotInstallationHandler { new AtomicReference<>(INVALID_TERM_INDEX); private final AtomicBoolean isSnapshotNull = new AtomicBoolean(); private final AtomicLong installedIndex = new AtomicLong(INVALID_LOG_INDEX); + private final AtomicInteger nextChunkIndex = new AtomicInteger(-1); + /** The callId of the chunk with index 0. */ + private final AtomicLong chunk0CallId = new AtomicLong(-1); SnapshotInstallationHandler(RaftServerImpl server, RaftProperties properties) { this.server = server; @@ -85,10 +99,9 @@ long getInProgressInstallSnapshotIndex() { } InstallSnapshotReplyProto installSnapshot(InstallSnapshotRequestProto request) throws IOException { - if (LOG.isInfoEnabled()) { - LOG.info("{}: receive installSnapshot: {}", getMemberId(), - ServerStringUtils.toInstallSnapshotRequestString(request)); - } + BatchLogger.print(BatchLogKey.INSTALL_SNAPSHOT_REQUEST, getMemberId(), + suffix -> LOG.info("{}: receive installSnapshot: {} {}", + getMemberId(), ServerStringUtils.toInstallSnapshotRequestString(request), suffix)); final InstallSnapshotReplyProto reply; try { reply = installSnapshotImpl(request); @@ -96,10 +109,9 @@ InstallSnapshotReplyProto installSnapshot(InstallSnapshotRequestProto request) t LOG.error("{}: installSnapshot failed", getMemberId(), e); throw e; } - if (LOG.isInfoEnabled()) { - LOG.info("{}: reply installSnapshot: {}", getMemberId(), - ServerStringUtils.toInstallSnapshotReplyString(reply)); - } + BatchLogger.print(BatchLogKey.INSTALL_SNAPSHOT_REPLY, getMemberId(), + suffix -> LOG.info("{}: reply installSnapshot: {} {}", + getMemberId(), ServerStringUtils.toInstallSnapshotReplyString(reply), suffix)); return reply; } @@ -110,19 +122,19 @@ private InstallSnapshotReplyProto installSnapshotImpl(InstallSnapshotRequestProt CodeInjectionForTesting.execute(RaftServerImpl.INSTALL_SNAPSHOT, server.getId(), leaderId, request); server.assertLifeCycleState(LifeCycle.States.STARTING_OR_RUNNING); - server.assertGroup(leaderId, leaderGroupId); + ServerImplUtils.assertGroup(getMemberId(), leaderId, leaderGroupId); InstallSnapshotReplyProto reply = null; // Check if install snapshot from Leader is enabled if (installSnapshotEnabled) { // Leader has sent InstallSnapshot request with SnapshotInfo. Install the snapshot. if (request.hasSnapshotChunk()) { - reply = checkAndInstallSnapshot(request, leaderId); + reply = checkAndInstallSnapshot(request, leaderId).join(); } } else { // Leader has only sent a notification to install snapshot. Inform State Machine to install snapshot. if (request.hasNotification()) { - reply = notifyStateMachineToInstallSnapshot(request, leaderId); + reply = notifyStateMachineToInstallSnapshot(request, leaderId).join(); } } @@ -130,6 +142,7 @@ private InstallSnapshotReplyProto installSnapshotImpl(InstallSnapshotRequestProt if (request.hasLastRaftConfigurationLogEntryProto()) { // Set the configuration included in the snapshot final LogEntryProto proto = request.getLastRaftConfigurationLogEntryProto(); + state.truncate(proto.getIndex()); if (!state.getRaftConf().equals(LogProtoUtils.toRaftConfiguration(proto))) { LOG.info("{}: set new configuration {} from snapshot", getMemberId(), proto); state.setRaftConf(proto); @@ -142,7 +155,7 @@ private InstallSnapshotReplyProto installSnapshotImpl(InstallSnapshotRequestProt } // There is a mismatch between configurations on leader and follower. - final InstallSnapshotReplyProto failedReply = ServerProtoUtils.toInstallSnapshotReplyProto( + final InstallSnapshotReplyProto failedReply = toInstallSnapshotReplyProto( leaderId, getMemberId(), state.getCurrentTerm(), InstallSnapshotResult.CONF_MISMATCH); LOG.error("{}: Configuration Mismatch ({}): Leader {} has it set to {} but follower {} has it set to {}", getMemberId(), RaftServerConfigKeys.Log.Appender.INSTALL_SNAPSHOT_ENABLED_KEY, @@ -150,41 +163,69 @@ private InstallSnapshotReplyProto installSnapshotImpl(InstallSnapshotRequestProt return failedReply; } - private InstallSnapshotReplyProto checkAndInstallSnapshot(InstallSnapshotRequestProto request, + private CompletableFuture checkAndInstallSnapshot(InstallSnapshotRequestProto request, RaftPeerId leaderId) throws IOException { final long currentTerm; final long leaderTerm = request.getLeaderTerm(); final InstallSnapshotRequestProto.SnapshotChunkProto snapshotChunkRequest = request.getSnapshotChunk(); final TermIndex lastIncluded = TermIndex.valueOf(snapshotChunkRequest.getTermIndex()); final long lastIncludedIndex = lastIncluded.getIndex(); + final CompletableFuture future; synchronized (server) { - final boolean recognized = state.recognizeLeader(leaderId, leaderTerm); + final boolean recognized = state.recognizeLeader(RaftServerProtocol.Op.INSTALL_SNAPSHOT, leaderId, leaderTerm); currentTerm = state.getCurrentTerm(); if (!recognized) { - final InstallSnapshotReplyProto reply = ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), - currentTerm, snapshotChunkRequest.getRequestIndex(), InstallSnapshotResult.NOT_LEADER); - LOG.warn("{}: Failed to recognize leader for installSnapshot chunk.", getMemberId()); - return reply; + return CompletableFuture.completedFuture(toInstallSnapshotReplyProto(leaderId, getMemberId(), + currentTerm, snapshotChunkRequest.getRequestIndex(), InstallSnapshotResult.NOT_LEADER)); } - server.changeToFollowerAndPersistMetadata(leaderTerm, true, "installSnapshot"); + future = server.changeToFollowerAndPersistMetadata(leaderTerm, true, "installSnapshot"); state.setLeader(leaderId, "installSnapshot"); server.updateLastRpcTime(FollowerState.UpdateType.INSTALL_SNAPSHOT_START); + long callId = chunk0CallId.get(); + // 1. leaderTerm < currentTerm will never come here + // 2. leaderTerm == currentTerm && callId == request.getCallId() + // means the snapshotRequest is staled with the same leader + // 3. leaderTerm > currentTerm means this is a new snapshot request from a new leader, + // chunk0CallId will be reset when a snapshot request with requestIndex == 0 is received . + if (callId > request.getServerRequest().getCallId() && currentTerm == leaderTerm) { + LOG.warn("{}: Snapshot Request Staled: chunk 0 callId is {} but {}", getMemberId(), callId, + ServerStringUtils.toInstallSnapshotRequestString(request)); + InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), + currentTerm, snapshotChunkRequest.getRequestIndex(), InstallSnapshotResult.SNAPSHOT_EXPIRED); + return future.thenApply(dummy -> reply); + } + if (snapshotChunkRequest.getRequestIndex() == 0) { + nextChunkIndex.set(0); + chunk0CallId.set(request.getServerRequest().getCallId()); + } else if (nextChunkIndex.get() != snapshotChunkRequest.getRequestIndex()) { + throw new IOException("Snapshot request already failed at chunk index " + nextChunkIndex.get() + + "; ignoring request with chunk index " + snapshotChunkRequest.getRequestIndex()); + } try { // Check and append the snapshot chunk. We simply put this in lock // considering a follower peer requiring a snapshot installation does not // have a lot of requests - Preconditions.assertTrue(state.getLog().getLastCommittedIndex() < lastIncludedIndex, - "%s log's commit index is %s, last included index in snapshot is %s", - getMemberId(), state.getLog().getLastCommittedIndex(), lastIncludedIndex); + if (state.getLog().getLastCommittedIndex() >= lastIncludedIndex) { + nextChunkIndex.set(snapshotChunkRequest.getRequestIndex() + 1); + final InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), + currentTerm, snapshotChunkRequest.getRequestIndex(), InstallSnapshotResult.ALREADY_INSTALLED); + return future.thenApply(dummy -> reply); + } //TODO: We should only update State with installed snapshot once the request is done. state.installSnapshot(request); + final int expectedChunkIndex = nextChunkIndex.getAndIncrement(); + if (expectedChunkIndex != snapshotChunkRequest.getRequestIndex()) { + throw new IOException("Unexpected request chunk index: " + snapshotChunkRequest.getRequestIndex() + + " (the expected index is " + expectedChunkIndex + ")"); + } // update the committed index // re-load the state machine if this is the last chunk if (snapshotChunkRequest.getDone()) { state.reloadStateMachine(lastIncluded); + chunk0CallId.set(-1); } } finally { server.updateLastRpcTime(FollowerState.UpdateType.INSTALL_SNAPSHOT_COMPLETE); @@ -193,27 +234,27 @@ private InstallSnapshotReplyProto checkAndInstallSnapshot(InstallSnapshotRequest if (snapshotChunkRequest.getDone()) { LOG.info("{}: successfully install the entire snapshot-{}", getMemberId(), lastIncludedIndex); } - return ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), + final InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), currentTerm, snapshotChunkRequest.getRequestIndex(), InstallSnapshotResult.SUCCESS); + return future.thenApply(dummy -> reply); } - private InstallSnapshotReplyProto notifyStateMachineToInstallSnapshot( + private CompletableFuture notifyStateMachineToInstallSnapshot( InstallSnapshotRequestProto request, RaftPeerId leaderId) throws IOException { final long currentTerm; final long leaderTerm = request.getLeaderTerm(); final TermIndex firstAvailableLogTermIndex = TermIndex.valueOf( request.getNotification().getFirstAvailableTermIndex()); final long firstAvailableLogIndex = firstAvailableLogTermIndex.getIndex(); + final CompletableFuture future; synchronized (server) { - final boolean recognized = state.recognizeLeader(leaderId, leaderTerm); + final boolean recognized = state.recognizeLeader("notifyInstallSnapshot", leaderId, leaderTerm); currentTerm = state.getCurrentTerm(); if (!recognized) { - final InstallSnapshotReplyProto reply = ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), - currentTerm, InstallSnapshotResult.NOT_LEADER); - LOG.warn("{}: Failed to recognize leader for installSnapshot notification.", getMemberId()); - return reply; + return CompletableFuture.completedFuture(toInstallSnapshotReplyProto(leaderId, getMemberId(), + currentTerm, InstallSnapshotResult.NOT_LEADER)); } - server.changeToFollowerAndPersistMetadata(leaderTerm, true, "installSnapshot"); + future = server.changeToFollowerAndPersistMetadata(leaderTerm, true, "installSnapshot"); state.setLeader(leaderId, "installSnapshot"); server.updateLastRpcTime(FollowerState.UpdateType.INSTALL_SNAPSHOT_NOTIFICATION); @@ -222,15 +263,17 @@ private InstallSnapshotReplyProto notifyStateMachineToInstallSnapshot( // Check if snapshot index is already at par or ahead of the first // available log index of the Leader. final long snapshotIndex = state.getLog().getSnapshotIndex(); - if (snapshotIndex + 1 >= firstAvailableLogIndex && firstAvailableLogIndex > INVALID_LOG_INDEX) { + if (snapshotIndex != INVALID_LOG_INDEX && snapshotIndex + 1 >= firstAvailableLogIndex && + firstAvailableLogIndex > INVALID_LOG_INDEX) { // State Machine has already installed the snapshot. Return the // latest snapshot index to the Leader. inProgressInstallSnapshotIndex.compareAndSet(firstAvailableLogIndex, INVALID_LOG_INDEX); LOG.info("{}: InstallSnapshot notification result: {}, current snapshot index: {}", getMemberId(), InstallSnapshotResult.ALREADY_INSTALLED, snapshotIndex); - return ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), currentTerm, + final InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), currentTerm, InstallSnapshotResult.ALREADY_INSTALLED, snapshotIndex); + return future.thenApply(dummy -> reply); } final RaftPeerProto leaderProto; @@ -307,8 +350,9 @@ private InstallSnapshotReplyProto notifyStateMachineToInstallSnapshot( inProgressInstallSnapshotIndex.set(INVALID_LOG_INDEX); server.getStateMachine().event().notifySnapshotInstalled( InstallSnapshotResult.SNAPSHOT_UNAVAILABLE, INVALID_LOG_INDEX, server.getPeer()); - return ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), + final InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), currentTerm, InstallSnapshotResult.SNAPSHOT_UNAVAILABLE); + return future.thenApply(dummy -> reply); } // If a snapshot has been installed, return SNAPSHOT_INSTALLED with the installed snapshot index and reset @@ -325,8 +369,9 @@ private InstallSnapshotReplyProto notifyStateMachineToInstallSnapshot( server.getStateMachine().event().notifySnapshotInstalled( InstallSnapshotResult.SNAPSHOT_INSTALLED, latestInstalledIndex, server.getPeer()); installedIndex.set(latestInstalledIndex); - return ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), + final InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), currentTerm, InstallSnapshotResult.SNAPSHOT_INSTALLED, latestInstalledSnapshotTermIndex.getIndex()); + return future.thenApply(dummy -> reply); } // Otherwise, Snapshot installation is in progress. @@ -334,15 +379,16 @@ private InstallSnapshotReplyProto notifyStateMachineToInstallSnapshot( LOG.debug("{}: InstallSnapshot notification result: {}", getMemberId(), InstallSnapshotResult.IN_PROGRESS); } - return ServerProtoUtils.toInstallSnapshotReplyProto(leaderId, getMemberId(), + final InstallSnapshotReplyProto reply = toInstallSnapshotReplyProto(leaderId, getMemberId(), currentTerm, InstallSnapshotResult.IN_PROGRESS); + return future.thenApply(dummy -> reply); } } private RoleInfoProto getRoleInfoProto(RaftPeer leader) { final RoleInfo role = server.getRole(); final Optional fs = role.getFollowerState(); - final ServerRpcProto leaderInfo = ServerProtoUtils.toServerRpcProto(leader, + final ServerRpcProto leaderInfo = toServerRpcProto(leader, fs.map(FollowerState::getLastRpcTime).map(Timestamp::elapsedTimeMs).orElse(0L)); final FollowerInfoProto.Builder followerInfo = FollowerInfoProto.newBuilder() .setLeaderInfo(leaderInfo) diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java index 43fbdd8843..b7c661235c 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/StateMachineUpdater.java @@ -29,6 +29,7 @@ import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.server.raftlog.RaftLogIndex; +import org.apache.ratis.server.util.ServerStringUtils; import org.apache.ratis.statemachine.SnapshotInfo; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.SnapshotRetentionPolicy; @@ -37,8 +38,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -74,6 +73,8 @@ enum State { private final boolean triggerSnapshotWhenStopEnabled; + private final boolean triggerSnapshotWhenRemoveEnabled; + private final Long autoSnapshotThreshold; private final boolean purgeUptoSnapshotIndex; @@ -91,9 +92,11 @@ enum State { private final Consumer appliedIndexConsumer; + private volatile boolean isRemoving; + StateMachineUpdater(StateMachine stateMachine, RaftServerImpl server, ServerState serverState, long lastAppliedIndex, RaftProperties properties, Consumer appliedIndexConsumer) { - this.name = serverState.getMemberId() + "-" + JavaUtils.getClassSimpleName(getClass()); + this.name = ServerStringUtils.generateUnifiedName(serverState.getMemberId(), getClass()); this.appliedIndexConsumer = appliedIndexConsumer; this.infoIndexChange = s -> LOG.info("{}: {}", name, s); this.debugIndexChange = s -> LOG.debug("{}: {}", name, s); @@ -106,6 +109,7 @@ enum State { this.snapshotIndex = new RaftLogIndex("snapshotIndex", lastAppliedIndex); this.triggerSnapshotWhenStopEnabled = RaftServerConfigKeys.Snapshot.triggerWhenStopEnabled(properties); + this.triggerSnapshotWhenRemoveEnabled = RaftServerConfigKeys.Snapshot.triggerWhenRemoveEnabled(properties); final boolean autoSnapshot = RaftServerConfigKeys.Snapshot.autoTriggerEnabled(properties); this.autoSnapshotThreshold = autoSnapshot? RaftServerConfigKeys.Snapshot.autoTriggerThreshold(properties): null; final int numSnapshotFilesRetained = RaftServerConfigKeys.Snapshot.retentionFileNum(properties); @@ -133,6 +137,8 @@ void start() { private void stop() { state = State.STOP; try { + LOG.info("{}: closing {}, lastApplied={}", name, + JavaUtils.getClassSimpleName(stateMachine.getClass()), stateMachine.getLastAppliedTermIndex()); stateMachine.close(); if (stateMachineMetrics.isInitialized()) { stateMachineMetrics.get().unregister(); @@ -175,23 +181,25 @@ public String toString() { @Override public void run() { + CompletableFuture applyLogFutures = CompletableFuture.completedFuture(null); for(; state != State.STOP; ) { try { - waitForCommit(); + waitForCommit(applyLogFutures); if (state == State.RELOAD) { reload(); } - final MemoizedSupplier>> futures = applyLog(); - checkAndTakeSnapshot(futures); + applyLogFutures = applyLog(applyLogFutures); + checkAndTakeSnapshot(applyLogFutures); if (shouldStop()) { - checkAndTakeSnapshot(futures); + applyLogFutures.get(); stop(); } } catch (Throwable t) { if (t instanceof InterruptedException && state == State.STOP) { + Thread.currentThread().interrupt(); LOG.info("{} was interrupted. Exiting ...", this); } else { state = State.EXCEPTION; @@ -202,12 +210,15 @@ public void run() { } } - private void waitForCommit() throws InterruptedException { + private void waitForCommit(CompletableFuture applyLogFutures) throws InterruptedException, ExecutionException { // When a peer starts, the committed is initialized to 0. // It will be updated only after the leader contacts other peers. // Thus it is possible to have applied > committed initially. final long applied = getLastAppliedIndex(); for(; applied >= raftLog.getLastCommittedIndex() && state == State.RUNNING && !shouldStop(); ) { + if (server.getSnapshotRequestHandler().shouldTriggerTakingSnapshot()) { + takeSnapshot(applyLogFutures); + } if (awaitForSignal.await(100, TimeUnit.MILLISECONDS)) { return; } @@ -228,15 +239,21 @@ private void reload() throws IOException { state = State.RUNNING; } - private MemoizedSupplier>> applyLog() throws RaftLogIOException { - final MemoizedSupplier>> futures = MemoizedSupplier.valueOf(ArrayList::new); + private CompletableFuture applyLog(CompletableFuture applyLogFutures) throws RaftLogIOException { final long committed = raftLog.getLastCommittedIndex(); for(long applied; (applied = getLastAppliedIndex()) < committed && state == State.RUNNING && !shouldStop(); ) { final long nextIndex = applied + 1; - final LogEntryProto next = raftLog.get(nextIndex); - if (next != null) { + final ReferenceCountedObject next = raftLog.retainLog(nextIndex); + if (next == null) { + LOG.debug("{}: logEntry {} is null. There may be snapshot to load. state:{}", + this, nextIndex, state); + break; + } + + try { + final LogEntryProto entry = next.get(); if (LOG.isTraceEnabled()) { - LOG.trace("{}: applying nextIndex={}, nextLog={}", this, nextIndex, LogProtoUtils.toLogEntryString(next)); + LOG.trace("{}: applying nextIndex={}, nextLog={}", this, nextIndex, LogProtoUtils.toLogEntryString(entry)); } else { LOG.debug("{}: applying nextIndex={}", this, nextIndex); } @@ -245,34 +262,34 @@ private MemoizedSupplier>> applyLog() throws Raf final long incremented = appliedIndex.incrementAndGet(debugIndexChange); Preconditions.assertTrue(incremented == nextIndex); if (f != null) { - futures.get().add(f); + CompletableFuture exceptionHandledFuture = f.exceptionally(ex -> { + LOG.error("Exception while {}: applying txn index={}, nextLog={}", this, nextIndex, + LogProtoUtils.toLogEntryString(entry), ex); + return null; + }); + applyLogFutures = applyLogFutures.thenCombine(exceptionHandledFuture, (v, message) -> null); f.thenAccept(m -> notifyAppliedIndex(incremented)); } else { notifyAppliedIndex(incremented); } - } else { - LOG.debug("{}: logEntry {} is null. There may be snapshot to load. state:{}", - this, nextIndex, state); - break; + } finally { + next.release(); } } - return futures; + return applyLogFutures; } - private void checkAndTakeSnapshot(MemoizedSupplier>> futures) + private void checkAndTakeSnapshot(CompletableFuture futures) throws ExecutionException, InterruptedException { // check if need to trigger a snapshot if (shouldTakeSnapshot()) { - if (futures.isInitialized()) { - JavaUtils.allOf(futures.get()).get(); - } - - takeSnapshot(); + takeSnapshot(futures); } } - private void takeSnapshot() { + private void takeSnapshot(CompletableFuture applyLogFutures) throws ExecutionException, InterruptedException { final long i; + applyLogFutures.get(); try { try(UncheckedAutoCloseable ignored = Timekeeper.start(stateMachineMetrics.get().getTakeSnapshotTimer())) { i = stateMachine.takeSnapshot(); @@ -320,12 +337,33 @@ private boolean shouldTakeSnapshot() { if (autoSnapshotThreshold == null) { return false; } else if (shouldStop()) { - return triggerSnapshotWhenStopEnabled && getLastAppliedIndex() - snapshotIndex.get() > 0; + return shouldTakeSnapshotAtStop() && getLastAppliedIndex() - snapshotIndex.get() > 0; } return state == State.RUNNING && getStateMachineLastAppliedIndex() - snapshotIndex.get() >= autoSnapshotThreshold; } + /** + * In view of the three variables triggerSnapshotWhenStopEnabled, triggerSnapshotWhenRemoveEnabled and isRemoving, + * we can draw the following 8 combination: + * true true true => true + * true true false => true + * true false true => false + * true false false => true + * false true true => true + * false true false => false + * false false true => false + * false false false => false + * @return result + */ + private boolean shouldTakeSnapshotAtStop() { + return isRemoving ? triggerSnapshotWhenRemoveEnabled : triggerSnapshotWhenStopEnabled; + } + + void setRemoving() { + this.isRemoving = true; + } + private long getLastAppliedIndex() { return appliedIndex.get(); } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/TransactionManager.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/TransactionManager.java index aa989cf989..cba310b34d 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/TransactionManager.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/TransactionManager.java @@ -17,28 +17,60 @@ */ package org.apache.ratis.server.impl; +import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.statemachine.TransactionContext; +import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.MemoizedSupplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Collections; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.function.Supplier; /** * Managing {@link TransactionContext}. */ class TransactionManager { - private final ConcurrentMap> contexts = new ConcurrentHashMap<>(); + static final Logger LOG = LoggerFactory.getLogger(TransactionManager.class); - TransactionContext get(long index) { - return Optional.ofNullable(contexts.get(index)).map(Supplier::get).orElse(null); + private final String name; + private final ConcurrentMap> contexts = new ConcurrentHashMap<>(); + + TransactionManager(Object name) { + this.name = name + "-" + JavaUtils.getClassSimpleName(getClass()); + } + + @VisibleForTesting + Map> getMap() { + LOG.debug("{}", this); + return Collections.unmodifiableMap(contexts); + } + + TransactionContext get(TermIndex termIndex) { + return Optional.ofNullable(contexts.get(termIndex)).map(MemoizedSupplier::get).orElse(null); + } + + TransactionContext computeIfAbsent(TermIndex termIndex, MemoizedSupplier constructor) { + final MemoizedSupplier m = contexts.computeIfAbsent(termIndex, i -> constructor); + if (!m.isInitialized()) { + LOG.debug("{}: {}", termIndex, this); + } + return m.get(); } - TransactionContext computeIfAbsent(long index, Supplier constructor) { - return contexts.computeIfAbsent(index, i -> constructor).get(); + void remove(TermIndex termIndex) { + final MemoizedSupplier removed = contexts.remove(termIndex); + if (removed != null) { + LOG.debug("{}: {}", termIndex, this); + } } - void remove(long index) { - contexts.remove(index); + @Override + public String toString() { + return name + ":size=" + contexts.size(); } } \ No newline at end of file diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java index 6375d652c0..3ba0c0e01e 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/VoteContext.java @@ -102,8 +102,10 @@ private boolean checkLeader() { if (info.isFollower()) { final RaftPeerId leader = impl.getState().getLeaderId(); if (leader != null + && !leader.equals(candidateId) && impl.getRole().getFollowerState().map(FollowerState::isCurrentLeaderValid).orElse(false)) { - return reject("this server is a follower and still has a valid leader " + leader); + return reject("this server is a follower and still has a valid leader " + leader + + " different than the candidate " + candidateId); } } return true; diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/WatchRequests.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/WatchRequests.java index f4c6200b9b..6988bfbae2 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/impl/WatchRequests.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/WatchRequests.java @@ -24,6 +24,7 @@ import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.exceptions.ResourceUnavailableException; import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.metrics.RaftServerMetricsImpl; import org.apache.ratis.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,11 +76,15 @@ private class WatchQueue { private final SortedMap q = new TreeMap<>( Comparator.comparingLong(PendingWatch::getIndex).thenComparing(PendingWatch::getCreationTime)); private final ResourceSemaphore resource; + private final RaftServerMetricsImpl raftServerMetrics; private volatile long index; //Invariant: q.isEmpty() or index < any element q - WatchQueue(ReplicationLevel replication, int elementLimit) { + WatchQueue(ReplicationLevel replication, int elementLimit, RaftServerMetricsImpl raftServerMetrics) { this.replication = replication; this.resource = new ResourceSemaphore(elementLimit); + this.raftServerMetrics = raftServerMetrics; + + raftServerMetrics.addNumPendingWatchRequestsGauge(resource::used, replication); } long getIndex() { @@ -103,6 +108,7 @@ CompletableFuture add(RaftClientRequest request) { if (computed == null) { // failed to acquire + raftServerMetrics.onWatchRequestQueueLimitHit(replication); return JavaUtils.completeExceptionally(new ResourceUnavailableException( "Failed to acquire a pending watch request in " + name + " for " + request)); } @@ -123,6 +129,7 @@ void handleTimeout(RaftClientRequest request, PendingWatch pending) { pending.getFuture().completeExceptionally( new NotReplicatedException(request.getCallId(), replication, pending.getIndex())); LOG.debug("{}: timeout {}, {}", name, pending, request); + raftServerMetrics.onWatchRequestTimeout(replication); } } @@ -162,6 +169,12 @@ synchronized void failAll(Exception e) { q.clear(); resource.close(); } + + void close() { + if (raftServerMetrics != null) { + raftServerMetrics.removeNumPendingWatchRequestsGauge(replication); + } + } } private final String name; @@ -171,7 +184,7 @@ synchronized void failAll(Exception e) { private final TimeDuration watchTimeoutDenominationNanos; private final TimeoutExecutor scheduler = TimeoutExecutor.getInstance(); - WatchRequests(Object name, RaftProperties properties) { + WatchRequests(Object name, RaftProperties properties, RaftServerMetricsImpl raftServerMetrics) { this.name = name + "-" + JavaUtils.getClassSimpleName(getClass()); final TimeDuration watchTimeout = RaftServerConfigKeys.Watch.timeout(properties); @@ -183,7 +196,8 @@ synchronized void failAll(Exception e) { + watchTimeoutDenomination + ")."); final int elementLimit = RaftServerConfigKeys.Watch.elementLimit(properties); - Arrays.stream(ReplicationLevel.values()).forEach(r -> queues.put(r, new WatchQueue(r, elementLimit))); + Arrays.stream(ReplicationLevel.values()).forEach(r -> queues.put(r, + new WatchQueue(r, elementLimit, raftServerMetrics))); } CompletableFuture add(RaftClientRequest request) { @@ -207,4 +221,8 @@ void update(ReplicationLevel replication, final long newIndex) { void failWatches(Exception e) { queues.values().forEach(q -> q.failAll(e)); } + + void close() { + queues.values().forEach(WatchQueue::close); + } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/leader/InstallSnapshotRequests.java b/ratis-server/src/main/java/org/apache/ratis/server/leader/InstallSnapshotRequests.java index cdb6603c29..6300ea483c 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/leader/InstallSnapshotRequests.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/leader/InstallSnapshotRequests.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.server.leader; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.ratis.proto.RaftProtos.FileChunkProto; import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto; import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto.SnapshotChunkProto; @@ -33,6 +32,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Iterator; +import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; import java.util.function.Function; @@ -59,14 +59,9 @@ class InstallSnapshotRequests implements Iterable { private final int snapshotChunkMaxSize; /** The total size of snapshot files. */ private final long totalSize; + /** The total number of snapshot files. */ + private final int numFiles; - /** The index of the current request. */ - private int requestIndex = 0; - - /** The index of the current file. */ - private int fileIndex = 0; - /** The current file. */ - private FileChunkReader current; InstallSnapshotRequests(RaftServer.Division server, RaftPeerId followerId, String requestId, SnapshotInfo snapshot, int snapshotChunkMaxSize) { @@ -75,8 +70,10 @@ class InstallSnapshotRequests implements Iterable { this.requestId = requestId; this.snapshot = snapshot; this.snapshotChunkMaxSize = snapshotChunkMaxSize; - this.totalSize = snapshot.getFiles().stream().mapToLong(FileInfo::getFileSize).reduce(Long::sum).orElseThrow( + final List files = snapshot.getFiles(); + this.totalSize = files.stream().mapToLong(FileInfo::getFileSize).reduce(Long::sum).orElseThrow( () -> new IllegalStateException("Failed to compute total size for snapshot " + snapshot)); + this.numFiles = files.size(); final File snapshotDir = server.getStateMachine().getStateMachineStorage().getSnapshotDir(); final Function relativize; @@ -96,67 +93,68 @@ class InstallSnapshotRequests implements Iterable { @Override public Iterator iterator() { - return new Iterator() { - @Override - public boolean hasNext() { - return fileIndex < snapshot.getFiles().size(); - } - - @Override - @SuppressFBWarnings("IT_NO_SUCH_ELEMENT") - public InstallSnapshotRequestProto next() { - return nextInstallSnapshotRequestProto(); - } - }; + return new Iter(); } - private InstallSnapshotRequestProto nextInstallSnapshotRequestProto() { - final int numFiles = snapshot.getFiles().size(); - if (fileIndex >= numFiles) { - throw new NoSuchElementException("fileIndex = " + fileIndex + " >= numFiles = " + numFiles); + private class Iter implements Iterator { + + /** The index of the current request. */ + private int requestIndex = 0; + /** The index of the current file. */ + private int fileIndex = 0; + /** The current file. */ + private FileChunkReader current; + + @Override + public boolean hasNext() { + return fileIndex < numFiles; } - final FileInfo info = snapshot.getFiles().get(fileIndex); - try { - if (current == null) { - current = new FileChunkReader(info, getRelativePath.apply(info)); - } - final FileChunkProto chunk = current.readFileChunk(snapshotChunkMaxSize); - if (chunk.getDone()) { - current.close(); - current = null; - fileIndex++; + + @Override + public InstallSnapshotRequestProto next() { + if (!hasNext()) { + throw new NoSuchElementException("fileIndex = " + fileIndex + " >= numFiles = " + numFiles); } - final boolean done = fileIndex == numFiles && chunk.getDone(); - return newInstallSnapshotRequest(chunk, done); - } catch (IOException e) { - if (current != null) { - try { + final FileInfo info = snapshot.getFiles().get(fileIndex); + try { + if (current == null) { + current = new FileChunkReader(info, getRelativePath.apply(info)); + } + final FileChunkProto chunk = current.readFileChunk(snapshotChunkMaxSize); + if (chunk.getDone()) { current.close(); current = null; - } catch (IOException ignored) { + fileIndex++; + } + + final boolean done = fileIndex == numFiles && chunk.getDone(); + return newInstallSnapshotRequest(chunk, done); + } catch (IOException e) { + if (current != null) { + try { + current.close(); + current = null; + } catch (IOException ignored) { + } } + throw new IllegalStateException("Failed to iterate installSnapshot requests: " + this, e); } - throw new IllegalStateException("Failed to iterate installSnapshot requests: " + this, e); } - } - private InstallSnapshotRequestProto newInstallSnapshotRequest(FileChunkProto chunk, boolean done) { - synchronized (server) { - final SnapshotChunkProto.Builder b = LeaderProtoUtils.toSnapshotChunkProtoBuilder( - requestId, requestIndex++, snapshot.getTermIndex(), chunk, totalSize, done); - return LeaderProtoUtils.toInstallSnapshotRequestProto(server, followerId, b); + private InstallSnapshotRequestProto newInstallSnapshotRequest(FileChunkProto chunk, boolean done) { + synchronized (server) { + final SnapshotChunkProto.Builder b = LeaderProtoUtils.toSnapshotChunkProtoBuilder( + requestId, requestIndex++, snapshot.getTermIndex(), chunk, totalSize, done); + return LeaderProtoUtils.toInstallSnapshotRequestProto(server, followerId, b); + } } } - @Override public String toString() { return server.getId() + "->" + followerId + JavaUtils.getClassSimpleName(getClass()) + ": requestId=" + requestId - + ", requestIndex=" + requestIndex - + ", fileIndex=" + fileIndex - + ", currentFile=" + current + ", snapshot=" + snapshot; } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java index 958cc6fa81..22cd16860d 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderBase.java @@ -33,14 +33,17 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.LongUnaryOperator; @@ -48,12 +51,90 @@ * An abstract implementation of {@link LogAppender}. */ public abstract class LogAppenderBase implements LogAppender { + /** For buffering log entries to create an {@link EntryList}. */ + private static class EntryBuffer { + /** A queue for limiting the byte size, number of elements and poll time. */ + private final DataQueue queue; + /** A map for releasing {@link ReferenceCountedObject}s. */ + private final Map> references = new HashMap<>(); + + EntryBuffer(Object name, RaftProperties properties) { + final SizeInBytes bufferByteLimit = RaftServerConfigKeys.Log.Appender.bufferByteLimit(properties); + final int bufferElementLimit = RaftServerConfigKeys.Log.Appender.bufferElementLimit(properties); + this.queue = new DataQueue<>(name, bufferByteLimit, bufferElementLimit, EntryWithData::getSerializedSize); + } + + boolean putNew(long index, ReferenceCountedObject retained) { + if (!queue.offer(retained.get())) { + retained.release(); + return false; + } + final ReferenceCountedObject previous = references.put(index, retained); + Preconditions.assertNull(previous, () -> "previous with index " + index); + return true; + } + + void releaseAllAndClear() { + for (ReferenceCountedObject ref : references.values()) { + ref.release(); + } + references.clear(); + queue.clear(); + } + + EntryList pollList(long heartbeatWaitTimeMs) throws RaftLogIOException { + final List protos; + try { + protos = queue.pollList(heartbeatWaitTimeMs, EntryWithData::getEntry, null); + } catch (Exception e) { + releaseAllAndClear(); + throw e; + } finally { + for (EntryWithData entry : queue) { + // Remove and release remaining entries. + final ReferenceCountedObject removed = references.remove(entry.getIndex()); + Objects.requireNonNull(removed, "removed == null"); + removed.release(); + } + queue.clear(); + } + return new EntryList(protos, references); + } + } + + /** Storing log entries and their references. */ + private static class EntryList { + private final List protos; + private final Collection> references; + + EntryList(List protos, Map> references) { + Preconditions.assertSame(references.size(), protos.size(), "#entries"); + this.protos = Collections.unmodifiableList(protos); + this.references = Collections.unmodifiableCollection(references.values()); + } + + List getProtos() { + return protos; + } + + void retain() { + for (ReferenceCountedObject ref : references) { + ref.retain(); + } + } + + void release() { + for (ReferenceCountedObject ref : references) { + ref.release(); + } + } + } + private final String name; private final RaftServer.Division server; private final LeaderState leaderState; private final FollowerInfo follower; - private final DataQueue buffer; private final int snapshotChunkMaxSize; private final LogAppenderDaemon daemon; @@ -71,9 +152,6 @@ protected LogAppenderBase(RaftServer.Division server, LeaderState leaderState, F final RaftProperties properties = server.getRaftServer().getProperties(); this.snapshotChunkMaxSize = RaftServerConfigKeys.Log.Appender.snapshotChunkSizeMax(properties).getSizeInt(); - final SizeInBytes bufferByteLimit = RaftServerConfigKeys.Log.Appender.bufferByteLimit(properties); - final int bufferElementLimit = RaftServerConfigKeys.Log.Appender.bufferElementLimit(properties); - this.buffer = new DataQueue<>(this, bufferByteLimit, bufferElementLimit, EntryWithData::getSerializedSize); this.daemon = new LogAppenderDaemon(this); this.eventAwaitForSignal = new AwaitForSignal(name); @@ -124,7 +202,7 @@ public void start() { @Override public boolean isRunning() { - return daemon.isWorking(); + return daemon.isWorking() && server.getInfo().isLeader(); } @Override @@ -206,51 +284,66 @@ protected LongUnaryOperator getNextIndexForError(long newNextIndex) { final long n = oldNextIndex <= 0L ? oldNextIndex : Math.min(oldNextIndex - 1, newNextIndex); if (m > n) { if (m > newNextIndex) { - LOG.info("Set nextIndex to matchIndex + 1 (= " + m + ")"); + LOG.info("{}: Set nextIndex to matchIndex + 1 (= {})", name, m); } return m; } else if (oldNextIndex <= 0L) { return oldNextIndex; // no change. } else { - LOG.info("Decrease nextIndex to " + n); + LOG.info("{}: Decrease nextIndex to {}", name, n); return n; } }; } - @Override - public AppendEntriesRequestProto newAppendEntriesRequest(long callId, boolean heartbeat) + public AppendEntriesRequestProto newAppendEntriesRequest(long callId, boolean heartbeat) { + throw new UnsupportedOperationException("Use nextAppendEntriesRequest(" + callId + ", " + heartbeat +") instead."); + } + + /** + * Create a {@link AppendEntriesRequestProto} object using the {@link FollowerInfo} of this {@link LogAppender}. + * The {@link AppendEntriesRequestProto} object may contain zero or more log entries. + * When there is zero log entries, the {@link AppendEntriesRequestProto} object is a heartbeat. + * + * @param callId The call id of the returned request. + * @param heartbeat the returned request must be a heartbeat. + * + * @return a retained reference of {@link AppendEntriesRequestProto} object. + * Since the returned reference is retained, + * the caller must call {@link ReferenceCountedObject#release()}} after use. + */ + protected ReferenceCountedObject nextAppendEntriesRequest(long callId, boolean heartbeat) throws RaftLogIOException { final long heartbeatWaitTimeMs = getHeartbeatWaitTimeMs(); final TermIndex previous = getPrevious(follower.getNextIndex()); if (heartbeatWaitTimeMs <= 0L || heartbeat) { // heartbeat - return leaderState.newAppendEntriesRequestProto(follower, Collections.emptyList(), - hasPendingDataRequests()? null : previous, callId); + AppendEntriesRequestProto heartbeatRequest = + leaderState.newAppendEntriesRequestProto(follower, Collections.emptyList(), + hasPendingDataRequests() ? null : previous, callId); + ReferenceCountedObject ref = ReferenceCountedObject.wrap(heartbeatRequest); + ref.retain(); + return ref; } - Preconditions.assertTrue(buffer.isEmpty(), () -> "buffer has " + buffer.getNumElements() + " elements."); - final long snapshotIndex = follower.getSnapshotIndex(); - final long leaderNext = getRaftLog().getNextIndex(); final long followerNext = follower.getNextIndex(); - final long halfMs = heartbeatWaitTimeMs/2; - for (long next = followerNext; leaderNext > next && getHeartbeatWaitTimeMs() - halfMs > 0; ) { - if (!buffer.offer(getRaftLog().getEntryWithData(next++))) { - break; - } - } - if (buffer.isEmpty()) { + final EntryBuffer entryBuffer = readLogEntries(followerNext, heartbeatWaitTimeMs); + if (entryBuffer == null) { return null; } - final List protos = buffer.pollList(getHeartbeatWaitTimeMs(), EntryWithData::getEntry, - (entry, time, exception) -> LOG.warn("Failed to get " + entry - + " in " + time.toString(TimeUnit.MILLISECONDS, 3), exception)); - buffer.clear(); + final EntryList entryList = entryBuffer.pollList(heartbeatWaitTimeMs); + final List protos = entryList.getProtos(); assertProtos(protos, followerNext, previous, snapshotIndex); - return leaderState.newAppendEntriesRequestProto(follower, protos, previous, callId); + AppendEntriesRequestProto appendEntriesProto = + leaderState.newAppendEntriesRequestProto(follower, protos, previous, callId); + final ReferenceCountedObject ref = ReferenceCountedObject.wrap( + appendEntriesProto, entryList::retain, entryList::release); + ref.retain(); + entryList.release(); + return ref; } private void assertProtos(List protos, long nextIndex, TermIndex previous, long snapshotIndex) { @@ -272,6 +365,31 @@ private void assertProtos(List protos, long nextIndex, TermIndex } } + private EntryBuffer readLogEntries(long followerNext, long heartbeatWaitTimeMs) throws RaftLogIOException { + final RaftLog raftLog = getRaftLog(); + final long leaderNext = raftLog.getNextIndex(); + final long halfMs = heartbeatWaitTimeMs/2; + EntryBuffer entryBuffer = null; + for (long next = followerNext; leaderNext > next && getHeartbeatWaitTimeMs() - halfMs > 0; next++) { + final ReferenceCountedObject retained; + try { + retained = raftLog.retainEntryWithData(next); + if (entryBuffer == null) { + entryBuffer = new EntryBuffer(name, server.getRaftServer().getProperties()); + } + if (!entryBuffer.putNew(next, retained)) { + break; + } + } catch (Exception e) { + if (entryBuffer != null) { + entryBuffer.releaseAllAndClear(); + } + throw e; + } + } + return entryBuffer; + } + @Override public InstallSnapshotRequestProto newInstallSnapshotNotificationRequest(TermIndex firstAvailableLogTermIndex) { Preconditions.assertTrue(firstAvailableLogTermIndex.getIndex() >= 0); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDaemon.java b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDaemon.java index 847617426f..5de3f3b4da 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDaemon.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDaemon.java @@ -108,8 +108,11 @@ private void run() { }; public CompletableFuture tryToClose() { - if (lifeCycle.transition(TRY_TO_CLOSE) == CLOSING) { + final State state = lifeCycle.transition(TRY_TO_CLOSE); + if (state == CLOSING) { daemon.interrupt(); + } else if (state == CLOSED) { + closeFuture.complete(state); } return closeFuture; } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDefault.java b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDefault.java index 21ef70d4df..8c1675c7c3 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDefault.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/leader/LogAppenderDefault.java @@ -23,9 +23,11 @@ import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto; import org.apache.ratis.rpc.CallId; import org.apache.ratis.server.RaftServer; +import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.server.util.ServerStringUtils; import org.apache.ratis.statemachine.SnapshotInfo; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.Timestamp; import java.io.IOException; @@ -33,6 +35,7 @@ import java.util.Comparator; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * The default implementation of {@link LogAppender} @@ -54,43 +57,38 @@ public Comparator getCallIdComparator() { } /** Send an appendEntries RPC; retry indefinitely. */ - private AppendEntriesReplyProto sendAppendEntriesWithRetries() + private AppendEntriesReplyProto sendAppendEntriesWithRetries(AtomicLong requestFirstIndex) throws InterruptedException, InterruptedIOException, RaftLogIOException { - int retry = 0; - - AppendEntriesRequestProto request = newAppendEntriesRequest(CallId.getAndIncrement(), false); - while (isRunning()) { // keep retrying for IOException + for(int retry = 0; isRunning(); retry++) { + final ReferenceCountedObject request = nextAppendEntriesRequest( + CallId.getAndIncrement(), false); + if (request == null) { + LOG.trace("{} no entries to send now, wait ...", this); + return null; + } try { - if (request == null || request.getEntriesCount() == 0) { - request = newAppendEntriesRequest(CallId.getAndIncrement(), false); - } - - if (request == null) { - LOG.trace("{} no entries to send now, wait ...", this); - return null; - } else if (!isRunning()) { + if (!isRunning()) { LOG.info("{} is stopped. Skip appendEntries.", this); return null; } - resetHeartbeatTrigger(); - final Timestamp sendTime = Timestamp.currentTime(); - getFollower().updateLastRpcSendTime(request.getEntriesCount() == 0); - final AppendEntriesReplyProto r = getServerRpc().appendEntries(request); - getFollower().updateLastRpcResponseTime(); - getFollower().updateLastRespondedAppendEntriesSendTime(sendTime); - - getLeaderState().onFollowerCommitIndex(getFollower(), r.getFollowerCommit()); - return r; + final AppendEntriesRequestProto proto = request.get(); + final AppendEntriesReplyProto reply = sendAppendEntries(proto); + final long first = proto.getEntriesCount() > 0 ? proto.getEntries(0).getIndex() : RaftLog.INVALID_LOG_INDEX; + requestFirstIndex.set(first); + return reply; } catch (InterruptedIOException | RaftLogIOException e) { throw e; } catch (IOException ioe) { // TODO should have more detailed retry policy here. - if (retry++ % 10 == 0) { // to reduce the number of messages + if (retry % 10 == 0) { // to reduce the number of messages LOG.warn("{}: Failed to appendEntries (retry={})", this, retry, ioe); } handleException(ioe); + } finally { + request.release(); } + if (isRunning()) { getServer().properties().rpcSleepTime().sleep(); } @@ -98,6 +96,18 @@ private AppendEntriesReplyProto sendAppendEntriesWithRetries() return null; } + private AppendEntriesReplyProto sendAppendEntries(AppendEntriesRequestProto request) throws IOException { + resetHeartbeatTrigger(); + final Timestamp sendTime = Timestamp.currentTime(); + getFollower().updateLastRpcSendTime(request.getEntriesCount() == 0); + final AppendEntriesReplyProto r = getServerRpc().appendEntries(request); + getFollower().updateLastRpcResponseTime(); + getFollower().updateLastRespondedAppendEntriesSendTime(sendTime); + + getLeaderState().onFollowerCommitIndex(getFollower(), r.getFollowerCommit()); + return r; + } + private InstallSnapshotReplyProto installSnapshot(SnapshotInfo snapshot) throws InterruptedIOException { String requestId = UUID.randomUUID().toString(); InstallSnapshotReplyProto reply = null; @@ -145,6 +155,7 @@ public void run() throws InterruptedException, IOException { case SUCCESS: case SNAPSHOT_UNAVAILABLE: case ALREADY_INSTALLED: + case SNAPSHOT_EXPIRED: getFollower().setAttemptedToInstallSnapshot(); break; default: @@ -153,9 +164,10 @@ public void run() throws InterruptedException, IOException { } // otherwise if r is null, retry the snapshot installation } else { - final AppendEntriesReplyProto r = sendAppendEntriesWithRetries(); + final AtomicLong requestFirstIndex = new AtomicLong(RaftLog.INVALID_LOG_INDEX); + final AppendEntriesReplyProto r = sendAppendEntriesWithRetries(requestFirstIndex); if (r != null) { - handleReply(r); + handleReply(r, requestFirstIndex.get()); } } } @@ -166,7 +178,8 @@ public void run() throws InterruptedException, IOException { } } - private void handleReply(AppendEntriesReplyProto reply) throws IllegalArgumentException { + private void handleReply(AppendEntriesReplyProto reply, long requestFirstIndex) + throws IllegalArgumentException { if (reply != null) { switch (reply.getResult()) { case SUCCESS: @@ -189,7 +202,7 @@ private void handleReply(AppendEntriesReplyProto reply) throws IllegalArgumentEx onFollowerTerm(reply.getTerm()); break; case INCONSISTENCY: - getFollower().decreaseNextIndex(reply.getNextIndex()); + getFollower().setNextIndex(getNextIndexForInconsistency(requestFirstIndex, reply.getNextIndex())); break; case UNRECOGNIZED: LOG.warn("{}: received {}", this, reply.getResult()); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/metrics/LeaderElectionMetrics.java b/ratis-server/src/main/java/org/apache/ratis/server/metrics/LeaderElectionMetrics.java index 7447498d42..1d044bb6fe 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/metrics/LeaderElectionMetrics.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/metrics/LeaderElectionMetrics.java @@ -51,6 +51,7 @@ public final class LeaderElectionMetrics extends RatisMetrics { private final Timekeeper electionTime = getRegistry().timer(LEADER_ELECTION_TIME_TAKEN); + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile Timestamp lastElectionTime; private LeaderElectionMetrics(RaftGroupMemberId serverId, LongSupplier getLastLeaderElapsedTimeMs) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java index cdbce6ee22..70711c3e3b 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java @@ -57,9 +57,14 @@ public final class RaftServerMetricsImpl extends RatisMetrics implements RaftSer public static final String REQUEST_QUEUE_LIMIT_HIT_COUNTER = "numRequestQueueLimitHits"; public static final String REQUEST_BYTE_SIZE_LIMIT_HIT_COUNTER = "numRequestsByteSizeLimitHits"; public static final String RESOURCE_LIMIT_HIT_COUNTER = "numResourceLimitHits"; + public static final String WATCH_REQUEST_QUEUE_LIMIT_HIT_COUNTER = "numWatch%sRequestQueueLimitHits"; public static final String REQUEST_QUEUE_SIZE = "numPendingRequestInQueue"; public static final String REQUEST_MEGA_BYTE_SIZE = "numPendingRequestMegaByteSize"; + + public static final String WATCH_REQUEST_QUEUE_SIZE = "numWatch%sRequestInQueue"; + public static final String WATCH_REQUEST_TIMEOUT_COUNTER = "numWatch%sRequestTimeout"; + public static final String RETRY_CACHE_ENTRY_COUNT_METRIC = "retryCacheEntryCount"; public static final String RETRY_CACHE_HIT_COUNT_METRIC = "retryCacheHitCount"; public static final String RETRY_CACHE_HIT_RATE_METRIC = "retryCacheHitRate"; @@ -76,6 +81,11 @@ public final class RaftServerMetricsImpl extends RatisMetrics implements RaftSer private final LongCounter numRequestQueueLimitHits = getRegistry().counter(REQUEST_QUEUE_LIMIT_HIT_COUNTER); private final LongCounter numRequestsByteSizeLimitHits = getRegistry().counter(REQUEST_BYTE_SIZE_LIMIT_HIT_COUNTER); private final LongCounter numResourceLimitHits = getRegistry().counter(RESOURCE_LIMIT_HIT_COUNTER); + private final Map numWatchRequestQueueLimitHits = newCounterMap(ReplicationLevel.class, + replication -> getRegistry().counter( + String.format(WATCH_REQUEST_QUEUE_LIMIT_HIT_COUNTER, Type.toString(replication)))); + private final Map numWatchRequestsTimeout = newCounterMap(ReplicationLevel.class, + replication -> getRegistry().counter(String.format(WATCH_REQUEST_TIMEOUT_COUNTER, Type.toString(replication)))); private final LongCounter numFailedClientStaleRead = getRegistry().counter(RATIS_SERVER_FAILED_CLIENT_STALE_READ_COUNT); @@ -150,6 +160,14 @@ public LongCounter getNumInstallSnapshot() { return numInstallSnapshot; } + public LongCounter getNumWatchRequestQueueLimitHits(ReplicationLevel replication) { + return numWatchRequestQueueLimitHits.get(replication); + } + + public LongCounter getNumWatchRequestsTimeout(ReplicationLevel replication) { + return numWatchRequestsTimeout.get(replication); + } + private static RatisMetricRegistry createRegistry(String serverId) { return create(new MetricRegistryInfo(serverId, RATIS_APPLICATION_NAME_METRICS, RATIS_SERVER_METRICS, @@ -237,6 +255,22 @@ public boolean removeNumPendingRequestsByteSize() { return getRegistry().remove(REQUEST_MEGA_BYTE_SIZE); } + public void onWatchRequestQueueLimitHit(ReplicationLevel replicationLevel) { + numWatchRequestQueueLimitHits.get(replicationLevel).inc(); + } + + public void onWatchRequestTimeout(ReplicationLevel replicationLevel) { + numWatchRequestsTimeout.get(replicationLevel).inc(); + } + + public void addNumPendingWatchRequestsGauge(Supplier queueSize, ReplicationLevel replication) { + getRegistry().gauge(String.format(WATCH_REQUEST_QUEUE_SIZE, Type.toString(replication)), () -> queueSize); + } + + public boolean removeNumPendingWatchRequestsGauge(ReplicationLevel replication) { + return getRegistry().remove(String.format(WATCH_REQUEST_QUEUE_SIZE, Type.toString(replication))); + } + public void onRequestByteSizeLimitHit() { numRequestsByteSizeLimitHits.inc(); } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java index b75777c9e7..e969eaa484 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/LogProtoUtils.java @@ -19,7 +19,6 @@ import org.apache.ratis.proto.RaftProtos.*; import org.apache.ratis.protocol.ClientId; -import org.apache.ratis.protocol.ClientInvocationId; import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.server.RaftConfiguration; @@ -27,6 +26,7 @@ import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.thirdparty.com.google.protobuf.AbstractMessage; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.ProtoUtils; @@ -45,25 +45,40 @@ public static String toLogEntryString(LogEntryProto entry, Function proto -> "" + ClientInvocationId.valueOf(proto)) - .apply(entry.getStateMachineLogEntry()); + if (function == null) { + function = LogProtoUtils::stateMachineLogEntryProtoToString; + } + s = ", " + function.apply(entry.getStateMachineLogEntry()); } else if (entry.hasMetadataEntry()) { final MetadataProto metadata = entry.getMetadataEntry(); s = "(c:" + metadata.getCommitIndex() + ")"; } else if (entry.hasConfigurationEntry()) { final RaftConfigurationProto config = entry.getConfigurationEntry(); - s = "(current:" + config.getPeersList().stream().map(AbstractMessage::toString).collect(Collectors.joining(",")) + - ", old:" + config.getOldPeersList().stream().map(AbstractMessage::toString).collect(Collectors.joining(",")) - + ")"; + s = "(current:" + peersToString(config.getPeersList()) + + ", old:" + peersToString(config.getOldPeersList()) + ")"; } else { s = ""; } return TermIndex.valueOf(entry) + ", " + entry.getLogEntryBodyCase() + s; } + static String peersToString(List peers) { + return peers.stream().map(AbstractMessage::toString) + .map(s -> s.replace("\n", "")) + .map(s -> s.replace(" ", "")) + .collect(Collectors.joining(", ")); + } + + static String stateMachineLogEntryProtoToString(StateMachineLogEntryProto p) { + final StateMachineEntryProto stateMachineEntry = p.getStateMachineEntry(); + return p.getType() + + ": logData.size=" + p.getLogData().size() + + ", stateMachineData.size=" + stateMachineEntry.getStateMachineData().size() + + ", logEntryProtoSerializedSize=" + stateMachineEntry.getLogEntryProtoSerializedSize(); + } + public static String toLogEntryString(LogEntryProto entry) { - return toLogEntryString(entry, null); + return toLogEntryString(entry, LogProtoUtils::stateMachineLogEntryProtoToString); } public static String toLogEntriesString(List entries) { @@ -71,10 +86,11 @@ public static String toLogEntriesString(List entries) { : entries.stream().map(LogProtoUtils::toLogEntryString).collect(Collectors.toList()).toString(); } - public static String toLogEntriesShortString(List entries) { + public static String toLogEntriesShortString(List entries, + Function stateMachineToString) { return entries == null ? null : entries.isEmpty()? "" - : "size=" + entries.size() + ", first=" + LogProtoUtils.toLogEntryString(entries.get(0)); + : "size=" + entries.size() + ", first=" + toLogEntryString(entries.get(0), stateMachineToString); } public static LogEntryProto toLogEntryProto(RaftConfiguration conf, Long term, long index) { @@ -126,8 +142,9 @@ public static LogEntryProto removeStateMachineData(LogEntryProto entry) { } private static LogEntryProto replaceStateMachineDataWithSerializedSize(LogEntryProto entry) { - return replaceStateMachineEntry(entry, + LogEntryProto replaced = replaceStateMachineEntry(entry, StateMachineEntryProto.newBuilder().setLogEntryProtoSerializedSize(entry.getSerializedSize())); + return copy(replaced); } private static LogEntryProto replaceStateMachineEntry(LogEntryProto proto, StateMachineEntryProto.Builder newEntry) { @@ -149,6 +166,13 @@ static LogEntryProto addStateMachineData(ByteString stateMachineData, LogEntryPr return replaceStateMachineEntry(entry, StateMachineEntryProto.newBuilder().setStateMachineData(stateMachineData)); } + public static boolean hasStateMachineData(LogEntryProto entry) { + return getStateMachineEntry(entry) + .map(StateMachineEntryProto::getStateMachineData) + .map(data -> !data.isEmpty()) + .orElse(false); + } + public static boolean isStateMachineDataEmpty(LogEntryProto entry) { return getStateMachineEntry(entry) .map(StateMachineEntryProto::getStateMachineData) @@ -211,4 +235,21 @@ public static RaftConfiguration toRaftConfiguration(LogEntryProto entry) { final List oldListener = ProtoUtils.toRaftPeers(proto.getOldListenersList()); return ServerImplUtils.newRaftConfiguration(conf, listener, entry.getIndex(), oldConf, oldListener); } + + public static LogEntryProto copy(LogEntryProto proto) { + if (proto == null) { + return null; + } + + if (!proto.hasStateMachineLogEntry() && !proto.hasMetadataEntry() && !proto.hasConfigurationEntry()) { + // empty entry, just return as is. + return proto; + } + + try { + return LogEntryProto.parseFrom(proto.toByteString()); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Failed to copy log entry " + TermIndex.valueOf(proto), e); + } + } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/RaftLogBase.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/RaftLogBase.java index 708b499cd3..05e45e97ac 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/RaftLogBase.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/RaftLogBase.java @@ -17,6 +17,7 @@ */ package org.apache.ratis.server.raftlog; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.proto.RaftProtos.LogEntryProto; @@ -31,7 +32,9 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.OpenCloseState; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.TimeDuration; +import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import java.io.IOException; import java.util.List; @@ -119,26 +122,33 @@ public boolean isOpened() { @Override public boolean updateCommitIndex(long majorityIndex, long currentTerm, boolean isLeader) { - try(AutoCloseableLock writeLock = writeLock()) { + try(AutoCloseableLock writeLock = tryWriteLock(TimeDuration.ONE_SECOND)) { final long oldCommittedIndex = getLastCommittedIndex(); final long newCommitIndex = Math.min(majorityIndex, getFlushIndex()); if (oldCommittedIndex < newCommitIndex) { if (!isLeader) { - commitIndex.updateIncreasingly(newCommitIndex, traceIndexChange); - return true; + return commitIndex.updateIncreasingly(newCommitIndex, traceIndexChange); } // Only update last committed index for current term. See §5.4.2 in paper for details. final TermIndex entry = getTermIndex(newCommitIndex); if (entry != null && entry.getTerm() == currentTerm) { - commitIndex.updateIncreasingly(newCommitIndex, traceIndexChange); - return true; + return commitIndex.updateIncreasingly(newCommitIndex, traceIndexChange); } } + } catch (InterruptedException e) { + LOG.warn("{}: Interrupted to updateCommitIndex: majorityIndex={}, currentTerm={}, isLeader={}", + getName(), majorityIndex, currentTerm, isLeader, e); } return false; } + protected void updatePurgeIndex(Long purged) { + if (purged != null) { + purgeIndex.updateToMax(purged, infoIndexChange); + } + } + protected void updateSnapshotIndexFromStateMachine() { updateSnapshotIndex(getSnapshotIndexFromStateMachine.getAsLong()); } @@ -185,12 +195,13 @@ private long appendImpl(long term, TransactionContext operation) throws StateMac throw new StateMachineException(memberId, new RaftLogIOException( "Log entry size " + entrySize + " exceeds the max buffer limit of " + maxBufferSize)); } - appendEntry(e, operation).whenComplete((returned, t) -> { + + appendEntry(operation.wrap(e), operation).whenComplete((returned, t) -> { if (t != null) { - LOG.error(name + ": Failed to write log entry " + LogProtoUtils.toLogEntryString(e), t); + LOG.error(name + ": Failed to write log entry " + toLogEntryString(e), t); } else if (returned != nextIndex) { LOG.error("{}: Indices mismatched: returned index={} but nextIndex={} for log entry {}", - name, returned, nextIndex, LogProtoUtils.toLogEntryString(e)); + name, returned, nextIndex, toLogEntryString(e)); } else { return; // no error } @@ -231,21 +242,10 @@ private boolean shouldAppendMetadata(long newCommitIndex) { if (newCommitIndex <= 0) { // do not log the first conf entry return false; - } else if (Optional.ofNullable(lastMetadataEntry.get()) - .filter(e -> e.getIndex() == newCommitIndex || e.getMetadataEntry().getCommitIndex() >= newCommitIndex) - .isPresent()) { - //log neither lastMetadataEntry, nor entries with a smaller commit index. - return false; - } - try { - if (get(newCommitIndex).hasMetadataEntry()) { - // do not log the metadata entry - return false; - } - } catch(RaftLogIOException e) { - LOG.error("Failed to get log entry for index " + newCommitIndex, e); } - return true; + final LogEntryProto last = lastMetadataEntry.get(); + // do not log entries with a smaller commit index. + return last == null || newCommitIndex > last.getMetadataEntry().getCommitIndex(); } @Override @@ -277,7 +277,7 @@ public final void open(long lastIndexInSnapshot, Consumer consume final long startIndex = getStartIndex(); if (startIndex > LEAST_VALID_LOG_INDEX) { - purgeIndex.updateIncreasingly(startIndex - 1, infoIndexChange); + purgeIndex.updateToMax(startIndex - 1, infoIndexChange); } } @@ -319,22 +319,28 @@ public final CompletableFuture truncate(long index) { @Override public final CompletableFuture purge(long suggestedIndex) { + final long adjustedIndex; if (purgePreservation > 0) { final long currentIndex = getNextIndex() - 1; - suggestedIndex = Math.min(suggestedIndex, currentIndex - purgePreservation); + adjustedIndex = Math.min(suggestedIndex, currentIndex - purgePreservation); + } else { + adjustedIndex = suggestedIndex; } final long lastPurge = purgeIndex.get(); - if (suggestedIndex - lastPurge < purgeGap) { + if (adjustedIndex - lastPurge < purgeGap) { return CompletableFuture.completedFuture(lastPurge); } - LOG.info("{}: purge {}", getName(), suggestedIndex); - final long finalSuggestedIndex = suggestedIndex; - return purgeImpl(suggestedIndex).whenComplete((purged, e) -> { - if (purged != null) { - purgeIndex.updateToMax(purged, infoIndexChange); - } + final long startIndex = getStartIndex(); + if (adjustedIndex < startIndex) { + LOG.info("{}: purge({}) is skipped: adjustedIndex = {} < startIndex = {}, purgePreservation = {}", + getName(), suggestedIndex, adjustedIndex, startIndex, purgePreservation); + return CompletableFuture.completedFuture(lastPurge); + } + LOG.info("{}: purge {}", getName(), adjustedIndex ); + return purgeImpl(adjustedIndex).whenComplete((purged, e) -> { + updatePurgeIndex(purged); if (e != null) { - LOG.warn(getName() + ": Failed to purge " + finalSuggestedIndex, e); + LOG.warn(getName() + ": Failed to purge " + adjustedIndex, e); } }); } @@ -343,26 +349,37 @@ public final CompletableFuture purge(long suggestedIndex) { @Override public final CompletableFuture appendEntry(LogEntryProto entry) { - return appendEntry(entry, null); + return appendEntry(ReferenceCountedObject.wrap(entry), null); } @Override - public final CompletableFuture appendEntry(LogEntryProto entry, TransactionContext context) { + public final CompletableFuture appendEntry(ReferenceCountedObject entry, + TransactionContext context) { return runner.runSequentially(() -> appendEntryImpl(entry, context)); } - protected abstract CompletableFuture appendEntryImpl(LogEntryProto entry, TransactionContext context); + protected abstract CompletableFuture appendEntryImpl(ReferenceCountedObject entry, + TransactionContext context); @Override - public final List> append(List entries) { + public final List> append(ReferenceCountedObject> entries) { return runner.runSequentially(() -> appendImpl(entries)); } - protected abstract List> appendImpl(List entries); + protected List> appendImpl(List entries) { + throw new UnsupportedOperationException(); + } + + protected List> appendImpl(ReferenceCountedObject> entriesRef) { + try(UncheckedAutoCloseableSupplier> entries = entriesRef.retainAndReleaseOnClose()) { + return appendImpl(entries.get()); + } + } @Override public String toString() { - return getName() + ":" + state + ":c" + getLastCommittedIndex(); + return getName() + ":" + state + ":c" + getLastCommittedIndex() + + (isOpened()? ":last" + getLastEntryTermIndex(): ""); } public AutoCloseableLock readLock() { @@ -373,6 +390,10 @@ public AutoCloseableLock writeLock() { return AutoCloseableLock.acquire(lock.writeLock()); } + public AutoCloseableLock tryWriteLock(TimeDuration timeout) throws InterruptedException { + return AutoCloseableLock.tryAcquire(lock.writeLock(), null, timeout); + } + public boolean hasWriteLock() { return this.lock.isWriteLockedByCurrentThread(); } @@ -390,8 +411,43 @@ public String getName() { return name; } - protected EntryWithData newEntryWithData(LogEntryProto logEntry, CompletableFuture future) { - return new EntryWithDataImpl(logEntry, future); + protected ReferenceCountedObject newEntryWithData(ReferenceCountedObject retained) { + return retained.delegate(new EntryWithDataImpl(retained.get(), null)); + } + + protected ReferenceCountedObject newEntryWithData(ReferenceCountedObject retained, + CompletableFuture> stateMachineDataFuture) { + final EntryWithDataImpl impl = new EntryWithDataImpl(retained.get(), stateMachineDataFuture); + return new ReferenceCountedObject() { + private CompletableFuture> future + = Objects.requireNonNull(stateMachineDataFuture, "stateMachineDataFuture == null"); + + @Override + public EntryWithData get() { + return impl; + } + + synchronized void updateFuture(Consumer> action) { + future = future.whenComplete((ref, e) -> { + if (ref != null) { + action.accept(ref); + } + }); + } + + @Override + public EntryWithData retain() { + retained.retain(); + updateFuture(ReferenceCountedObject::retain); + return impl; + } + + @Override + public boolean release() { + updateFuture(ReferenceCountedObject::release); + return retained.release(); + } + }; } /** @@ -399,20 +455,25 @@ protected EntryWithData newEntryWithData(LogEntryProto logEntry, CompletableFutu */ class EntryWithDataImpl implements EntryWithData { private final LogEntryProto logEntry; - private final CompletableFuture future; + private final CompletableFuture> future; - EntryWithDataImpl(LogEntryProto logEntry, CompletableFuture future) { + EntryWithDataImpl(LogEntryProto logEntry, CompletableFuture> future) { this.logEntry = logEntry; this.future = future == null? null: future.thenApply(this::checkStateMachineData); } - private ByteString checkStateMachineData(ByteString data) { + private ReferenceCountedObject checkStateMachineData(ReferenceCountedObject data) { if (data == null) { - throw new IllegalStateException("State machine data is null for log entry " + logEntry); + throw new IllegalStateException("State machine data is null for log entry " + this); } return data; } + @Override + public long getIndex() { + return logEntry.getIndex(); + } + @Override public int getSerializedSize() { return LogProtoUtils.getSerializedSize(logEntry); @@ -420,14 +481,15 @@ public int getSerializedSize() { @Override public LogEntryProto getEntry(TimeDuration timeout) throws RaftLogIOException, TimeoutException { - LogEntryProto entryProto; if (future == null) { return logEntry; } + final LogEntryProto entryProto; + ReferenceCountedObject data; try { - entryProto = future.thenApply(data -> LogProtoUtils.addStateMachineData(data, logEntry)) - .get(timeout.getDuration(), timeout.getUnit()); + data = future.get(timeout.getDuration(), timeout.getUnit()); + entryProto = LogProtoUtils.addStateMachineData(data.get(), logEntry); } catch (TimeoutException t) { if (timeout.compareTo(stateMachineDataReadTimeout) > 0) { getRaftLogMetrics().onStateMachineDataReadTimeout(); @@ -437,14 +499,14 @@ public LogEntryProto getEntry(TimeDuration timeout) throws RaftLogIOException, T if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } - final String err = getName() + ": Failed readStateMachineData for " + toLogEntryString(logEntry); + final String err = getName() + ": Failed readStateMachineData for " + this; LOG.error(err, e); throw new RaftLogIOException(err, JavaUtils.unwrapCompletionException(e)); } // by this time we have already read the state machine data, // so the log entry data should be set now if (LogProtoUtils.isStateMachineDataEmpty(entryProto)) { - final String err = getName() + ": State machine data not set for " + toLogEntryString(logEntry); + final String err = getName() + ": State machine data not set for " + this; LOG.error(err); throw new RaftLogIOException(err); } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLog.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLog.java index ebb1e27d77..f4b6dc452e 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLog.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLog.java @@ -22,12 +22,15 @@ import org.apache.ratis.server.metrics.RaftLogMetricsBase; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.server.raftlog.LogProtoUtils; import org.apache.ratis.server.raftlog.RaftLogBase; import org.apache.ratis.server.raftlog.LogEntryHeader; +import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.server.storage.RaftStorageMetadata; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.util.AutoCloseableLock; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import java.io.IOException; import java.util.ArrayList; @@ -62,18 +65,22 @@ int size() { void truncate(int index) { if (entries.size() > index) { - entries.subList(index, entries.size()).clear(); + clear(index, entries.size()); } } void purge(int index) { if (entries.size() > index) { - entries.subList(0, index).clear(); + clear(0, index); } } - void add(LogEntryProto entry) { - entries.add(entry); + void clear(int from, int to) { + entries.subList(from, to).clear(); + } + + void add(LogEntryProto entryRef) { + entries.add(entryRef); } } @@ -100,16 +107,35 @@ public RaftLogMetricsBase getRaftLogMetrics() { } @Override - public LogEntryProto get(long index) { + public LogEntryProto get(long index) throws RaftLogIOException { + final ReferenceCountedObject ref = retainLog(index); + try { + return LogProtoUtils.copy(ref.get()); + } finally { + ref.release(); + } + } + + @Override + public ReferenceCountedObject retainLog(long index) { checkLogState(); - try(AutoCloseableLock readLock = readLock()) { - return entries.get(Math.toIntExact(index)); + try (AutoCloseableLock readLock = readLock()) { + final LogEntryProto entry = entries.get(Math.toIntExact(index)); + final ReferenceCountedObject ref = ReferenceCountedObject.wrap(entry); + ref.retain(); + return ref; } } @Override - public EntryWithData getEntryWithData(long index) { - return newEntryWithData(get(index), null); + public EntryWithData getEntryWithData(long index) throws RaftLogIOException { + throw new UnsupportedOperationException("Use retainEntryWithData(" + index + ") instead."); + } + + @Override + public ReferenceCountedObject retainEntryWithData(long index) { + final ReferenceCountedObject ref = retainLog(index); + return newEntryWithData(ref); } @Override @@ -166,11 +192,15 @@ public TermIndex getLastEntryTermIndex() { } @Override - protected CompletableFuture appendEntryImpl(LogEntryProto entry, TransactionContext context) { + protected CompletableFuture appendEntryImpl(ReferenceCountedObject entryRef, + TransactionContext context) { checkLogState(); - try(AutoCloseableLock writeLock = writeLock()) { + LogEntryProto entry = entryRef.retain(); + try (AutoCloseableLock writeLock = writeLock()) { validateLogEntry(entry); entries.add(entry); + } finally { + entryRef.release(); } return CompletableFuture.completedFuture(entry.getIndex()); } @@ -181,12 +211,14 @@ public long getStartIndex() { } @Override - public List> appendImpl(List logEntryProtos) { + public List> appendImpl(ReferenceCountedObject> entriesRef) { checkLogState(); + final List logEntryProtos = entriesRef.retain(); if (logEntryProtos == null || logEntryProtos.isEmpty()) { + entriesRef.release(); return Collections.emptyList(); } - try(AutoCloseableLock writeLock = writeLock()) { + try (AutoCloseableLock writeLock = writeLock()) { // Before truncating the entries, we first need to check if some // entries are duplicated. If the leader sends entry 6, entry 7, then // entry 6 again, without this check the follower may truncate entry 7 @@ -214,10 +246,12 @@ public List> appendImpl(List logEntryProt } for (int i = index; i < logEntryProtos.size(); i++) { LogEntryProto logEntryProto = logEntryProtos.get(i); - this.entries.add(logEntryProto); + entries.add(LogProtoUtils.copy(logEntryProto)); futures.add(CompletableFuture.completedFuture(logEntryProto.getIndex())); } return futures; + } finally { + entriesRef.release(); } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/BufferedWriteChannel.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/BufferedWriteChannel.java index 7ad1a48eef..634fca4ce4 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/BufferedWriteChannel.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/BufferedWriteChannel.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.server.raftlog.segmented; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.function.CheckedBiFunction; import org.apache.ratis.util.function.CheckedConsumer; @@ -177,7 +176,6 @@ boolean isOpen() { } @Override - @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") public void close() throws IOException { if (!isOpen()) { return; diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java index 1e0ef666e9..e26de524df 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegment.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.server.raftlog.segmented; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.server.RaftServerConfigKeys.Log.CorruptionPolicy; import org.apache.ratis.server.metrics.SegmentedRaftLogMetrics; @@ -27,9 +26,9 @@ import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.ratis.thirdparty.com.google.common.cache.CacheLoader; import org.apache.ratis.thirdparty.com.google.protobuf.CodedOutputStream; import org.apache.ratis.util.FileUtils; +import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.SizeInBytes; @@ -38,12 +37,14 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -56,7 +57,8 @@ * * This class will be protected by the {@link SegmentedRaftLog}'s read-write lock. */ -public final class LogSegment implements Comparable { +public final class LogSegment { + static final Logger LOG = LoggerFactory.getLogger(LogSegment.class); enum Op { @@ -72,7 +74,7 @@ static long getEntrySize(LogEntryProto entry, Op op) { case CHECK_SEGMENT_FILE_FULL: case LOAD_SEGMENT_FILE: case WRITE_CACHE_WITH_STATE_MACHINE_CACHE: - Preconditions.assertTrue(entry == LogProtoUtils.removeStateMachineData(entry), + Preconditions.assertTrue(!LogProtoUtils.hasStateMachineData(entry), () -> "Unexpected LogEntryProto with StateMachine data: op=" + op + ", entry=" + entry); break; case WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE: @@ -108,6 +110,44 @@ long getOffset() { } } + private static class Records { + private final ConcurrentNavigableMap map = new ConcurrentSkipListMap<>(); + + int size() { + return map.size(); + } + + LogRecord getFirst() { + final Map.Entry first = map.firstEntry(); + return first != null? first.getValue() : null; + } + + LogRecord getLast() { + final Map.Entry last = map.lastEntry(); + return last != null? last.getValue() : null; + } + + LogRecord get(long i) { + return map.get(i); + } + + long append(LogRecord record) { + final long index = record.getTermIndex().getIndex(); + final LogRecord previous = map.put(index, record); + Preconditions.assertNull(previous, "previous"); + return index; + } + + LogRecord removeLast() { + final Map.Entry last = map.pollLastEntry(); + return Objects.requireNonNull(last, "last == null").getValue(); + } + + void clear() { + map.clear(); + } + } + static LogSegment newOpenSegment(RaftStorage storage, long start, SizeInBytes maxOpSize, SegmentedRaftLogMetrics raftLogMetrics) { Preconditions.assertTrue(start >= 0); @@ -132,8 +172,7 @@ public static int readSegmentFile(File file, LogSegmentStartEnd startEnd, SizeIn Consumer> entryConsumer) throws IOException { int count = 0; - try (SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream( - file, startEnd.getStartIndex(), startEnd.getEndIndex(), startEnd.isOpen(), maxOpSize, raftLogMetrics)) { + try(SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, startEnd, maxOpSize, raftLogMetrics)) { for(LogEntryProto prev = null, next; (next = in.nextEntry()) != null; prev = next) { if (prev != null) { Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1, @@ -184,7 +223,8 @@ static LogSegment loadSegment(RaftStorage storage, File file, LogSegmentStartEnd if (entryCount == 0) { // The segment does not have any entries, delete the file. - FileUtils.deleteFile(file); + final Path deleted = FileUtils.deleteFile(file); + LOG.info("Deleted RaftLog segment since entry count is zero: startEnd={}, path={}", startEnd, deleted); return null; } else if (file.length() > segment.getTotalFileSize()) { // The segment has extra padding, truncate it. @@ -206,10 +246,12 @@ private void assertSegment(long expectedStart, int expectedEntryCount, boolean c final long expectedLastIndex = expectedStart + expectedEntryCount - 1; Preconditions.assertSame(expectedLastIndex, getEndIndex(), "Segment end index"); - final LogRecord last = getLastRecord(); + final LogRecord last = records.getLast(); if (last != null) { Preconditions.assertSame(expectedLastIndex, last.getTermIndex().getIndex(), "Index at the last record"); - Preconditions.assertSame(expectedStart, records.get(0).getTermIndex().getIndex(), "Index at the first record"); + final LogRecord first = records.getFirst(); + Objects.requireNonNull(first, "first record"); + Preconditions.assertSame(expectedStart, first.getTermIndex().getIndex(), "Index at the first record"); } if (!corrupted) { Preconditions.assertSame(expectedEnd, expectedLastIndex, "End/last Index"); @@ -223,66 +265,128 @@ private void assertSegment(long expectedStart, int expectedEntryCount, boolean c * * In the future we can make the cache loader configurable if necessary. */ - class LogEntryLoader extends CacheLoader { + class LogEntryLoader { private final SegmentedRaftLogMetrics raftLogMetrics; LogEntryLoader(SegmentedRaftLogMetrics raftLogMetrics) { this.raftLogMetrics = raftLogMetrics; } - @Override - public LogEntryProto load(LogRecord key) throws IOException { + ReferenceCountedObject load(TermIndex key) throws IOException { final File file = getFile(); // note the loading should not exceed the endIndex: it is possible that // the on-disk log file should be truncated but has not been done yet. - final AtomicReference toReturn = new AtomicReference<>(); + final AtomicReference> toReturn = new AtomicReference<>(); final LogSegmentStartEnd startEnd = LogSegmentStartEnd.valueOf(startIndex, endIndex, isOpen); readSegmentFile(file, startEnd, maxOpSize, getLogCorruptionPolicy(), raftLogMetrics, entryRef -> { final LogEntryProto entry = entryRef.retain(); - final TermIndex ti = TermIndex.valueOf(entry); - putEntryCache(ti, entryRef, Op.LOAD_SEGMENT_FILE); - if (ti.equals(key.getTermIndex())) { - toReturn.set(entry); + try { + final TermIndex ti = TermIndex.valueOf(entry); + putEntryCache(ti, entryRef, Op.LOAD_SEGMENT_FILE); + if (ti.equals(key)) { + entryRef.retain(); + toReturn.set(entryRef); + } + } finally { + entryRef.release(); } }); loadingTimes.incrementAndGet(); - return Objects.requireNonNull(toReturn.get()); + final ReferenceCountedObject proto = toReturn.get(); + if (proto == null) { + throw new RaftLogIOException("Failed to load log entry " + key); + } + return proto; + } + } + + private static class Item { + private final AtomicReference> ref; + private final long serializedSize; + + Item(ReferenceCountedObject obj, long serializedSize) { + this.ref = new AtomicReference<>(obj); + this.serializedSize = serializedSize; + } + + ReferenceCountedObject get() { + return ref.get(); + } + + long release() { + final ReferenceCountedObject entry = ref.getAndSet(null); + if (entry == null) { + return 0; + } + entry.release(); + return serializedSize; } } - static class EntryCache { - private final Map> map = new ConcurrentHashMap<>(); + class EntryCache { + private Map map = new HashMap<>(); private final AtomicLong size = new AtomicLong(); + @Override + public String toString() { + return JavaUtils.getClassSimpleName(getClass()) + "-" + LogSegment.this; + } + long size() { return size.get(); } - LogEntryProto get(TermIndex ti) { - return Optional.ofNullable(map.get(ti)) - .map(ReferenceCountedObject::get) - .orElse(null); + synchronized ReferenceCountedObject get(TermIndex ti) { + if (map == null) { + return null; + } + final Item ref = map.get(ti); + return ref == null? null: ref.get(); + } + + /** After close(), the cache CANNOT be used again. */ + synchronized void close() { + if (map == null) { + return; + } + evict(); + map = null; + LOG.info("Successfully closed {}", this); } - void clear() { - map.values().forEach(ReferenceCountedObject::release); - map.clear(); - size.set(0); + /** After evict(), the cache can be used again. */ + synchronized void evict() { + if (map == null) { + return; + } + for (Iterator> i = map.entrySet().iterator(); i.hasNext(); i.remove()) { + release(i.next().getValue()); + } } - void put(TermIndex key, ReferenceCountedObject valueRef, Op op) { + synchronized void put(TermIndex key, ReferenceCountedObject valueRef, Op op) { + if (map == null) { + return; + } valueRef.retain(); - Optional.ofNullable(map.put(key, valueRef)).ifPresent(this::release); - size.getAndAdd(getEntrySize(valueRef.get(), op)); + final long serializedSize = getEntrySize(valueRef.get(), op); + release(map.put(key, new Item(valueRef, serializedSize))); + size.getAndAdd(serializedSize); } - private void release(ReferenceCountedObject entry) { - size.getAndAdd(-getEntrySize(entry.get(), Op.REMOVE_CACHE)); - entry.release(); + private void release(Item ref) { + if (ref == null) { + return; + } + final long serializedSize = ref.release(); + size.getAndAdd(-serializedSize); } - void remove(TermIndex key) { - Optional.ofNullable(map.remove(key)).ifPresent(this::release); + synchronized void remove(TermIndex key) { + if (map == null) { + return; + } + release(map.remove(key)); } } @@ -293,10 +397,10 @@ File getFile() { private volatile boolean isOpen; private long totalFileSize = SegmentedRaftLogFormat.getHeaderLength(); /** Segment start index, inclusive. */ - private long startIndex; + private final long startIndex; /** Segment end index, inclusive. */ private volatile long endIndex; - private RaftStorage storage; + private final RaftStorage storage; private final SizeInBytes maxOpSize; private final LogEntryLoader cacheLoader; /** later replace it with a metric */ @@ -305,7 +409,7 @@ File getFile() { /** * the list of records is more like the index of a segment */ - private final List records = new ArrayList<>(); + private final Records records = new Records(); /** * the entryCache caches the content of log entries. */ @@ -350,10 +454,13 @@ private void append(Op op, ReferenceCountedObject entryRef, boolean keepEntryInCache, Consumer logConsumer) { final LogEntryProto entry = entryRef.retain(); try { - final LogRecord record = appendLogRecord(op, entry); + final LogRecord record = new LogRecord(totalFileSize, entry); if (keepEntryInCache) { putEntryCache(record.getTermIndex(), entryRef, op); } + appendLogRecord(op, record); + totalFileSize += getEntrySize(entry, op); + if (logConsumer != null) { logConsumer.accept(entry); } @@ -362,60 +469,57 @@ private void append(Op op, ReferenceCountedObject entryRef, } } + private void appendLogRecord(Op op, LogRecord record) { + Objects.requireNonNull(record, "record == null"); + final LogRecord currentLast = records.getLast(); - private LogRecord appendLogRecord(Op op, LogEntryProto entry) { - Objects.requireNonNull(entry, "entry == null"); - if (records.isEmpty()) { - Preconditions.assertTrue(entry.getIndex() == startIndex, - "gap between start index %s and first entry to append %s", - startIndex, entry.getIndex()); - } - - final LogRecord currentLast = getLastRecord(); - if (currentLast != null) { - Preconditions.assertTrue(entry.getIndex() == currentLast.getTermIndex().getIndex() + 1, - "gap between entries %s and %s", entry.getIndex(), currentLast.getTermIndex().getIndex()); + final long index = record.getTermIndex().getIndex(); + if (currentLast == null) { + Preconditions.assertTrue(index == startIndex, + "%s: gap between start index %s and the entry to append %s", op, startIndex, index); + } else { + final long currentLastIndex = currentLast.getTermIndex().getIndex(); + Preconditions.assertTrue(index == currentLastIndex + 1, + "%s: gap between last entry %s and the entry to append %s", op, currentLastIndex, index); } - final LogRecord record = new LogRecord(totalFileSize, entry); - records.add(record); - totalFileSize += getEntrySize(entry, op); - endIndex = entry.getIndex(); - return record; + records.append(record); + endIndex = index; } - LogEntryProto getEntryFromCache(TermIndex ti) { + ReferenceCountedObject getEntryFromCache(TermIndex ti) { return entryCache.get(ti); } /** * Acquire LogSegment's monitor so that there is no concurrent loading. */ - synchronized LogEntryProto loadCache(LogRecord record) throws RaftLogIOException { - LogEntryProto entry = entryCache.get(record.getTermIndex()); + synchronized ReferenceCountedObject loadCache(TermIndex ti) throws RaftLogIOException { + final ReferenceCountedObject entry = entryCache.get(ti); if (entry != null) { - return entry; + try { + entry.retain(); + return entry; + } catch (IllegalStateException ignored) { + // The entry could be removed from the cache and released. + // The exception can be safely ignored since it is the same as cache miss. + } } try { - return cacheLoader.load(record); + return cacheLoader.load(ti); + } catch (RaftLogIOException e) { + throw e; } catch (Exception e) { - throw new RaftLogIOException(e); + throw new RaftLogIOException("Failed to loadCache for log entry " + ti, e); } } LogRecord getLogRecord(long index) { - if (index >= startIndex && index <= endIndex) { - return records.get(Math.toIntExact(index - startIndex)); - } - return null; - } - - private LogRecord getLastRecord() { - return records.isEmpty() ? null : records.get(records.size() - 1); + return records.get(index); } TermIndex getLastTermIndex() { - LogRecord last = getLastRecord(); + final LogRecord last = records.getLast(); return last == null ? null : last.getTermIndex(); } @@ -433,7 +537,8 @@ long getTotalCacheSize() { synchronized void truncate(long fromIndex) { Preconditions.assertTrue(fromIndex >= startIndex && fromIndex <= endIndex); for (long index = endIndex; index >= fromIndex; index--) { - LogRecord removed = records.remove(Math.toIntExact(index - startIndex)); + final LogRecord removed = records.removeLast(); + Preconditions.assertSame(index, removed.getTermIndex().getIndex(), "removedIndex"); removeEntryCache(removed.getTermIndex()); totalFileSize = removed.offset; } @@ -452,16 +557,24 @@ public String toString() { "log-" + startIndex + "_" + endIndex; } - @Override - @SuppressFBWarnings("EQ_COMPARETO_USE_OBJECT_EQUALS") - public int compareTo(Long l) { + /** Comparator to find index in list of LogSegments. */ + static final Comparator SEGMENT_TO_INDEX_COMPARATOR = (o1, o2) -> { + if (o1 instanceof LogSegment && o2 instanceof Long) { + return ((LogSegment) o1).compareTo((Long) o2); + } else if (o1 instanceof Long && o2 instanceof LogSegment) { + return Integer.compare(0, ((LogSegment) o2).compareTo((Long) o1)); + } + throw new IllegalStateException("Unexpected objects to compare(" + o1 + "," + o2 + ")"); + }; + + private int compareTo(Long l) { return (l >= getStartIndex() && l <= getEndIndex()) ? 0 : (this.getEndIndex() < l ? -1 : 1); } synchronized void clear() { records.clear(); - evictCache(); + entryCache.close(); endIndex = startIndex - 1; } @@ -470,7 +583,7 @@ int getLoadingTimes() { } void evictCache() { - entryCache.clear(); + entryCache.evict(); } void putEntryCache(TermIndex key, ReferenceCountedObject valueRef, Op op) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentPath.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentPath.java index e1de7a5431..45dc9a3351 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentPath.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentPath.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.server.raftlog.segmented; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; @@ -30,6 +29,7 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.regex.Matcher; @@ -38,9 +38,12 @@ * * This is a value-based class. */ -public final class LogSegmentPath implements Comparable { +public final class LogSegmentPath { static final Logger LOG = LoggerFactory.getLogger(LogSegmentPath.class); + private static final Comparator COMPARATOR = + Comparator.comparing(LogSegmentPath::getStartEnd); + private final Path path; private final LogSegmentStartEnd startEnd; @@ -57,12 +60,6 @@ public LogSegmentStartEnd getStartEnd() { return startEnd; } - @Override - @SuppressFBWarnings("EQ_COMPARETO_USE_OBJECT_EQUALS") - public int compareTo(LogSegmentPath that) { - return Comparator.comparing(LogSegmentPath::getStartEnd).compare(this, that); - } - @Override public String toString() { return path+ "(" + startEnd + ")"; @@ -84,7 +81,7 @@ private static List getLogSegmentPaths(Path dir) throws IOExcept Optional.ofNullable(matchLogSegment(path)).ifPresent(list::add); } } - list.sort(Comparator.naturalOrder()); + list.sort(COMPARATOR); return list; } @@ -100,9 +97,9 @@ public static LogSegmentPath matchLogSegment(Path path) { return Optional.ofNullable(matchCloseSegment(path)).orElseGet(() -> matchOpenSegment(path)); } - @SuppressFBWarnings("NP_NULL_ON_SOME_PATH") private static LogSegmentPath matchCloseSegment(Path path) { - final Matcher matcher = LogSegmentStartEnd.getClosedSegmentPattern().matcher(path.getFileName().toString()); + final String fileName = String.valueOf(Objects.requireNonNull(path).getFileName()); + final Matcher matcher = LogSegmentStartEnd.getClosedSegmentPattern().matcher(fileName); if (matcher.matches()) { Preconditions.assertTrue(matcher.groupCount() == 2); return newInstance(path, matcher.group(1), matcher.group(2)); @@ -110,9 +107,9 @@ private static LogSegmentPath matchCloseSegment(Path path) { return null; } - @SuppressFBWarnings("NP_NULL_ON_SOME_PATH") private static LogSegmentPath matchOpenSegment(Path path) { - final Matcher matcher = LogSegmentStartEnd.getOpenSegmentPattern().matcher(path.getFileName().toString()); + final String fileName = String.valueOf(Objects.requireNonNull(path).getFileName()); + final Matcher matcher = LogSegmentStartEnd.getOpenSegmentPattern().matcher(fileName); if (matcher.matches()) { if (path.toFile().length() > 0L) { return newInstance(path, matcher.group(1), null); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java index 6fbaeab62a..4f0734ca9e 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/LogSegmentStartEnd.java @@ -22,14 +22,12 @@ import org.apache.ratis.util.Preconditions; import java.io.File; -import java.util.Comparator; import java.util.Objects; -import java.util.Optional; import java.util.regex.Pattern; /** * The start index and an end index of a log segment. - * + *

* This is a value-based class. */ public final class LogSegmentStartEnd implements Comparable { @@ -76,21 +74,24 @@ static LogSegmentStartEnd valueOf(long startIndex, long endIndex, boolean isOpen private final Long endIndex; private LogSegmentStartEnd(long startIndex, Long endIndex) { - Preconditions.assertTrue(startIndex >= RaftLog.LEAST_VALID_LOG_INDEX); - Preconditions.assertTrue(endIndex == null || endIndex >= startIndex); this.startIndex = startIndex; this.endIndex = endIndex; + + Preconditions.assertTrue(startIndex >= RaftLog.LEAST_VALID_LOG_INDEX, this); + if (endIndex != null) { + Preconditions.assertTrue(endIndex >= startIndex, this); + } } - public long getStartIndex() { + long getStartIndex() { return startIndex; } - public long getEndIndex() { - return Optional.ofNullable(endIndex).orElse(RaftLog.INVALID_LOG_INDEX); + long getEndIndex() { + return Objects.requireNonNull(endIndex, "endIndex"); } - public boolean isOpen() { + boolean isOpen() { return endIndex == null; } @@ -108,9 +109,21 @@ File getFile(RaftStorage storage) { @Override public int compareTo(LogSegmentStartEnd that) { - return Comparator.comparingLong(LogSegmentStartEnd::getStartIndex) - .thenComparingLong(LogSegmentStartEnd::getEndIndex) - .compare(this, that); + if (this == that) { + return 0; + } + // startIndex always non-null + final int diff = Long.compare(this.getStartIndex(), that.getStartIndex()); + if (diff != 0) { + return diff; + } + + // same startIndex, compare endIndex + if (this.isOpen()) { + return that.isOpen()? 0 : -1; //open first + } else { + return that.isOpen() ? 1 : Long.compare(this.endIndex, that.endIndex); + } } @Override @@ -131,6 +144,6 @@ public int hashCode() { @Override public String toString() { - return startIndex + "-" + Optional.ofNullable(endIndex).map(Object::toString).orElse(""); + return startIndex + "-" + (endIndex != null? endIndex : ""); } } \ No newline at end of file diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java index 1cfb5933f8..82115ff140 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLog.java @@ -36,6 +36,7 @@ import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.TransactionContext; +import org.apache.ratis.statemachine.impl.TransactionContextImpl; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.util.AutoCloseableLock; import org.apache.ratis.util.AwaitToRun; @@ -54,7 +55,6 @@ import java.util.concurrent.CompletionException; import java.util.function.BiFunction; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.LongSupplier; import org.apache.ratis.util.UncheckedAutoCloseable; @@ -101,6 +101,9 @@ void done() { completeFuture(); } + void discard() { + } + final void completeFuture() { final boolean completed = future.complete(getEndIndex()); Preconditions.assertTrue(completed, @@ -180,11 +183,17 @@ public long getLastAppliedIndex() { @Override public void notifyTruncatedLogEntry(TermIndex ti) { + ReferenceCountedObject ref = null; try { - final LogEntryProto entry = get(ti.getIndex()); + ref = retainLog(ti.getIndex()); + final LogEntryProto entry = ref != null ? ref.get() : null; notifyTruncatedLogEntry.accept(entry); } catch (RaftLogIOException e) { LOG.error("{}: Failed to read log {}", getName(), ti, e); + } finally { + if (ref != null) { + ref.release(); + } } } @@ -205,6 +214,7 @@ public TransactionContext getTransactionContext(LogEntryProto entry, boolean cre private final boolean stateMachineCachingEnabled; private final SegmentedRaftLogMetrics metrics; + @SuppressWarnings({"squid:S2095"}) // Suppress closeable warning private SegmentedRaftLog(Builder b) { super(b.memberId, b.snapshotIndexSupplier, b.properties); this.metrics = new SegmentedRaftLogMetrics(b.memberId); @@ -265,98 +275,116 @@ private void loadLogSegments(long lastIndexInSnapshot, if (!cache.isEmpty() && cache.getEndIndex() < lastIndexInSnapshot) { LOG.warn("End log index {} is smaller than last index in snapshot {}", cache.getEndIndex(), lastIndexInSnapshot); - purgeImpl(lastIndexInSnapshot); + purgeImpl(lastIndexInSnapshot).whenComplete((purged, e) -> updatePurgeIndex(purged)); } } } @Override public LogEntryProto get(long index) throws RaftLogIOException { + final ReferenceCountedObject ref = retainLog(index); + if (ref == null) { + return null; + } + try { + return LogProtoUtils.copy(ref.get()); + } finally { + ref.release(); + } + } + + @Override + public ReferenceCountedObject retainLog(long index) throws RaftLogIOException { checkLogState(); - final LogSegment segment; - final LogRecord record; - try (AutoCloseableLock readLock = readLock()) { - segment = cache.getSegment(index); - if (segment == null) { - return null; - } - record = segment.getLogRecord(index); - if (record == null) { - return null; - } - final LogEntryProto entry = segment.getEntryFromCache(record.getTermIndex()); - if (entry != null) { + final LogSegment segment = cache.getSegment(index); + if (segment == null) { + return null; + } + final LogRecord record = segment.getLogRecord(index); + if (record == null) { + return null; + } + final TermIndex ti = record.getTermIndex(); + final ReferenceCountedObject entry = segment.getEntryFromCache(ti); + if (entry != null) { + try { + entry.retain(); getRaftLogMetrics().onRaftLogCacheHit(); return entry; + } catch (IllegalStateException ignored) { + // The entry could be removed from the cache and released. + // The exception can be safely ignored since it is the same as cache miss. } } // the entry is not in the segment's cache. Load the cache without holding the lock. getRaftLogMetrics().onRaftLogCacheMiss(); cacheEviction.signal(); - return segment.loadCache(record); + return segment.loadCache(ti); } @Override public EntryWithData getEntryWithData(long index) throws RaftLogIOException { - final LogEntryProto entry = get(index); - if (entry == null) { + throw new UnsupportedOperationException("Use retainEntryWithData(" + index + ") instead."); + } + + @Override + public ReferenceCountedObject retainEntryWithData(long index) throws RaftLogIOException { + final ReferenceCountedObject entryRef = retainLog(index); + if (entryRef == null) { throw new RaftLogIOException("Log entry not found: index = " + index); } + + final LogEntryProto entry = entryRef.get(); if (!LogProtoUtils.isStateMachineDataEmpty(entry)) { - return newEntryWithData(entry, null); + return newEntryWithData(entryRef); } try { - CompletableFuture future = null; + CompletableFuture> future = null; if (stateMachine != null) { - future = stateMachine.data().read(entry, server.getTransactionContext(entry, false)).exceptionally(ex -> { + future = stateMachine.data().retainRead(entry, server.getTransactionContext(entry, false)).exceptionally(ex -> { stateMachine.event().notifyLogFailed(ex, entry); throw new CompletionException("Failed to read state machine data for log entry " + entry, ex); }); } - return newEntryWithData(entry, future); + return future != null? newEntryWithData(entryRef, future): newEntryWithData(entryRef); } catch (Exception e) { - final String err = getName() + ": Failed readStateMachineData for " + - LogProtoUtils.toLogEntryString(entry); + final String err = getName() + ": Failed readStateMachineData for " + toLogEntryString(entry); LOG.error(err, e); + entryRef.release(); throw new RaftLogIOException(err, JavaUtils.unwrapCompletionException(e)); } } private void checkAndEvictCache() { if (cache.shouldEvict()) { - // TODO if the cache is hitting the maximum size and we cannot evict any - // segment's cache, should block the new entry appending or new segment - // allocation. - cache.evictCache(server.getFollowerNextIndices(), fileLogWorker.getSafeCacheEvictIndex(), - server.getLastAppliedIndex()); + try (AutoCloseableLock ignored = writeLock()){ + // TODO if the cache is hitting the maximum size and we cannot evict any + // segment's cache, should block the new entry appending or new segment + // allocation. + cache.evictCache(server.getFollowerNextIndices(), fileLogWorker.getSafeCacheEvictIndex(), + server.getLastAppliedIndex()); + } } } @Override public TermIndex getTermIndex(long index) { checkLogState(); - try(AutoCloseableLock readLock = readLock()) { - LogRecord record = cache.getLogRecord(index); - return record != null ? record.getTermIndex() : null; - } + return cache.getTermIndex(index); } @Override public LogEntryHeader[] getEntries(long startIndex, long endIndex) { checkLogState(); - try(AutoCloseableLock readLock = readLock()) { - return cache.getTermIndices(startIndex, endIndex); - } + return cache.getTermIndices(startIndex, endIndex); } @Override public TermIndex getLastEntryTermIndex() { checkLogState(); - try(AutoCloseableLock readLock = readLock()) { - return cache.getLastTermIndex(); - } + return cache.getLastTermIndex(); } @Override @@ -378,18 +406,21 @@ protected CompletableFuture purgeImpl(long index) { try (AutoCloseableLock writeLock = writeLock()) { SegmentedRaftLogCache.TruncationSegments ts = cache.purge(index); updateSnapshotIndexFromStateMachine(); - LOG.debug("purging segments:{}", ts); if (ts != null) { + LOG.info("{}: {}", getName(), ts); Task task = fileLogWorker.purge(ts); return task.getFuture(); } } + LOG.debug("{}: purge({}) found nothing to purge.", getName(), index); return CompletableFuture.completedFuture(index); } @Override - protected CompletableFuture appendEntryImpl(LogEntryProto entry, TransactionContext context) { + protected CompletableFuture appendEntryImpl(ReferenceCountedObject entryRef, + TransactionContext context) { checkLogState(); + LogEntryProto entry = entryRef.retain(); if (LOG.isTraceEnabled()) { LOG.trace("{}: appendEntry {}", getName(), LogProtoUtils.toLogEntryString(entry)); } @@ -424,20 +455,25 @@ protected CompletableFuture appendEntryImpl(LogEntryProto entry, Transacti // If the entry has state machine data, then the entry should be inserted // to statemachine first and then to the cache. Not following the order // will leave a spurious entry in the cache. - final Task write = fileLogWorker.writeLogEntry(entry, removedStateMachineData, context); - final Function> wrap = context != null ? - context::wrap : ReferenceCountedObject::wrap; + final Task write = fileLogWorker.writeLogEntry(entryRef, removedStateMachineData, context); if (stateMachineCachingEnabled) { // The stateMachineData will be cached inside the StateMachine itself. - cache.appendEntry(LogSegment.Op.WRITE_CACHE_WITH_STATE_MACHINE_CACHE, wrap.apply(removedStateMachineData)); + if (removedStateMachineData != entry) { + cache.appendEntry(LogSegment.Op.WRITE_CACHE_WITH_STATE_MACHINE_CACHE, + ReferenceCountedObject.wrap(removedStateMachineData)); + } else { + cache.appendEntry(LogSegment.Op.WRITE_CACHE_WITH_STATE_MACHINE_CACHE, + ReferenceCountedObject.wrap(LogProtoUtils.copy(removedStateMachineData))); + } } else { - cache.appendEntry(LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE, wrap.apply(entry) - ); + cache.appendEntry(LogSegment.Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE, entryRef); } return write.getFuture().whenComplete((clientReply, exception) -> appendEntryTimerContext.stop()); } catch (Exception e) { - LOG.error("{}: Failed to append {}", getName(), LogProtoUtils.toLogEntryString(entry), e); + LOG.error("{}: Failed to append {}", getName(), toLogEntryString(entry), e); throw e; + } finally { + entryRef.release(); } } @@ -454,12 +490,14 @@ private boolean isSegmentFull(LogSegment segment, LogEntryProto entry) { } @Override - public List> appendImpl(List entries) { + protected List> appendImpl(ReferenceCountedObject> entriesRef) { checkLogState(); + final List entries = entriesRef.retain(); if (entries == null || entries.isEmpty()) { + entriesRef.release(); return Collections.emptyList(); } - try(AutoCloseableLock writeLock = writeLock()) { + try (AutoCloseableLock writeLock = writeLock()) { final TruncateIndices ti = cache.computeTruncateIndices(server::notifyTruncatedLogEntry, entries); final long truncateIndex = ti.getTruncateIndex(); final int index = ti.getArrayIndex(); @@ -474,9 +512,12 @@ public List> appendImpl(List entries) { } for (int i = index; i < entries.size(); i++) { final LogEntryProto entry = entries.get(i); - futures.add(appendEntry(entry, server.getTransactionContext(entry, true))); + TransactionContextImpl transactionContext = (TransactionContextImpl) server.getTransactionContext(entry, true); + futures.add(appendEntry(entriesRef.delegate(entry), transactionContext)); } return futures; + } finally { + entriesRef.release(); } } @@ -519,12 +560,13 @@ public CompletableFuture onSnapshotInstalled(long lastSnapshotIndex) { cacheEviction.signal(); } } - return purgeImpl(lastSnapshotIndex); + return purgeImpl(lastSnapshotIndex).whenComplete((purged, e) -> updatePurgeIndex(purged)); } @Override public void close() throws IOException { try(AutoCloseableLock writeLock = writeLock()) { + LOG.info("Start closing {}", this); super.close(); cacheEviction.close(); cache.close(); @@ -532,6 +574,7 @@ public void close() throws IOException { fileLogWorker.close(); storage.close(); getRaftLogMetrics().unregister(); + LOG.info("Successfully closed {}", this); } SegmentedRaftLogCache getRaftLogCache() { @@ -540,7 +583,8 @@ SegmentedRaftLogCache getRaftLogCache() { @Override public String toLogEntryString(LogEntryProto logEntry) { - return LogProtoUtils.toLogEntryString(logEntry, stateMachine::toStateMachineLogEntryString); + return LogProtoUtils.toLogEntryString(logEntry, stateMachine != null ? + stateMachine::toStateMachineLogEntryString : null); } public static Builder newBuilder() { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java index 81f467726f..8d79c58d37 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogCache.java @@ -39,9 +39,19 @@ import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Consumer; +import java.util.stream.Collectors; /** * In-memory RaftLog Cache. Currently we provide a simple implementation that @@ -123,9 +133,14 @@ public String toString() { } static class TruncationSegments { + private final String reason; private final SegmentFileInfo toTruncate; // name of the file to be truncated private final SegmentFileInfo[] toDelete; // names of the files to be deleted + String getReason() { + return reason; + } + public SegmentFileInfo getToTruncate() { return toTruncate; } @@ -134,10 +149,10 @@ public SegmentFileInfo[] getToDelete() { return toDelete; } - TruncationSegments(SegmentFileInfo toTruncate, - List toDelete) { - this.toDelete = SegmentFileInfo.toSortedArray(toDelete); + TruncationSegments(String reason, SegmentFileInfo toTruncate, List toDelete) { + this.reason = reason; this.toTruncate = toTruncate; + this.toDelete = SegmentFileInfo.toSortedArray(toDelete); } long maxEndIndex() { @@ -153,8 +168,9 @@ long maxEndIndex() { @Override public String toString() { - return "toTruncate: " + toTruncate - + "\n toDelete: " + Arrays.toString(toDelete); + return reason + + "\n toTruncate: " + toTruncate + + "\n toDelete: " + Arrays.toString(toDelete); } } @@ -244,13 +260,13 @@ LogSegment get(int i) { int binarySearch(long index) { try(AutoCloseableLock readLock = readLock()) { - return Collections.binarySearch(segments, index); + return Collections.binarySearch(segments, index, LogSegment.SEGMENT_TO_INDEX_COMPARATOR); } } LogSegment search(long index) { try(AutoCloseableLock readLock = readLock()) { - final int i = Collections.binarySearch(segments, index); + final int i = Collections.binarySearch(segments, index, LogSegment.SEGMENT_TO_INDEX_COMPARATOR); return i < 0? null: segments.get(i); } } @@ -261,7 +277,7 @@ LogEntryHeader[] getTermIndex(long startIndex, long realEnd, LogSegment openSegm long index = startIndex; try(AutoCloseableLock readLock = readLock()) { - searchIndex = Collections.binarySearch(segments, startIndex); + searchIndex = Collections.binarySearch(segments, startIndex, LogSegment.SEGMENT_TO_INDEX_COMPARATOR); if (searchIndex >= 0) { for(int i = searchIndex; i < segments.size() && index < realEnd; i++) { final LogSegment s = segments.get(i); @@ -298,6 +314,7 @@ void clear() { } TruncationSegments truncate(long index, LogSegment openSegment, Runnable clearOpenSegment) { + final String reason = "truncate(" + index + ")"; try(AutoCloseableLock writeLock = writeLock()) { final int segmentIndex = binarySearch(index); if (segmentIndex == -segments.size() - 1) { @@ -306,7 +323,7 @@ TruncationSegments truncate(long index, LogSegment openSegment, Runnable clearOp if (index == openSegment.getStartIndex()) { // the open segment should be deleted final SegmentFileInfo deleted = deleteOpenSegment(openSegment, clearOpenSegment); - return new TruncationSegments(null, Collections.singletonList(deleted)); + return new TruncationSegments(reason, null, Collections.singletonList(deleted)); } else { openSegment.truncate(index); Preconditions.assertTrue(!openSegment.isOpen(), @@ -316,7 +333,7 @@ TruncationSegments truncate(long index, LogSegment openSegment, Runnable clearOp segments.add(openSegment); sizeInBytes += openSegment.getTotalFileSize(); clearOpenSegment.run(); - return new TruncationSegments(info, Collections.emptyList()); + return new TruncationSegments(reason, info, Collections.emptyList()); } } } else if (segmentIndex >= 0) { @@ -341,7 +358,7 @@ TruncationSegments truncate(long index, LogSegment openSegment, Runnable clearOp } SegmentFileInfo t = ts.numOfEntries() == 0? null: new SegmentFileInfo(ts.getStartIndex(), oldEnd, false, ts.getTotalFileSize(), ts.getEndIndex()); - return new TruncationSegments(t, list); + return new TruncationSegments(reason, t, list); } return null; } @@ -350,12 +367,14 @@ TruncationSegments truncate(long index, LogSegment openSegment, Runnable clearOp TruncationSegments purge(long index) { try (AutoCloseableLock writeLock = writeLock()) { int segmentIndex = binarySearch(index); - List list = new ArrayList<>(); + if (segmentIndex == -1) { + // nothing to purge + return null; + } + List list = new LinkedList<>(); if (segmentIndex == -segments.size() - 1) { - for (LogSegment ls : segments) { - list.add(SegmentFileInfo.newClosedSegmentFileInfo(ls)); - } + list.addAll(segments); segments.clear(); sizeInBytes = 0; } else if (segmentIndex >= 0) { @@ -368,13 +387,16 @@ TruncationSegments purge(long index) { for (int i = 0; i <= startIndex; i++) { LogSegment segment = segments.remove(0); // must remove the first segment to avoid gaps. sizeInBytes -= segment.getTotalFileSize(); - list.add(SegmentFileInfo.newClosedSegmentFileInfo(segment)); + list.add(segment); } } else { throw new IllegalStateException("Unexpected gap in segments: binarySearch(" + index + ") returns " + segmentIndex + ", segments=" + segments); } - return list.isEmpty() ? null : new TruncationSegments(null, list); + list.forEach(LogSegment::evictCache); + List toDelete = list.stream().map(SegmentFileInfo::newClosedSegmentFileInfo) + .collect(Collectors.toList()); + return list.isEmpty() ? null : new TruncationSegments("purge(" + index + ")", null, toDelete); } } @@ -394,6 +416,7 @@ public String toString() { } private final String name; + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile LogSegment openSegment; private final LogSegmentList closedSegments; private final RaftStorage storage; @@ -520,16 +543,21 @@ void rollOpenSegment(boolean createNewOpen) { } LogSegment getSegment(long index) { - if (openSegment != null && index >= openSegment.getStartIndex()) { - return openSegment; + final LogSegment open = this.openSegment; + if (open != null && index >= open.getStartIndex()) { + return open; } else { return closedSegments.search(index); } } - LogRecord getLogRecord(long index) { + TermIndex getTermIndex(long index) { LogSegment segment = getSegment(index); - return segment == null ? null : segment.getLogRecord(index); + if (segment == null) { + return null; + } + final LogRecord record = segment.getLogRecord(index); + return record != null ? record.getTermIndex() : null; } /** @@ -600,7 +628,7 @@ TermIndex getLastTermIndex() { void appendEntry(LogSegment.Op op, ReferenceCountedObject entry) { // SegmentedRaftLog does the segment creation/rolling work. Here we just // simply append the entry into the open segment. - Preconditions.assertNotNull(openSegment, "openSegment"); + Objects.requireNonNull(openSegment, "openSegment == null"); openSegment.appendToOpenSegment(op, entry); } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java index 481f837f5b..3cc8767fa4 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogInputStream.java @@ -21,6 +21,7 @@ import java.io.EOFException; import java.io.File; import java.io.IOException; +import java.nio.channels.ClosedByInterruptException; import java.util.Optional; import org.apache.ratis.proto.RaftProtos.LogEntryProto; @@ -33,6 +34,7 @@ import org.slf4j.LoggerFactory; import static org.apache.ratis.server.raftlog.RaftLog.INVALID_LOG_INDEX; +import static org.apache.ratis.server.raftlog.RaftLog.LEAST_VALID_LOG_INDEX; public class SegmentedRaftLogInputStream implements Closeable { static final Logger LOG = LoggerFactory.getLogger(SegmentedRaftLogInputStream.class); @@ -62,27 +64,17 @@ boolean hasCorruptHeader() { } private final File logFile; - private final long startIndex; - private final long endIndex; - private final boolean isOpen; + private final LogSegmentStartEnd startEnd; private final OpenCloseState state; private SegmentedRaftLogReader reader; private final SizeInBytes maxOpSize; private final SegmentedRaftLogMetrics raftLogMetrics; - SegmentedRaftLogInputStream(File log, long startIndex, long endIndex, boolean isOpen, + SegmentedRaftLogInputStream(File log, LogSegmentStartEnd startEnd, SizeInBytes maxOpSize, SegmentedRaftLogMetrics raftLogMetrics) { this.maxOpSize = maxOpSize; - if (isOpen) { - Preconditions.assertTrue(endIndex == INVALID_LOG_INDEX); - } else { - Preconditions.assertTrue(endIndex >= startIndex); - } - this.logFile = log; - this.startIndex = startIndex; - this.endIndex = endIndex; - this.isOpen = isOpen; + this.startEnd = startEnd; this.state = new OpenCloseState(getName()); this.raftLogMetrics = raftLogMetrics; } @@ -104,14 +96,6 @@ private void init() throws IOException { } } - long getStartIndex() { - return startIndex; - } - - long getEndIndex() { - return endIndex; - } - String getName() { return logFile.getName(); } @@ -121,7 +105,11 @@ public LogEntryProto nextEntry() throws IOException { try { init(); } catch (Exception e) { - LOG.error("caught exception initializing " + this, e); + if (e.getCause() instanceof ClosedByInterruptException) { + LOG.warn("Initialization is interrupted: {}", this, e); + } else { + LOG.error("Failed to initialize {}", this, e); + } throw IOUtils.asIOException(e); } } @@ -131,7 +119,7 @@ public LogEntryProto nextEntry() throws IOException { final LogEntryProto entry = reader.readEntry(); if (entry != null) { long index = entry.getIndex(); - if (!isOpen() && index >= endIndex) { + if (!startEnd.isOpen() && index >= startEnd.getEndIndex()) { /* * The end index may be derived from the segment recovery * process. It is possible that we still have some uncleaned garbage @@ -139,8 +127,8 @@ public LogEntryProto nextEntry() throws IOException { */ long skipAmt = logFile.length() - reader.getPos(); if (skipAmt > 0) { - LOG.debug("skipping {} bytes at the end of log '{}': reached" + - " entry {} out of {}", skipAmt, getName(), index, endIndex); + LOG.info("Skipping {} bytes at the end of log '{}': reached entry {} out of [{}]", + skipAmt, getName(), index, startEnd); reader.skipFully(skipAmt); } } @@ -172,10 +160,6 @@ public void close() throws IOException { } } - boolean isOpen() { - return isOpen; - } - @Override public String toString() { return getName(); @@ -188,24 +172,18 @@ public String toString() { * ID. The file portion beyond this ID is * potentially being updated. * @return Result of the validation - * @throws IOException */ static LogValidation scanEditLog(File file, long maxTxIdToScan, SizeInBytes maxOpSize) throws IOException { - SegmentedRaftLogInputStream in; - try { - in = new SegmentedRaftLogInputStream(file, INVALID_LOG_INDEX, INVALID_LOG_INDEX, false, maxOpSize, null); - // read the header, initialize the inputstream - in.init(); - } catch (EOFException e) { - LOG.warn("Log file " + file + " has no valid header", e); - return new LogValidation(0, INVALID_LOG_INDEX, true); - } - - try { + final LogSegmentStartEnd startEnd = LogSegmentStartEnd.valueOf(LEAST_VALID_LOG_INDEX); + try(SegmentedRaftLogInputStream in = new SegmentedRaftLogInputStream(file, startEnd, maxOpSize, null)) { + try { + in.init(); + } catch (EOFException e) { + LOG.warn("Invalid header for RaftLog segment {}", file, e); + return new LogValidation(0, INVALID_LOG_INDEX, true); + } return scanEditLog(in, maxTxIdToScan); - } finally { - IOUtils.cleanup(LOG, in); } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogReader.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogReader.java index 7d03105b95..57baffb2fd 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogReader.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogReader.java @@ -42,6 +42,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.ClosedByInterruptException; import java.util.Optional; import java.util.zip.Checksum; @@ -169,7 +170,14 @@ public long skip(long amt) throws IOException { */ boolean verifyHeader() throws IOException { final int headerLength = SegmentedRaftLogFormat.getHeaderLength(); - final int readLength = in.read(temp, 0, headerLength); + final int readLength; + try{ + readLength = in.read(temp, 0, headerLength); + } catch (ClosedByInterruptException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while reading the header of " + file, e); + } + Preconditions.assertTrue(readLength <= headerLength); final int matchLength = SegmentedRaftLogFormat.matchHeader(temp, 0, readLength); Preconditions.assertTrue(matchLength <= readLength); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java index 0d1ea763b6..3382eb9fae 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogWorker.java @@ -38,6 +38,7 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.StateMachine.DataStream; import org.apache.ratis.statemachine.TransactionContext; +import org.apache.ratis.thirdparty.io.netty.util.internal.PlatformDependent; import org.apache.ratis.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,11 +46,15 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.LinkedList; import java.util.Objects; import java.util.Optional; import java.util.Queue; -import java.util.concurrent.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Supplier; @@ -62,6 +67,9 @@ class SegmentedRaftLogWorker { static final TimeDuration ONE_SECOND = TimeDuration.valueOf(1, TimeUnit.SECONDS); + private static final String CLASS_NAME = JavaUtils.getClassSimpleName(SegmentedRaftLogWorker.class); + static final String RUN_WORKER = CLASS_NAME + ".runWorker"; + static class StateMachineDataPolicy { private final boolean sync; private final TimeDuration syncTimeout; @@ -143,6 +151,7 @@ synchronized void updateIndex(long i) { private volatile boolean running = true; private final ExecutorService workerThreadExecutor; private final RaftStorage storage; + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile SegmentedRaftLogOutputStream out; private final Runnable submitUpdateCommitEvent; private final StateMachine stateMachine; @@ -237,11 +246,13 @@ void start(long latestIndex, long evictIndex, File openSegmentFile) throws IOExc } void close() { + queue.close(); this.running = false; + ConcurrentUtils.shutdownAndWait(TimeDuration.ONE_MINUTE, workerThreadExecutor, + timeout -> LOG.warn("{}: shutdown timeout in {}", name, timeout)); Optional.ofNullable(flushExecutor).ifPresent(ExecutorService::shutdown); - ConcurrentUtils.shutdownAndWait(TimeDuration.ONE_SECOND.multiply(3), - workerThreadExecutor, timeout -> LOG.warn("{}: shutdown timeout in " + timeout, name)); IOUtils.cleanup(LOG, out); + PlatformDependent.freeDirectBuffer(writeBuffer); LOG.info("{} close()", name); } @@ -281,6 +292,7 @@ private Task addIOTask(Task task) { LOG.error("Failed to add IO task {}", task, e); Optional.ofNullable(server).ifPresent(RaftServer.Division::close); } + task.discard(); } task.startTimerOnEnqueue(raftLogMetrics.getEnqueuedTimer()); return task; @@ -294,6 +306,7 @@ private void run() { // if and when a log task encounters an exception RaftLogIOException logIOException = null; + CodeInjectionForTesting.execute(RUN_WORKER, server == null ? null : server.getId(), null, queue); while (running) { try { Task task = queue.poll(ONE_SECOND); @@ -333,7 +346,7 @@ private void run() { LOG.info(Thread.currentThread().getName() + " was interrupted, exiting. There are " + queue.getNumElements() + " tasks remaining in the queue."); - return; + break; } catch (Exception e) { if (!running) { LOG.info("{} got closed and hit exception", @@ -344,6 +357,8 @@ private void run() { } } } + + queue.clear(Task::discard); } private boolean shouldFlush() { @@ -352,7 +367,7 @@ private boolean shouldFlush() { } else if (pendingFlushNum >= forceSyncNum) { return true; } - return pendingFlushNum > 0 && queue.isEmpty(); + return pendingFlushNum > 0 && !(queue.peek() instanceof WriteLog); } private void flushIfNecessary() throws IOException { @@ -438,7 +453,8 @@ void rollLogSegment(LogSegment segmentToClose) { addIOTask(new StartLogSegment(segmentToClose.getEndIndex() + 1)); } - Task writeLogEntry(LogEntryProto entry, LogEntryProto removedStateMachineData, TransactionContext context) { + Task writeLogEntry(ReferenceCountedObject entry, + LogEntryProto removedStateMachineData, TransactionContext context) { return addIOTask(new WriteLog(entry, removedStateMachineData, context)); } @@ -468,8 +484,10 @@ private PurgeLog(TruncationSegments segments) { void execute() throws IOException { if (segments.getToDelete() != null) { try(UncheckedAutoCloseable ignored = raftLogMetrics.startPurgeTimer()) { - for (SegmentFileInfo fileInfo : segments.getToDelete()) { - FileUtils.deleteFile(fileInfo.getFile(storage)); + SegmentFileInfo[] toDeletes = segments.getToDelete(); + for (int i = toDeletes.length - 1; i >= 0; i--) { + final Path deleted = FileUtils.deleteFile(toDeletes[i].getFile(storage)); + LOG.info("{}: Purged RaftLog segment: info={}, path={}", name, toDeletes[i], deleted); } } } @@ -485,26 +503,32 @@ private class WriteLog extends Task { private final LogEntryProto entry; private final CompletableFuture stateMachineFuture; private final CompletableFuture combined; + private final AtomicReference> ref = new AtomicReference<>(); - WriteLog(LogEntryProto entry, LogEntryProto removedStateMachineData, TransactionContext context) { + WriteLog(ReferenceCountedObject entryRef, LogEntryProto removedStateMachineData, + TransactionContext context) { + LogEntryProto origEntry = entryRef.get(); this.entry = removedStateMachineData; - if (this.entry == entry) { - final StateMachineLogEntryProto proto = entry.hasStateMachineLogEntry()? entry.getStateMachineLogEntry(): null; + if (this.entry == origEntry) { + final StateMachineLogEntryProto proto = origEntry.hasStateMachineLogEntry() ? + origEntry.getStateMachineLogEntry(): null; if (stateMachine != null && proto != null && proto.getType() == StateMachineLogEntryProto.Type.DATASTREAM) { final ClientInvocationId invocationId = ClientInvocationId.valueOf(proto); final CompletableFuture removed = server.getDataStreamMap().remove(invocationId); - this.stateMachineFuture = removed == null? stateMachine.data().link(null, entry) - : removed.thenApply(stream -> stateMachine.data().link(stream, entry)); + this.stateMachineFuture = removed == null? stateMachine.data().link(null, origEntry) + : removed.thenApply(stream -> stateMachine.data().link(stream, origEntry)); } else { this.stateMachineFuture = null; } + entryRef.retain(); + this.ref.set(entryRef); } else { try { - // this.entry != entry iff the entry has state machine data - this.stateMachineFuture = stateMachine.data().write(entry, context); + // this.entry != origEntry if it has state machine data + this.stateMachineFuture = stateMachine.data().write(entryRef, context); } catch (Exception e) { - LOG.error(name + ": writeStateMachineData failed for index " + entry.getIndex() - + ", entry=" + LogProtoUtils.toLogEntryString(entry, stateMachine::toStateMachineLogEntryString), e); + LOG.error(name + ": writeStateMachineData failed for index " + origEntry.getIndex() + + ", entry=" + LogProtoUtils.toLogEntryString(origEntry, stateMachine::toStateMachineLogEntryString), e); throw e; } } @@ -516,6 +540,7 @@ private class WriteLog extends Task { void failed(IOException e) { stateMachine.event().notifyLogFailed(e, entry); super.failed(e); + discard(); } @Override @@ -531,6 +556,15 @@ CompletableFuture getFuture() { @Override void done() { writeTasks.offerOrCompleteFuture(this); + discard(); + } + + @Override + void discard() { + final ReferenceCountedObject entryRef = ref.getAndSet(null); + if (entryRef != null) { + entryRef.release(); + } } @Override @@ -561,8 +595,8 @@ public String toString() { } } - File getFile(long startIndex, Long endIndex) { - return LogSegmentStartEnd.valueOf(startIndex, endIndex).getFile(storage); + private File getFile(LogSegmentStartEnd startEnd) { + return startEnd.getFile(storage); } private class FinalizeLogSegment extends Task { @@ -579,19 +613,20 @@ private class FinalizeLogSegment extends Task { public void execute() throws IOException { freeSegmentedRaftLogOutputStream(); - final File openFile = getFile(startIndex, null); + final LogSegmentStartEnd openStartEnd = LogSegmentStartEnd.valueOf(startIndex); + final File openFile = getFile(openStartEnd); Preconditions.assertTrue(openFile.exists(), () -> name + ": File " + openFile + " to be rolled does not exist"); if (endIndex - startIndex + 1 > 0) { // finalize the current open segment - final File dstFile = getFile(startIndex, endIndex); + final File dstFile = getFile(LogSegmentStartEnd.valueOf(startIndex, endIndex)); Preconditions.assertTrue(!dstFile.exists()); FileUtils.move(openFile, dstFile); LOG.info("{}: Rolled log segment from {} to {}", name, openFile, dstFile); } else { // delete the file of the empty segment - FileUtils.deleteFile(openFile); - LOG.info("{}: Deleted empty log segment {}", name, openFile); + final Path deleted = FileUtils.deleteFile(openFile); + LOG.info("{}: Deleted empty RaftLog segment: startEnd={}, path={}", name, openStartEnd, deleted); } updateFlushedIndexIncreasingly(); safeCacheEvictIndex.updateToMax(endIndex, traceIndexChange); @@ -624,7 +659,7 @@ private class StartLogSegment extends Task { @Override void execute() throws IOException { - final File openFile = getFile(newStartIndex, null); + final File openFile = getFile(LogSegmentStartEnd.valueOf(newStartIndex)); Preconditions.assertTrue(!openFile.exists(), "open file %s exists for %s", openFile, name); Preconditions.assertTrue(pendingFlushNum == 0); @@ -666,8 +701,8 @@ void execute() throws IOException { final File delFile = del.getFile(storage); Preconditions.assertTrue(delFile.exists(), "File %s to be deleted does not exist", delFile); - FileUtils.deleteFile(delFile); - LOG.info("{}: Deleted log file {}", name, delFile); + final Path deleted = FileUtils.deleteFile(delFile); + LOG.info("{}: Deleted RaftLog segment for {}: path={}", name, segments.getReason(), deleted); minStart = Math.min(minStart, del.getStartIndex()); } if (segments.getToTruncate() == null) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/FileChunkReader.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/FileChunkReader.java index 65bfc8b809..a12818443e 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/storage/FileChunkReader.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/FileChunkReader.java @@ -56,7 +56,7 @@ public FileChunkReader(FileInfo info, Path relativePath) throws IOException { this.relativePath = relativePath; final File f = info.getPath().toFile(); if (info.getFileDigest() == null) { - digester = MD5Hash.getDigester(); + digester = MD5Hash.newDigester(); this.in = new DigestInputStream(FileUtils.newInputStream(f), digester); } else { digester = null; diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java index e7f69d1e24..119f7922db 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageDirectoryImpl.java @@ -210,6 +210,7 @@ void lock() throws IOException { * null if storage is already locked. * @throws IOException if locking fails. */ + @SuppressWarnings({"squid:S2095"}) // Suppress closeable warning private FileLock tryLock(File lockF) throws IOException { boolean deletionHookAdded = false; if (!lockF.exists()) { diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java index fbb7bf7d46..ce809cad8f 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/RaftStorageImpl.java @@ -19,7 +19,6 @@ import java.io.InputStream; import java.io.OutputStream; -import java.nio.file.NoSuchFileException; import java.util.concurrent.atomic.AtomicReference; import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.server.RaftConfiguration; @@ -153,14 +152,16 @@ public void writeRaftConfiguration(LogEntryProto conf) { public RaftConfiguration readRaftConfiguration() { File confFile = storageDir.getMetaConfFile(); - try (InputStream fio = FileUtils.newInputStream(confFile)) { - LogEntryProto confProto = LogEntryProto.newBuilder().mergeFrom(fio).build(); - return LogProtoUtils.toRaftConfiguration(confProto); - } catch (FileNotFoundException | NoSuchFileException e) { - return null; - } catch (Exception e) { - LOG.error("Failed reading configuration from file:" + confFile, e); + if (!confFile.exists()) { return null; + } else { + try (InputStream fio = FileUtils.newInputStream(confFile)) { + LogEntryProto confProto = LogEntryProto.newBuilder().mergeFrom(fio).build(); + return LogProtoUtils.toRaftConfiguration(confProto); + } catch (Exception e) { + LOG.error("Failed reading configuration from file:" + confFile, e); + return null; + } } } diff --git a/ratis-server/src/main/java/org/apache/ratis/server/storage/SnapshotManager.java b/ratis-server/src/main/java/org/apache/ratis/server/storage/SnapshotManager.java index c49a86ec59..91c1ba5b95 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/storage/SnapshotManager.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/storage/SnapshotManager.java @@ -27,7 +27,6 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.StateMachineStorage; import org.apache.ratis.util.FileUtils; -import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.MD5FileUtil; import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.Preconditions; @@ -63,7 +62,7 @@ public class SnapshotManager { private final Supplier snapshotDir; private final Supplier snapshotTmpDir; private final Function getRelativePath; - private final Supplier digester = JavaUtils.memoize(MD5Hash::getDigester); + private MessageDigest digester; SnapshotManager(RaftPeerId selfId, Supplier dir, StateMachineStorage smStorage) { this.selfId = selfId; @@ -77,6 +76,7 @@ public class SnapshotManager { new File(dir.get().getRoot(), c.getFilename()).toPath()).toString(); } + @SuppressWarnings({"squid:S2095"}) // Suppress closeable warning private FileChannel open(FileChunkProto chunk, File tmpSnapshotFile) throws IOException { final FileChannel out; final boolean exists = tmpSnapshotFile.exists(); @@ -87,7 +87,7 @@ private FileChannel open(FileChunkProto chunk, File tmpSnapshotFile) throws IOEx } // create the temp snapshot file and put padding inside out = FileUtils.newFileChannel(tmpSnapshotFile, StandardOpenOption.WRITE, StandardOpenOption.CREATE); - digester.get().reset(); + digester = MD5Hash.newDigester(); } else { if (!exists) { throw new FileNotFoundException("Chunk offset is non-zero but file is not found: " + tmpSnapshotFile @@ -113,7 +113,6 @@ public void installSnapshot(InstallSnapshotRequestProto request, StateMachine st // TODO: Make sure that subsequent requests for the same installSnapshot are coming in order, // and are not lost when whole request cycle is done. Check requestId and requestIndex here - for (FileChunkProto chunk : snapshotChunkRequest.getFileChunksList()) { SnapshotInfo pi = stateMachine.getLatestSnapshot(); if (pi != null && pi.getTermIndex().getIndex() >= lastIncludedIndex) { @@ -127,7 +126,7 @@ public void installSnapshot(InstallSnapshotRequestProto request, StateMachine st try (FileChannel out = open(chunk, tmpSnapshotFile)) { final ByteBuffer data = chunk.getData().asReadOnlyByteBuffer(); - digester.get().update(data.duplicate()); + digester.update(data.duplicate()); int written = 0; for(; data.remaining() > 0; ) { @@ -143,7 +142,7 @@ public void installSnapshot(InstallSnapshotRequestProto request, StateMachine st new MD5Hash(chunk.getFileDigest().toByteArray()); // calculate the checksum of the snapshot file and compare it with the // file digest in the request - final MD5Hash digest = new MD5Hash(digester.get().digest()); + final MD5Hash digest = new MD5Hash(digester.digest()); if (!digest.equals(expectedDigest)) { LOG.warn("The snapshot md5 digest {} does not match expected {}", digest, expectedDigest); diff --git a/ratis-server/src/main/java/org/apache/ratis/server/util/ServerStringUtils.java b/ratis-server/src/main/java/org/apache/ratis/server/util/ServerStringUtils.java index 25223c0f4d..3a5db62859 100644 --- a/ratis-server/src/main/java/org/apache/ratis/server/util/ServerStringUtils.java +++ b/ratis-server/src/main/java/org/apache/ratis/server/util/ServerStringUtils.java @@ -21,16 +21,23 @@ import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto; import org.apache.ratis.proto.RaftProtos.InstallSnapshotReplyProto; import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto; +import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto; +import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.LogProtoUtils; +import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.ProtoUtils; +import java.util.List; +import java.util.function.Function; + /** * This class provides convenient utilities for converting Protocol Buffers messages to strings. * The output strings are for information purpose only. * They are concise and compact compared to the Protocol Buffers implementations of {@link Object#toString()}. - * + *

* The output messages or the output formats may be changed without notice. * Callers of this class should not try to parse the output strings for any purposes. * Instead, they should use the public APIs provided by Protocol Buffers. @@ -38,16 +45,19 @@ public final class ServerStringUtils { private ServerStringUtils() {} - public static String toAppendEntriesRequestString(AppendEntriesRequestProto request) { + public static String toAppendEntriesRequestString(AppendEntriesRequestProto request, + Function stateMachineToString) { if (request == null) { return null; } + final List entries = request.getEntriesList(); return ProtoUtils.toString(request.getServerRequest()) + "-t" + request.getLeaderTerm() + ",previous=" + TermIndex.valueOf(request.getPreviousLog()) + ",leaderCommit=" + request.getLeaderCommit() + ",initializing? " + request.getInitializing() - + ",entries: " + LogProtoUtils.toLogEntriesShortString(request.getEntriesList()); + + "," + (entries.isEmpty()? "HEARTBEAT" : "entries: " + + LogProtoUtils.toLogEntriesShortString(entries, stateMachineToString)); } public static String toAppendEntriesReplyString(AppendEntriesReplyProto reply) { @@ -108,6 +118,16 @@ public static String toRequestVoteReplyString(RequestVoteReplyProto proto) { if (proto == null) { return null; } - return ProtoUtils.toString(proto.getServerReply()) + "-t" + proto.getTerm(); + return ProtoUtils.toString(proto.getServerReply()) + "-t" + proto.getTerm() + + "-last:" + TermIndex.valueOf(proto.getLastEntry()); + } + + /** + * Used to generate the necessary unified name in the submodules under + * {@link org.apache.ratis.server.impl.RaftServerImpl}, which consists + * of {@link org.apache.ratis.server.impl.ServerState#memberId} and the specific class. + */ + public static String generateUnifiedName(RaftGroupMemberId memberId, Class clazz) { + return memberId + "-" + JavaUtils.getClassSimpleName(clazz); } } diff --git a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/BaseStateMachine.java b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/BaseStateMachine.java index 3c20bc5714..7ed1124295 100644 --- a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/BaseStateMachine.java +++ b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/BaseStateMachine.java @@ -18,8 +18,7 @@ package org.apache.ratis.statemachine.impl; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientRequest; import org.apache.ratis.protocol.RaftGroupId; @@ -51,6 +50,7 @@ public class BaseStateMachine implements StateMachine, StateMachine.DataApi, StateMachine.EventApi, StateMachine.LeaderEventApi, StateMachine.FollowerEventApi { private final CompletableFuture server = new CompletableFuture<>(); + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile RaftGroupId groupId; private final LifeCycle lifeCycle = new LifeCycle(JavaUtils.getClassSimpleName(getClass())); @@ -59,7 +59,7 @@ public class BaseStateMachine implements StateMachine, StateMachine.DataApi, private final SortedMap> transactionFutures = new TreeMap<>(); public BaseStateMachine() { - setLastAppliedTermIndex(TermIndex.valueOf(0, -1)); + setLastAppliedTermIndex(TermIndex.INITIAL_VALUE); } public RaftPeerId getId() { @@ -111,10 +111,10 @@ public TransactionContext applyTransactionSerial(TransactionContext trx) throws @Override public CompletableFuture applyTransaction(TransactionContext trx) { // return the same message contained in the entry - RaftProtos.LogEntryProto entry = Objects.requireNonNull(trx.getLogEntry()); + final LogEntryProto entry = Objects.requireNonNull(trx.getLogEntryUnsafe()); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); return CompletableFuture.completedFuture( - Message.valueOf(trx.getLogEntry().getStateMachineLogEntry().getLogData())); + Message.valueOf(entry.getStateMachineLogEntry().getLogData())); } @Override @@ -197,7 +197,6 @@ public CompletableFuture queryStale(Message request, long minIndex) { } @Override - @SuppressFBWarnings("NP_NULL_PARAM_DEREF") public CompletableFuture query(Message request) { return CompletableFuture.completedFuture(null); } diff --git a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java index 37e94a8a0a..7e8afbaa85 100644 --- a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java +++ b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/SimpleStateMachineStorage.java @@ -116,7 +116,7 @@ public void cleanupOldSnapshots(SnapshotRetentionPolicy snapshotRetentionPolicy) final List allSnapshotFiles = getSingleFileSnapshotInfos(stateMachineDir.toPath()); - if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) { + if (allSnapshotFiles.size() > numSnapshotsRetained) { allSnapshotFiles.sort(Comparator.comparing(SingleFileSnapshotInfo::getIndex).reversed()); allSnapshotFiles.subList(numSnapshotsRetained, allSnapshotFiles.size()) .stream() @@ -217,6 +217,10 @@ public SingleFileSnapshotInfo getLatestSnapshot() { if (s != null) { return s; } + return loadLatestSnapshot(); + } + + public SingleFileSnapshotInfo loadLatestSnapshot() { final File dir = stateMachineDir; if (dir == null) { return null; diff --git a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/TransactionContextImpl.java b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/TransactionContextImpl.java index 7c4f1782de..e582fb9e12 100644 --- a/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/TransactionContextImpl.java +++ b/ratis-server/src/main/java/org/apache/ratis/statemachine/impl/TransactionContextImpl.java @@ -25,12 +25,14 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.ReferenceCountedObject; import java.io.IOException; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; /** * Implementation of {@link TransactionContext} @@ -47,6 +49,7 @@ public class TransactionContextImpl implements TransactionContext { private final RaftClientRequest clientRequest; /** Exception from the {@link StateMachine} or from the log */ + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile Exception exception; /** Data from the {@link StateMachine} */ @@ -58,6 +61,7 @@ public class TransactionContextImpl implements TransactionContext { * {@link StateMachine#startTransaction(RaftClientRequest)} and * {@link StateMachine#applyTransaction(TransactionContext)}. */ + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile Object stateMachineContext; /** @@ -68,8 +72,14 @@ public class TransactionContextImpl implements TransactionContext { private boolean shouldCommit = true; /** Committed LogEntry. */ + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile LogEntryProto logEntry; + /** Committed LogEntry copy. */ + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type + private volatile Supplier logEntryCopy; + /** For wrapping {@link #logEntry} in order to release the underlying buffer. */ + @SuppressWarnings({"squid:S3077"}) // Suppress volatile for generic type private volatile ReferenceCountedObject delegatedRef; private final CompletableFuture logIndexFuture = new CompletableFuture<>(); @@ -112,7 +122,7 @@ private static StateMachineLogEntryProto get(StateMachineLogEntryProto stateMach */ TransactionContextImpl(RaftPeerRole serverRole, StateMachine stateMachine, LogEntryProto logEntry) { this(serverRole, null, stateMachine, logEntry.getStateMachineLogEntry()); - this.logEntry = logEntry; + setLogEntry(logEntry); this.logIndexFuture.complete(logEntry.getIndex()); } @@ -135,8 +145,10 @@ public ReferenceCountedObject wrap(LogEntryProto entry) { if (delegatedRef == null) { return TransactionContext.super.wrap(entry); } - Preconditions.assertSame(getLogEntry().getTerm(), entry.getTerm(), "entry.term"); - Preconditions.assertSame(getLogEntry().getIndex(), entry.getIndex(), "entry.index"); + final LogEntryProto expected = getLogEntryUnsafe(); + Objects.requireNonNull(expected, "logEntry == null"); + Preconditions.assertSame(expected.getTerm(), entry.getTerm(), "entry.term"); + Preconditions.assertSame(expected.getIndex(), entry.getIndex(), "entry.index"); return delegatedRef.delegate(entry); } @@ -168,18 +180,31 @@ public LogEntryProto initLogEntry(long term, long index) { Objects.requireNonNull(stateMachineLogEntry, "stateMachineLogEntry == null"); logIndexFuture.complete(index); - return logEntry = LogProtoUtils.toLogEntryProto(stateMachineLogEntry, term, index); + return setLogEntry(LogProtoUtils.toLogEntryProto(stateMachineLogEntry, term, index)); } public CompletableFuture getLogIndexFuture() { return logIndexFuture; } + private LogEntryProto setLogEntry(LogEntryProto entry) { + this.logEntry = entry; + this.logEntryCopy = MemoizedSupplier.valueOf(() -> LogProtoUtils.copy(entry)); + return entry; + } + + @Override public LogEntryProto getLogEntry() { + return logEntryCopy == null ? null : logEntryCopy.get(); + } + + @Override + public LogEntryProto getLogEntryUnsafe() { return logEntry; } + @Override public TransactionContext setException(Exception ioe) { this.exception = ioe; @@ -209,4 +234,8 @@ public TransactionContext cancelTransaction() throws IOException { // call this to let the SM know that Transaction cannot be synced return stateMachine.cancelTransaction(this); } + + public static LogEntryProto getLogEntry(TransactionContext context) { + return ((TransactionContextImpl) context).logEntry; + } } diff --git a/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java b/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java index 15dafb88c8..ba6bca5b5d 100644 --- a/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotFromLeaderTests.java @@ -21,7 +21,12 @@ import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.exceptions.RaftException; +import org.apache.ratis.protocol.exceptions.RaftRetryFailureException; +import org.apache.ratis.protocol.exceptions.ReconfigurationTimeoutException; +import org.apache.ratis.retry.RetryPolicies; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.impl.MiniRaftCluster; @@ -39,8 +44,7 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.SizeInBytes; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,59 +78,119 @@ public abstract class InstallSnapshotFromLeaderTests { + for (RaftServer.Division follower : cluster.getFollowers()) { + final SnapshotInfo info = follower.getStateMachine().getLatestSnapshot(); + Assertions.assertNotNull(info); + Assertions.assertEquals(3, info.getFiles().size()); } + }, 10, ONE_SECOND, "check snapshot", LOG); + } - final SnapshotInfo snapshot = cluster.getLeader().getStateMachine().getLatestSnapshot(); - Assert.assertEquals(3, snapshot.getFiles().size()); - - // add two more peers - final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true, - true); - // trigger setConfiguration - cluster.setConfiguration(change.allPeersInNewConf); - - RaftServerTestUtil - .waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null); - - // Check the installed snapshot file number on each Follower matches with the - // leader snapshot. - JavaUtils.attempt(() -> { - for (RaftServer.Division follower : cluster.getFollowers()) { - final SnapshotInfo info = follower.getStateMachine().getLatestSnapshot(); - Assert.assertNotNull(info); - Assert.assertEquals(3, info.getFiles().size()); - } - }, 10, ONE_SECOND, "check snapshot", LOG); - } finally { - cluster.shutdown(); + private void testInstallSnapshotDuringLeaderSwitch(CLUSTER cluster) throws Exception { + RaftTestUtil.waitForLeader(cluster); + final RaftPeerId leaderId = cluster.getLeader().getId(); + + // perform operations and force all peers to take snapshot + try (final RaftClient client = cluster.createClient(leaderId)) { + for (int i = 0; i < SNAPSHOT_TRIGGER_THRESHOLD * 2; i++) { + final RaftClientReply + reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i)); + Assertions.assertTrue(reply.isSuccess()); + } + + for (final RaftPeer peer: cluster.getPeers()) { + final RaftClientReply snapshotReply = client.getSnapshotManagementApi(leaderId).create(3000); + Assertions.assertTrue(snapshotReply.isSuccess()); + } + } + final SnapshotInfo snapshot = cluster.getLeader().getStateMachine().getLatestSnapshot(); + Assertions.assertNotNull(snapshot); + + // isolate two followers (majority) in old configuration + final List oldFollowers = cluster.getFollowers(); + for (RaftServer.Division f: oldFollowers) { + RaftTestUtil.isolate(cluster, f.getId()); + } + + // add two more peers and install snapshot from leaders + final MiniRaftCluster.PeerChanges change = cluster.addNewPeers(2, true, + true); + try (final RaftClient client = cluster.createClient(leaderId, RetryPolicies.noRetry())) { + final RaftException e = Assertions.assertThrows(RaftException.class, + () -> client.admin().setConfiguration(change.allPeersInNewConf)); + Assertions.assertTrue( e instanceof RaftRetryFailureException + || e instanceof ReconfigurationTimeoutException, + () -> "Unexpected exception: " + e); + } + + final SnapshotInfo snapshotInfo = cluster.getDivision(change.newPeers[0].getId()) + .getStateMachine().getLatestSnapshot(); + Assertions.assertNotNull(snapshotInfo); + + // recover the old followers and isolate the leader to force leader switch + RaftTestUtil.isolate(cluster, leaderId); + for (RaftServer.Division f: oldFollowers) { + RaftTestUtil.deIsolate(cluster, f.getId()); + } + RaftTestUtil.waitForLeader(cluster); + + try (final RaftClient client = cluster.createClient(cluster.getLeader().getId())) { + // successfully setConfiguration during leader switch + final RaftClientReply setConf = client.admin().setConfiguration(change.allPeersInNewConf); + Assertions.assertTrue(setConf.isSuccess()); + + RaftTestUtil.deIsolate(cluster, leaderId); + final RaftClientReply + reply = client.io().send(new RaftTestUtil.SimpleMessage("final")); + Assertions.assertTrue(reply.isSuccess()); } } @@ -137,7 +201,8 @@ private static class StateMachineWithMultiNestedSnapshotFile extends SimpleState File file2; @Override - public synchronized void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage) throws IOException { + public synchronized void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage) + throws IOException { super.initialize(server, groupId, raftStorage); // contains two snapshot files @@ -182,8 +247,8 @@ public long takeSnapshot() { return RaftLog.INVALID_LOG_INDEX; } - Assert.assertTrue(file1.exists()); - Assert.assertTrue(file2.exists()); + Assertions.assertTrue(file1.exists()); + Assertions.assertTrue(file2.exists()); return super.takeSnapshot(); } @@ -199,7 +264,7 @@ public SnapshotInfo getLatestSnapshot() { files.add(new FileInfo( file2.toPath(), null)); - Assert.assertEquals(2, files.size()); + Assertions.assertEquals(2, files.size()); SnapshotInfo info = super.getLatestSnapshot(); if (info == null) { @@ -218,14 +283,15 @@ private static class StateMachineWithSeparatedSnapshotPath extends SimpleStateMa private File tmpDir; @Override - public synchronized void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage) throws IOException { + public synchronized void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage) + throws IOException { super.initialize(server, groupId, raftStorage); this.root = new File("/tmp/ratis-tests/statemachine/" + getId().toString()); this.snapshotDir = new File(root, "snapshot"); this.tmpDir = new File(root, "tmp"); FileUtils.deleteFully(root); - Assert.assertTrue(this.snapshotDir.mkdirs()); - Assert.assertTrue(this.tmpDir.mkdirs()); + Assertions.assertTrue(this.snapshotDir.mkdirs()); + Assertions.assertTrue(this.tmpDir.mkdirs()); this.root.deleteOnExit(); } @@ -241,18 +307,19 @@ public synchronized void pause() { public long takeSnapshot() { final TermIndex lastApplied = getLastAppliedTermIndex(); final File snapshotTmpDir = new File(tmpDir, UUID.randomUUID().toString()); - final File snapshotRealDir = new File(snapshotDir, String.format("%d_%d", lastApplied.getTerm(), lastApplied.getIndex())); + final File snapshotRealDir = new File(snapshotDir, + String.format("%d_%d", lastApplied.getTerm(), lastApplied.getIndex())); try { FileUtils.deleteFully(snapshotRealDir); FileUtils.deleteFully(snapshotTmpDir); - Assert.assertTrue(snapshotTmpDir.mkdirs()); + Assertions.assertTrue(snapshotTmpDir.mkdirs()); final File snapshotFile1 = new File(snapshotTmpDir, "deer"); final File snapshotFile2 = new File(snapshotTmpDir, "loves"); final File snapshotFile3 = new File(snapshotTmpDir, "vegetable"); - Assert.assertTrue(snapshotFile1.createNewFile()); - Assert.assertTrue(snapshotFile2.createNewFile()); - Assert.assertTrue(snapshotFile3.createNewFile()); + Assertions.assertTrue(snapshotFile1.createNewFile()); + Assertions.assertTrue(snapshotFile2.createNewFile()); + Assertions.assertTrue(snapshotFile3.createNewFile()); FileUtils.move(snapshotTmpDir, snapshotRealDir); } catch (IOException ioe) { LOG.error("create snapshot data file failed", ioe); diff --git a/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java b/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java index 72ddd06f26..ae5d792246 100644 --- a/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/InstallSnapshotNotificationTests.java @@ -83,7 +83,7 @@ public abstract class InstallSnapshotNotificationTests leaderSnapshotInfoRef = new AtomicReference<>(); + private static final AtomicReference LEADER_SNAPSHOT_INFO_REF = new AtomicReference<>(); private static final AtomicInteger numSnapshotRequests = new AtomicInteger(); private static final AtomicInteger numNotifyInstallSnapshotFinished = new AtomicInteger(); @@ -102,7 +102,7 @@ public CompletableFuture notifyInstallSnapshotFromLeader( } numSnapshotRequests.incrementAndGet(); - final SingleFileSnapshotInfo leaderSnapshotInfo = (SingleFileSnapshotInfo) leaderSnapshotInfoRef.get(); + final SingleFileSnapshotInfo leaderSnapshotInfo = (SingleFileSnapshotInfo) LEADER_SNAPSHOT_INFO_REF.get(); LOG.info("{}: leaderSnapshotInfo = {}", getId(), leaderSnapshotInfo); if (leaderSnapshotInfo == null) { return super.notifyInstallSnapshotFromLeader(roleInfoProto, termIndex); @@ -137,7 +137,7 @@ public void notifySnapshotInstalled(RaftProtos.InstallSnapshotResult result, lon return; } numNotifyInstallSnapshotFinished.incrementAndGet(); - final SingleFileSnapshotInfo leaderSnapshotInfo = (SingleFileSnapshotInfo) leaderSnapshotInfoRef.get(); + final SingleFileSnapshotInfo leaderSnapshotInfo = (SingleFileSnapshotInfo) LEADER_SNAPSHOT_INFO_REF.get(); File leaderSnapshotFile = leaderSnapshotInfo.getFile().getPath().toFile(); synchronized (this) { try { @@ -184,7 +184,7 @@ public void testAddNewFollowersNoSnapshot() throws Exception { private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Exception { final boolean shouldInstallSnapshot = numRequests >= SNAPSHOT_TRIGGER_THRESHOLD; - leaderSnapshotInfoRef.set(null); + LEADER_SNAPSHOT_INFO_REF.set(null); final List logs; int i = 0; try { @@ -220,7 +220,7 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except // delete the log segments from the leader LOG.info("Delete logs {}", logs); for (LogSegmentPath path : logs) { - FileUtils.deleteFully(path.getPath()); // the log may be already puged + FileUtils.deleteFully(path.getPath()); // the log may be already purged } // restart the peer @@ -236,7 +236,7 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except final SnapshotInfo leaderSnapshotInfo = cluster.getLeader().getStateMachine().getLatestSnapshot(); LOG.info("LeaderSnapshotInfo: {}", leaderSnapshotInfo.getTermIndex()); - final boolean set = leaderSnapshotInfoRef.compareAndSet(null, leaderSnapshotInfo); + final boolean set = LEADER_SNAPSHOT_INFO_REF.compareAndSet(null, leaderSnapshotInfo); Assert.assertTrue(set); // add two more peers @@ -252,7 +252,7 @@ private void testAddNewFollowers(CLUSTER cluster, int numRequests) throws Except // Check the installed snapshot index on each Follower matches with the // leader snapshot. for (RaftServer.Division follower : cluster.getFollowers()) { - final long expected = shouldInstallSnapshot ? leaderSnapshotInfo.getIndex() : RaftLog.INVALID_LOG_INDEX; + final long expected = leaderSnapshotInfo.getIndex(); Assert.assertEquals(expected, RaftServerTestUtil.getLatestInstalledSnapshotIndex(follower)); RaftSnapshotBaseTest.assertLogContent(follower, false); } @@ -271,7 +271,7 @@ public void testRestartFollower() throws Exception { } private void testRestartFollower(CLUSTER cluster) throws Exception { - leaderSnapshotInfoRef.set(null); + LEADER_SNAPSHOT_INFO_REF.set(null); int i = 0; final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster); final RaftPeerId leaderId = leader.getId(); @@ -321,7 +321,7 @@ public void testInstallSnapshotNotificationCount() throws Exception { private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Exception { - leaderSnapshotInfoRef.set(null); + LEADER_SNAPSHOT_INFO_REF.set(null); numSnapshotRequests.set(0); int i = 0; @@ -352,7 +352,7 @@ private void testInstallSnapshotNotificationCount(CLUSTER cluster) throws Except Assert.assertEquals(20, snapshotIndex); final SnapshotInfo leaderSnapshotInfo = cluster.getLeader().getStateMachine().getLatestSnapshot(); Assert.assertEquals(20, leaderSnapshotInfo.getIndex()); - final boolean set = leaderSnapshotInfoRef.compareAndSet(null, leaderSnapshotInfo); + final boolean set = LEADER_SNAPSHOT_INFO_REF.compareAndSet(null, leaderSnapshotInfo); Assert.assertTrue(set); // Wait for the snapshot to be done. @@ -425,7 +425,7 @@ public void testInstallSnapshotInstalledEvent() throws Exception{ } private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception{ - leaderSnapshotInfoRef.set(null); + LEADER_SNAPSHOT_INFO_REF.set(null); numNotifyInstallSnapshotFinished.set(0); final List logs; int i = 0; @@ -475,7 +475,7 @@ private void testInstallSnapshotInstalledEvent(CLUSTER cluster) throws Exception final SnapshotInfo leaderSnapshotInfo = cluster.getLeader().getStateMachine().getLatestSnapshot(); LOG.info("LeaderSnapshotInfo: {}", leaderSnapshotInfo.getTermIndex()); - final boolean set = leaderSnapshotInfoRef.compareAndSet(null, leaderSnapshotInfo); + final boolean set = LEADER_SNAPSHOT_INFO_REF.compareAndSet(null, leaderSnapshotInfo); Assert.assertTrue(set); // add one new peer @@ -524,7 +524,7 @@ public void testInstallSnapshotDuringBootstrap() throws Exception { } private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exception { - leaderSnapshotInfoRef.set(null); + LEADER_SNAPSHOT_INFO_REF.set(null); numSnapshotRequests.set(0); int i = 0; try { @@ -553,7 +553,7 @@ private void testInstallSnapshotDuringBootstrap(CLUSTER cluster) throws Exceptio RaftSnapshotBaseTest.assertLeaderContent(cluster); final SnapshotInfo leaderSnapshotInfo = cluster.getLeader().getStateMachine().getLatestSnapshot(); - final boolean set = leaderSnapshotInfoRef.compareAndSet(null, leaderSnapshotInfo); + final boolean set = LEADER_SNAPSHOT_INFO_REF.compareAndSet(null, leaderSnapshotInfo); Assert.assertTrue(set); // add two more peers diff --git a/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java b/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java index ce08e4a74f..2f1492d433 100644 --- a/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java @@ -168,6 +168,7 @@ public void testFollowerHeartbeatMetric() throws IOException, InterruptedExcepti assertTrue(t.getTimer().getMeanRate() > 0.0d); assertTrue(t.getTimer().getCount() > 0L); } + cluster.shutdown(); } } diff --git a/ratis-server/src/test/java/org/apache/ratis/OutputStreamBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/OutputStreamBaseTest.java index d86170d0b4..0c1a5164fc 100644 --- a/ratis-server/src/test/java/org/apache/ratis/OutputStreamBaseTest.java +++ b/ratis-server/src/test/java/org/apache/ratis/OutputStreamBaseTest.java @@ -22,13 +22,11 @@ import org.apache.ratis.proto.RaftProtos.LogEntryProto.LogEntryBodyCase; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.impl.MiniRaftCluster; -import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.LogEntryHeader; import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.StringUtils; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftAsyncExceptionTests.java b/ratis-server/src/test/java/org/apache/ratis/RaftAsyncExceptionTests.java index 1ad53a8381..33e0857338 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RaftAsyncExceptionTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/RaftAsyncExceptionTests.java @@ -104,7 +104,7 @@ private void runTestTimeoutException(CLUSTER cluster) throws Exception { final RaftClientReply reply = client.io().send(new SimpleMessage("m0")); Assert.assertTrue(reply.isSuccess()); - RaftClientConfigKeys.Rpc.setRequestTimeout(properties.get(), ONE_SECOND); + RaftClientConfigKeys.Rpc.setRequestTimeout(PROPERTIES.get(), ONE_SECOND); // Block StartTransaction StreamSupport.stream(cluster.getServers().spliterator(), false) .map(cluster::getDivision) diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java b/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java index 260f6013e8..a1c16df8f2 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/RaftAsyncTests.java @@ -24,6 +24,7 @@ import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.proto.RaftProtos.CommitInfoProto; import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.proto.RaftProtos.ReplicationLevel; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.exceptions.AlreadyClosedException; @@ -46,6 +47,7 @@ import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.PlatformUtils; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.function.CheckedRunnable; @@ -77,11 +79,15 @@ public abstract class RaftAsyncTests extends Ba public static final int NUM_SERVERS = 3; - private static final DelayLocalExecutionInjection logSyncDelay = RaftServerTestUtil.getLogSyncDelay(); + private static final DelayLocalExecutionInjection LOG_SYNC_DELAY = RaftServerTestUtil.getLogSyncDelay(); { getProperties().setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class); + if (!PlatformUtils.LINUX) { + getProperties().setBoolean("raft.netty.server.use-epoll", false); + getProperties().setBoolean("raft.netty.client.use-epoll", false); + } } @Test @@ -280,18 +286,18 @@ public void testStaleReadAsync() throws Exception { } void runTestStaleReadAsync(CLUSTER cluster) throws Exception { - final int numMesssages = 10; - try (RaftClient client = cluster.createClient()) { - RaftTestUtil.waitForLeader(cluster); + final int numMessages = 10; + RaftServer.Division division = waitForLeader(cluster); + try (RaftClient client = cluster.createClient(division.getId())) { // submit some messages final List> futures = new ArrayList<>(); - for (int i = 0; i < numMesssages; i++) { + for (int i = 0; i < numMessages; i++) { final String s = "" + i; LOG.info("sendAsync " + s); futures.add(client.async().send(new SimpleMessage(s))); } - Assert.assertEquals(numMesssages, futures.size()); + Assert.assertEquals(numMessages, futures.size()); final List replies = new ArrayList<>(); for (CompletableFuture f : futures) { final RaftClientReply r = f.join(); @@ -303,6 +309,7 @@ void runTestStaleReadAsync(CLUSTER cluster) throws Exception { // Use a follower with the max commit index final RaftClientReply lastWriteReply = replies.get(replies.size() - 1); final RaftPeerId leader = lastWriteReply.getServerId(); + Assert.assertEquals(leader, lastWriteReply.getServerId()); LOG.info("leader = " + leader); final Collection commitInfos = lastWriteReply.getCommitInfos(); LOG.info("commitInfos = " + commitInfos); @@ -321,7 +328,7 @@ void runTestStaleReadAsync(CLUSTER cluster) throws Exception { StateMachineException.class, IndexOutOfBoundsException.class); // test sendStaleReadAsync - for (int i = 0; i < numMesssages; i++) { + for (int i = 0; i < numMessages; i++) { final RaftClientReply reply = replies.get(i); final String query = "" + i; LOG.info("query=" + query + ", reply=" + reply); @@ -357,6 +364,38 @@ void runTestStaleReadAsync(CLUSTER cluster) throws Exception { } } + @Test + public void testWriteAsyncCustomReplicationLevel() throws Exception { + // verify that send(msg, ALL_COMMITTED) would reply with all servers committed past the log index + runWithNewCluster(NUM_SERVERS, this::runTestWriteAsyncCustomReplicationLevel); + } + + void runTestWriteAsyncCustomReplicationLevel(CLUSTER cluster) throws Exception { + final int numMessages = 20; + final RaftPeerId leader = waitForLeader(cluster).getId(); + try (RaftClient client = cluster.createClient(leader)) { + + // submit some messages + for (int i = 0; i < numMessages; i++) { + final String s = "" + i; + LOG.info("sendAsync with ALL_COMMITTED " + s); + client.async().send(new SimpleMessage(s), ReplicationLevel.ALL_COMMITTED).whenComplete((reply, exception) -> { + if (exception != null) { + LOG.error("Failed to send message " + s, exception); + // reply should be null in case of exception + Assert.assertNull(reply); + return; + } + Assert.assertTrue(reply.isSuccess()); + Assert.assertNull(reply.getException()); + // verify that all servers have caught up to log index when the reply is returned + reply.getCommitInfos().forEach(commitInfoProto -> + Assert.assertTrue(commitInfoProto.getCommitIndex() >= reply.getLogIndex())); + }); + } + } + } + @Test public void testRequestTimeout() throws Exception { final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties()); @@ -372,7 +411,7 @@ public void testRequestTimeout() throws Exception { @Test public void testStateMachineMetrics() throws Exception { runWithNewCluster(NUM_SERVERS, cluster -> - RaftBasicTests.testStateMachineMetrics(true, cluster, LOG)); + RaftBasicTests.runTestStateMachineMetrics(true, cluster)); } @Test @@ -384,13 +423,13 @@ void runTestAppendEntriesTimeout(CLUSTER cluster) throws Exception { LOG.info("Running testAppendEntriesTimeout"); final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties()); RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), TimeDuration.valueOf(20, TimeUnit.SECONDS)); - waitForLeader(cluster); + final RaftPeerId leader = waitForLeader(cluster).getId(); long time = System.currentTimeMillis(); long waitTime = 5000; try (final RaftClient client = cluster.createClient()) { // block append requests cluster.getServerAliveStream() - .filter(impl -> !impl.getInfo().isLeader()) + .filter(impl -> !impl.getInfo().isLeader() && !impl.getPeer().getId().equals(leader)) .map(SimpleStateMachine4Testing::get) .forEach(SimpleStateMachine4Testing::blockWriteStateMachineData); @@ -400,7 +439,7 @@ void runTestAppendEntriesTimeout(CLUSTER cluster) throws Exception { Assert.assertFalse(replyFuture.isDone()); // unblock append request. cluster.getServerAliveStream() - .filter(impl -> !impl.getInfo().isLeader()) + .filter(impl -> !impl.getInfo().isLeader() && !impl.getPeer().getId().equals(leader)) .map(SimpleStateMachine4Testing::get) .forEach(SimpleStateMachine4Testing::unblockWriteStateMachineData); @@ -430,7 +469,7 @@ void runTestCheckLeadershipFailure(CLUSTER cluster) throws Exception { cluster.getServerAliveStream() .filter(impl -> !impl.getInfo().isLeader()) .map(SimpleStateMachine4Testing::get) - .forEach(peer -> logSyncDelay.setDelayMs(peer.getId().toString(), 1000)); + .forEach(peer -> LOG_SYNC_DELAY.setDelayMs(peer.getId().toString(), 1000)); // trigger append entries request client.async().send(new SimpleMessage("abc")); @@ -446,7 +485,7 @@ void runTestCheckLeadershipFailure(CLUSTER cluster) throws Exception { } finally { // unblock append entries request - logSyncDelay.clear(); + LOG_SYNC_DELAY.clear(); } waitForLeader(cluster); diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java b/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java index 47c9b0e085..2ce1706cf9 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/RaftBasicTests.java @@ -17,6 +17,7 @@ */ package org.apache.ratis; +import org.apache.ratis.test.tag.Flaky; import org.apache.ratis.thirdparty.com.codahale.metrics.Gauge; import org.apache.ratis.RaftTestUtil.SimpleMessage; import org.apache.ratis.client.RaftClient; @@ -33,19 +34,18 @@ import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.impl.BlockRequestHandlingInjection; import org.apache.ratis.server.impl.MiniRaftCluster; -import org.apache.ratis.server.impl.RaftServerTestUtil; import org.apache.ratis.server.impl.RetryCacheTestUtil; import org.apache.ratis.server.metrics.ServerMetricsTestUtils; import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.JavaUtils; -import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; -import org.slf4j.event.Level; import java.io.IOException; import java.util.List; @@ -73,9 +73,6 @@ public abstract class RaftBasicTests extends BaseTest implements MiniRaftCluster.Factory.Get { { - Slf4jUtils.setLogLevel(RaftServer.Division.LOG, Level.DEBUG); - RaftServerTestUtil.setStateMachineUpdaterLogLevel(Level.DEBUG); - RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), TimeDuration.valueOf(5, TimeUnit.SECONDS)); } @@ -94,27 +91,27 @@ public void testBasicAppendEntriesKillLeader() throws Exception { } static CompletableFuture killAndRestartServer( - RaftPeerId id, long killSleepMs, long restartSleepMs, MiniRaftCluster cluster, Logger LOG) { + RaftPeerId id, long killSleepMs, long restartSleepMs, MiniRaftCluster cluster, Logger log) { final CompletableFuture future = new CompletableFuture<>(); new Thread(() -> { try { Thread.sleep(killSleepMs); cluster.killServer(id); Thread.sleep(restartSleepMs); - LOG.info("restart server: " + id); + log.info("restart server: " + id); cluster.restartServer(id, false); future.complete(null); } catch (Exception e) { - ExitUtils.terminate(-1, "Failed to kill/restart server: " + id, e, LOG); + ExitUtils.terminate(-1, "Failed to kill/restart server: " + id, e, log); } }).start(); return future; } static void runTestBasicAppendEntries( - boolean async, boolean killLeader, int numMessages, MiniRaftCluster cluster, Logger LOG) + boolean async, boolean killLeader, int numMessages, MiniRaftCluster cluster, Logger log) throws Exception { - LOG.info("runTestBasicAppendEntries: async? {}, killLeader={}, numMessages={}", + log.info("runTestBasicAppendEntries: async? {}, killLeader={}, numMessages={}", async, killLeader, numMessages); for (RaftServer s : cluster.getServers()) { cluster.restartServer(s.getId(), false); @@ -123,16 +120,16 @@ static void runTestBasicAppendEntries( final long term = leader.getInfo().getCurrentTerm(); final CompletableFuture killAndRestartFollower = killAndRestartServer( - cluster.getFollowers().get(0).getId(), 0, 1000, cluster, LOG); + cluster.getFollowers().get(0).getId(), 0, 1000, cluster, log); final CompletableFuture killAndRestartLeader; if (killLeader) { - LOG.info("killAndRestart leader " + leader.getId()); - killAndRestartLeader = killAndRestartServer(leader.getId(), 2000, 4000, cluster, LOG); + log.info("killAndRestart leader " + leader.getId()); + killAndRestartLeader = killAndRestartServer(leader.getId(), 2000, 4000, cluster, log); } else { killAndRestartLeader = CompletableFuture.completedFuture(null); } - LOG.info(cluster.printServers()); + log.info(cluster.printServers()); final SimpleMessage[] messages = SimpleMessage.create(numMessages); @@ -152,28 +149,26 @@ static void runTestBasicAppendEntries( }); } else { final RaftClientReply reply = client.io().send(message); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } if (async) { f.join(); - Assert.assertEquals(messages.length, asyncReplyCount.get()); + Assertions.assertEquals(messages.length, asyncReplyCount.get()); } } Thread.sleep(cluster.getTimeoutMax().toIntExact(TimeUnit.MILLISECONDS) + 100); - LOG.info(cluster.printAllLogs()); + log.info(cluster.printAllLogs()); killAndRestartFollower.join(); killAndRestartLeader.join(); final List divisions = cluster.getServerAliveStream().collect(Collectors.toList()); for(RaftServer.Division impl: divisions) { - JavaUtils.attempt(() -> RaftTestUtil.assertLogEntries(impl, term, messages), - 50, TimeDuration.valueOf(1, TimeUnit.SECONDS), impl.getId() + " assertLogEntries", LOG); + RaftTestUtil.assertLogEntries(impl, term, messages, 50, log); } } - @Test public void testOldLeaderCommit() throws Exception { runWithNewCluster(NUM_SERVERS, this::runTestOldLeaderCommit); @@ -196,7 +191,7 @@ void runTestOldLeaderCommit(CLUSTER cluster) throws Exception { Thread.sleep(cluster.getTimeoutMax().toLong(TimeUnit.MILLISECONDS) + 100); for (RaftServer.Division followerToSendLog : followersToSendLog) { RaftLog followerLog = followerToSendLog.getRaftLog(); - Assert.assertTrue(RaftTestUtil.logEntriesContains(followerLog, messages)); + Assertions.assertTrue(RaftTestUtil.logEntriesContains(followerLog, messages)); } LOG.info(String.format("killing old leader: %s", leaderId.toString())); @@ -214,11 +209,11 @@ void runTestOldLeaderCommit(CLUSTER cluster) throws Exception { Set followersToSendLogIds = followersToSendLog.stream().map(f -> f.getId()).collect(Collectors.toSet()); - Assert.assertTrue(followersToSendLogIds.contains(newLeaderId)); + Assertions.assertTrue(followersToSendLogIds.contains(newLeaderId)); cluster.getServerAliveStream() .map(RaftServer.Division::getRaftLog) - .forEach(log -> RaftTestUtil.assertLogEntries(log, term, messages)); + .forEach(log -> RaftTestUtil.assertLogEntries(log, term, messages, System.out::println)); } @Test @@ -236,8 +231,8 @@ void runTestOldLeaderNotCommit(CLUSTER cluster) throws Exception { cluster.killServer(followers.get(i).getId()); } } catch (IndexOutOfBoundsException e) { - throw new org.junit.AssumptionViolatedException("The assumption is follower.size() = NUM_SERVERS - 1, " - + "actual NUM_SERVERS is " + NUM_SERVERS + ", and actual follower.size() is " + followers.size(), e); + Assumptions.abort("The assumption is follower.size() = NUM_SERVERS - 1, " + + "actual NUM_SERVERS is " + NUM_SERVERS + ", and actual follower.size() is " + followers.size()); } SimpleMessage[] messages = SimpleMessage.create(1); @@ -271,16 +266,16 @@ static class Client4TestWithLoad extends Thread { final AtomicReference exceptionInClientThread = new AtomicReference<>(); final MiniRaftCluster cluster; - final Logger LOG; + final Logger log; Client4TestWithLoad(int index, int numMessages, boolean useAsync, - MiniRaftCluster cluster, Logger LOG) { + MiniRaftCluster cluster, Logger log) { super("client-" + index); this.index = index; this.messages = SimpleMessage.create(numMessages, index + "-"); this.useAsync = useAsync; this.cluster = cluster; - this.LOG = LOG; + this.log = log; } boolean isRunning() { @@ -295,7 +290,7 @@ public void run() { if (!useAsync) { final RaftClientReply reply = client.io().send(messages[step.getAndIncrement()]); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } else { final CompletableFuture replyFuture = client.async().send(messages[i]); @@ -307,20 +302,20 @@ public void run() { if (step.incrementAndGet() == messages.length) { f.complete(null); } - Assert.assertTrue(r.isSuccess()); + Assertions.assertTrue(r.isSuccess()); }); } } if (useAsync) { f.join(); - Assert.assertTrue(step.get() == messages.length); + Assertions.assertEquals(step.get(), messages.length); } } catch(Exception t) { if (exceptionInClientThread.compareAndSet(null, t)) { - LOG.error(this + " failed", t); + log.error(this + " failed", t); } else { exceptionInClientThread.get().addSuppressed(t); - LOG.error(this + " failed again!", t); + log.error(this + " failed again!", t); } } finally { isRunning.set(false); @@ -339,20 +334,21 @@ public String toString() { } @Test + @Timeout(value = 300) public void testWithLoad() throws Exception { runWithNewCluster(NUM_SERVERS, cluster -> testWithLoad(10, 300, false, cluster, LOG)); } static void testWithLoad(final int numClients, final int numMessages, - boolean useAsync, MiniRaftCluster cluster, Logger LOG) throws Exception { - LOG.info("Running testWithLoad: numClients=" + numClients + boolean useAsync, MiniRaftCluster cluster, Logger log) throws Exception { + log.info("Running testWithLoad: numClients=" + numClients + ", numMessages=" + numMessages + ", async=" + useAsync); waitForLeader(cluster); final List clients = Stream.iterate(0, i -> i+1).limit(numClients) - .map(i -> new Client4TestWithLoad(i, numMessages, useAsync, cluster, LOG)) + .map(i -> new Client4TestWithLoad(i, numMessages, useAsync, cluster, log)) .collect(Collectors.toList()); final AtomicInteger lastStep = new AtomicInteger(); @@ -362,24 +358,24 @@ static void testWithLoad(final int numClients, final int numMessages, @Override public void run() { - LOG.info(cluster.printServers()); - LOG.info(BlockRequestHandlingInjection.getInstance().toString()); - LOG.info(cluster.toString()); - clients.forEach(c -> LOG.info(" " + c)); - JavaUtils.dumpAllThreads(s -> LOG.info(s)); + log.info(cluster.printServers()); + log.info(BlockRequestHandlingInjection.getInstance().toString()); + log.info(cluster.toString()); + clients.forEach(c -> log.info(" " + c)); + JavaUtils.dumpAllThreads(s -> log.info(s)); final int last = lastStep.get(); if (last != previousLastStep) { previousLastStep = last; } else { final RaftServer.Division leader = cluster.getLeader(); - LOG.info("NO PROGRESS at " + last + ", try to restart leader=" + leader); + log.info("NO PROGRESS at " + last + ", try to restart leader=" + leader); if (leader != null) { try { cluster.restartServer(leader.getId(), false); - LOG.info("Restarted leader=" + leader); + log.info("Restarted leader=" + leader); } catch (IOException e) { - LOG.error("Failed to restart leader=" + leader); + log.error("Failed to restart leader=" + leader); } } } @@ -395,7 +391,7 @@ public void run() { } final int n = clients.stream().mapToInt(c -> c.step.get()).sum(); - Assert.assertTrue(n >= lastStep.get()); + Assertions.assertTrue(n >= lastStep.get()); if (n - lastStep.get() < 50 * numClients) { // Change leader at least 50 steps. Thread.sleep(10); @@ -410,10 +406,10 @@ public void run() { RaftTestUtil.changeLeader(cluster, leader.getId()); } } catch (IllegalStateException e) { - LOG.error("Failed to change leader ", e); + log.error("Failed to change leader ", e); } } - LOG.info("Leader change count=" + count); + log.info("Leader change count=" + count); timer.cancel(); for(Client4TestWithLoad c : clients) { @@ -424,7 +420,7 @@ public void run() { } } - public static void testRequestTimeout(boolean async, MiniRaftCluster cluster, Logger LOG) throws Exception { + public static void testRequestTimeout(boolean async, MiniRaftCluster cluster, Logger log) throws Exception { waitForLeader(cluster); final Timestamp startTime = Timestamp.currentTime(); try (final RaftClient client = cluster.createClient()) { @@ -449,14 +445,19 @@ public static void testRequestTimeout(boolean async, MiniRaftCluster cluster, Lo // The duration for which the client waits should be more than the retryCacheExpiryDuration. final TimeDuration duration = startTime.elapsedTime(); TimeDuration retryCacheExpiryDuration = RaftServerConfigKeys.RetryCache.expiryTime(cluster.getProperties()); - Assert.assertTrue(duration.compareTo(retryCacheExpiryDuration) >= 0); + Assertions.assertTrue(duration.compareTo(retryCacheExpiryDuration) >= 0); } } - public static void testStateMachineMetrics(boolean async, - MiniRaftCluster cluster, Logger LOG) throws Exception { + @Flaky("RATIS-2262") + @Test + public void testStateMachineMetrics() throws Exception { + runWithNewCluster(NUM_SERVERS, cluster -> runTestStateMachineMetrics(false, cluster)); + } + + static void runTestStateMachineMetrics(boolean async, MiniRaftCluster cluster) throws Exception { RaftServer.Division leader = waitForLeader(cluster); - try (final RaftClient client = cluster.createClient()) { + try (final RaftClient client = cluster.createClient(leader.getId())) { Gauge appliedIndexGauge = getStatemachineGaugeWithName(leader, STATEMACHINE_APPLIED_INDEX_GAUGE); Gauge smAppliedIndexGauge = getStatemachineGaugeWithName(leader, @@ -477,10 +478,10 @@ public static void testStateMachineMetrics(boolean async, long smAppliedIndexAfter = (Long) smAppliedIndexGauge.getValue(); checkFollowerCommitLagsLeader(cluster); - Assert.assertTrue("StateMachine Applied Index not incremented", - appliedIndexAfter > appliedIndexBefore); - Assert.assertTrue("StateMachine Apply completed Index not incremented", - smAppliedIndexAfter > smAppliedIndexBefore); + Assertions.assertTrue(appliedIndexAfter > appliedIndexBefore, + "StateMachine Applied Index not incremented"); + Assertions.assertTrue(smAppliedIndexAfter > smAppliedIndexBefore, + "StateMachine Apply completed Index not incremented"); } } @@ -493,12 +494,12 @@ private static void checkFollowerCommitLagsLeader(MiniRaftCluster cluster) { for (RaftServer.Division f : followers) { final RaftGroupMemberId follower = f.getMemberId(); Gauge followerCommitGauge = ServerMetricsTestUtils.getPeerCommitIndexGauge(leader, follower.getPeerId()); - Assert.assertTrue((Long)leaderCommitGauge.getValue() >= + Assertions.assertTrue((Long)leaderCommitGauge.getValue() >= (Long)followerCommitGauge.getValue()); Gauge followerMetric = ServerMetricsTestUtils.getPeerCommitIndexGauge(follower, follower.getPeerId()); System.out.println(followerCommitGauge.getValue()); System.out.println(followerMetric.getValue()); - Assert.assertTrue((Long)followerCommitGauge.getValue() <= (Long)followerMetric.getValue()); + Assertions.assertTrue((Long)followerCommitGauge.getValue() <= (Long)followerMetric.getValue()); } } @@ -509,7 +510,7 @@ private static Gauge getStatemachineGaugeWithName(RaftServer.Division server, St RATIS_STATEMACHINE_METRICS, RATIS_STATEMACHINE_METRICS_DESC); Optional metricRegistry = MetricRegistries.global().get(info); - Assert.assertTrue(metricRegistry.isPresent()); + Assertions.assertTrue(metricRegistry.isPresent()); return ServerMetricsTestUtils.getGaugeWithName(gaugeName, metricRegistry::get); } diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftExceptionBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/RaftExceptionBaseTest.java index 9b6d811211..5fa03328e4 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RaftExceptionBaseTest.java +++ b/ratis-server/src/test/java/org/apache/ratis/RaftExceptionBaseTest.java @@ -138,7 +138,7 @@ void runTestNotLeaderExceptionWithReconf(CLUSTER cluster) throws Exception { @Test public void testGroupMismatchException() throws Exception { - runWithSameCluster(NUM_PEERS, this::runTestGroupMismatchException); + runWithNewCluster(NUM_PEERS, this::runTestGroupMismatchException); } void runTestGroupMismatchException(CLUSTER cluster) throws Exception { @@ -171,7 +171,7 @@ void runTestGroupMismatchException(CLUSTER cluster) throws Exception { @Test public void testStaleReadException() throws Exception { - runWithSameCluster(NUM_PEERS, this::runTestStaleReadException); + runWithNewCluster(NUM_PEERS, this::runTestStaleReadException); } void runTestStaleReadException(CLUSTER cluster) throws Exception { @@ -186,7 +186,7 @@ void runTestStaleReadException(CLUSTER cluster) throws Exception { @Test public void testLogAppenderBufferCapacity() throws Exception { - runWithSameCluster(NUM_PEERS, this::runTestLogAppenderBufferCapacity); + runWithNewCluster(NUM_PEERS, this::runTestLogAppenderBufferCapacity); } void runTestLogAppenderBufferCapacity(CLUSTER cluster) throws Exception { diff --git a/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java b/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java index 1d45c68214..36b5f36bc3 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java +++ b/ratis-server/src/test/java/org/apache/ratis/RaftTestUtil.java @@ -25,34 +25,42 @@ import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.impl.BlockRequestHandlingInjection; import org.apache.ratis.server.impl.DelayLocalExecutionInjection; import org.apache.ratis.server.impl.MiniRaftCluster; +import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.LogEntryHeader; import org.apache.ratis.server.raftlog.LogProtoUtils; import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.server.raftlog.RaftLogBase; -import org.apache.ratis.thirdparty.com.google.common.base.Preconditions; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.apache.ratis.util.AutoCloseableLock; import org.apache.ratis.util.CollectionUtils; import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.ProtoUtils; import org.apache.ratis.util.TimeDuration; import org.junit.Assert; import org.junit.AssumptionViolatedException; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.EnumMap; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -144,8 +152,9 @@ static RaftPeerId waitAndKillLeader(MiniRaftCluster cluster) throws InterruptedE static void waitFor(Supplier check, int checkEveryMillis, int waitForMillis) throws TimeoutException, InterruptedException { - Preconditions.checkNotNull(check); - Preconditions.checkArgument(waitForMillis >= checkEveryMillis); + Objects.requireNonNull(check, "check == null"); + Preconditions.assertTrue(waitForMillis >= checkEveryMillis, + () -> "waitFor: " + waitForMillis + " < checkEvery: " + checkEveryMillis); long st = System.currentTimeMillis(); boolean result = check.get(); @@ -180,8 +189,11 @@ static boolean logEntriesContains(RaftLog log, long startIndex, long endIndex, S log.get(termIndices[idxEntries].getIndex()).getStateMachineLogEntry().getLogData().toByteArray())) { ++idxExpected; } - } catch (IOException e) { - throw new RuntimeException(e); + } catch (Exception e) { + throw new IllegalStateException("Failed logEntriesContains: startIndex=" + startIndex + + ", endIndex=" + endIndex + + ", #expectedMessages=" + expectedMessages.length + + ", log=" + log, e); } ++idxEntries; } @@ -248,19 +260,16 @@ static void assertLogEntries(MiniRaftCluster cluster, SimpleMessage expectedMess } } - static void assertLogEntries(RaftServer.Division server, long expectedTerm, SimpleMessage... expectedMessages) { - LOG.info("checking raft log for {}", server.getMemberId()); - final RaftLog log = server.getRaftLog(); - try { - RaftTestUtil.assertLogEntries(log, expectedTerm, expectedMessages); - } catch (AssertionError e) { - LOG.error("Unexpected raft log in {}", server.getMemberId(), e); - throw e; - } + static void assertLogEntries(RaftServer.Division server, long expectedTerm, SimpleMessage[] expectedMessages, + int numAttempts, Logger log) throws Exception { + final String name = server.getId() + " assertLogEntries"; + final Function> print = i -> i < numAttempts? s -> {}: System.out::println; + JavaUtils.attempt(i -> assertLogEntries(server.getRaftLog(), expectedTerm, expectedMessages, print.apply(i)), + numAttempts, TimeDuration.ONE_SECOND, () -> name, log); } static Iterable getLogEntryProtos(RaftLog log) { - return CollectionUtils.as(log.getEntries(0, Long.MAX_VALUE), ti -> { + return CollectionUtils.as(log.getEntries(0, log.getLastEntryTermIndex().getIndex() + 1), ti -> { try { return log.get(ti.getIndex()); } catch (IOException exception) { @@ -269,17 +278,54 @@ static Iterable getLogEntryProtos(RaftLog log) { }); } - static List getStateMachineLogEntries(RaftLog log) { + Comparator LOG_ENTRY_PROTO_COMPARATOR = Comparator.comparing( + e -> e.getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer()); + Comparator SIMPLE_MESSAGE_COMPARATOR = Comparator.comparing( + m -> m.getContent().asReadOnlyByteBuffer()); + + /** @return a map of message-array-index to {@link LogEntryProto} for the entries found in the given log. */ + static Map getStateMachineLogEntries(RaftLog log, SimpleMessage[] messages) { + if (messages.length == 0) { + return Collections.emptyMap(); + } + final List entries = getStateMachineLogEntries(log, s -> {}); + if (entries.isEmpty()) { + return Collections.emptyMap(); + } + + entries.sort(LOG_ENTRY_PROTO_COMPARATOR); + Arrays.sort(messages, SIMPLE_MESSAGE_COMPARATOR); + + final Map found = new HashMap<>(); + for (int e = 0, m = 0; e < entries.size() && m < messages.length; ) { + final int diff = messages[m].getContent().asReadOnlyByteBuffer().compareTo( + entries.get(e).getStateMachineLogEntry().getLogData().asReadOnlyByteBuffer()); + if (diff == 0) { + found.put(m, entries.get(e)); + m++; + e++; + } else if (diff < 0) { + m++; // message < entry + } else { + e++; // message > entry + } + } + + Assertions.assertEquals(messages.length, found.size()); + return found; + } + + static List getStateMachineLogEntries(RaftLog log, Consumer print) { final List entries = new ArrayList<>(); for (LogEntryProto e : getLogEntryProtos(log)) { final String s = LogProtoUtils.toLogEntryString(e); if (e.hasStateMachineLogEntry()) { - LOG.info(s + ", " + e.getStateMachineLogEntry().toString().trim().replace("\n", ", ")); + print.accept(entries.size() + ") " + s); entries.add(e); } else if (e.hasConfigurationEntry()) { - LOG.info("Found {}, ignoring it.", s); + print.accept("Ignoring " + s); } else if (e.hasMetadataEntry()) { - LOG.info("Found {}, ignoring it.", s); + print.accept("Ignoring " + s); } else { throw new AssertionError("Unexpected LogEntryBodyCase " + e.getLogEntryBodyCase() + " at " + s); } @@ -287,13 +333,15 @@ static List getStateMachineLogEntries(RaftLog log) { return entries; } - static void assertLogEntries(RaftLog log, long expectedTerm, SimpleMessage... expectedMessages) { - final List entries = getStateMachineLogEntries(log); + static Void assertLogEntries(RaftLog log, long expectedTerm, SimpleMessage[] expectedMessages, + Consumer print) { + final List entries = getStateMachineLogEntries(log, print); try { assertLogEntries(entries, expectedTerm, expectedMessages); } catch(Exception t) { throw new AssertionError("entries: " + entries, t); } + return null; } static void assertLogEntries(List entries, long expectedTerm, SimpleMessage... expectedMessages) { @@ -307,8 +355,7 @@ static void assertLogEntries(List entries, long expectedTerm, Sim } Assert.assertTrue(e.getIndex() > logIndex); logIndex = e.getIndex(); - Assert.assertArrayEquals(expectedMessages[i].getContent().toByteArray(), - e.getStateMachineLogEntry().getLogData().toByteArray()); + Assert.assertEquals(expectedMessages[i].getContent(), e.getStateMachineLogEntry().getLogData()); } } @@ -366,8 +413,8 @@ public ByteString getContent() { } class SimpleOperation { - private static final ClientId clientId = ClientId.randomId(); - private static final AtomicLong callId = new AtomicLong(); + private static final ClientId CLIENT_ID = ClientId.randomId(); + private static final AtomicLong CALL_ID = new AtomicLong(); private final String op; private final StateMachineLogEntryProto smLogEntryProto; @@ -377,7 +424,7 @@ public SimpleOperation(String op) { } public SimpleOperation(String op, boolean hasStateMachineData) { - this(clientId, callId.incrementAndGet(), op, hasStateMachineData); + this(CLIENT_ID, CALL_ID.incrementAndGet(), op, hasStateMachineData); } private SimpleOperation(ClientId clientId, long callId, String op, boolean hasStateMachineData) { @@ -422,6 +469,15 @@ static void delay(IntSupplier getDelayMs) throws InterruptedException { } } + static List getPeersWithPriority(List peers, RaftPeer suggestedLeader) { + List peersWithPriority = new ArrayList<>(); + for (RaftPeer peer : peers) { + final int priority = peer.equals(suggestedLeader) ? 2 : 1; + peersWithPriority.add(RaftPeer.newBuilder(peer).setPriority(priority).build()); + } + return peersWithPriority; + } + static RaftPeerId changeLeader(MiniRaftCluster cluster, RaftPeerId oldLeader) throws Exception { return changeLeader(cluster, oldLeader, AssumptionViolatedException::new); @@ -472,6 +528,20 @@ static void blockQueueAndSetDelay(Iterable servers, Thread.sleep(3 * maxTimeout.toLong(TimeUnit.MILLISECONDS)); } + static void isolate(MiniRaftCluster cluster, RaftPeerId id) { + try { + BlockRequestHandlingInjection.getInstance().blockReplier(id.toString()); + cluster.setBlockRequestsFrom(id.toString(), true); + } catch (Exception e) { + e.printStackTrace(); + } + } + + static void deIsolate(MiniRaftCluster cluster, RaftPeerId id) { + BlockRequestHandlingInjection.getInstance().unblockReplier(id.toString()); + cluster.setBlockRequestsFrom(id.toString(), false); + } + static Thread sendMessageInNewThread(MiniRaftCluster cluster, RaftPeerId leaderId, SimpleMessage... messages) { Thread t = new Thread(() -> { try (final RaftClient client = cluster.createClient(leaderId)) { @@ -491,7 +561,7 @@ static void assertSameLog(RaftLog expected, RaftLog computed) throws Exception { final long lastIndex = expected.getNextIndex() - 1; Assert.assertEquals(expected.getLastEntryTermIndex().getIndex(), lastIndex); for(long i = 0; i < lastIndex; i++) { - Assert.assertEquals(expected.get(i), computed.get(i)); + Assert.assertEquals("Checking " + TermIndex.valueOf(expected.get(i)), expected.get(i), computed.get(i)); } } @@ -525,4 +595,18 @@ static void assertSuccessReply(RaftClientReply reply) { Assert.assertNotNull("reply == null", reply); Assert.assertTrue("reply is not success: " + reply, reply.isSuccess()); } + + static void gc() throws InterruptedException { + // use WeakReference to detect gc + Object obj = new Object(); + final WeakReference weakRef = new WeakReference<>(obj); + obj = null; + + // loop until gc has completed. + for (int i = 0; weakRef.get() != null; i++) { + LOG.info("gc {}", i); + System.gc(); + Thread.sleep(100); + } + } } diff --git a/ratis-server/src/test/java/org/apache/ratis/ReadOnlyRequestTests.java b/ratis-server/src/test/java/org/apache/ratis/ReadOnlyRequestTests.java index eea75592ef..ead2a8be8e 100644 --- a/ratis-server/src/test/java/org/apache/ratis/ReadOnlyRequestTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/ReadOnlyRequestTests.java @@ -19,6 +19,7 @@ import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.protocol.Message; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeerId; @@ -366,10 +367,11 @@ private void timeoutIncrement() { @Override public CompletableFuture applyTransaction(TransactionContext trx) { - LOG.debug("apply trx with index=" + trx.getLogEntry().getIndex()); - updateLastAppliedTermIndex(trx.getLogEntry().getTerm(), trx.getLogEntry().getIndex()); + final RaftProtos.LogEntryProto logEntry = trx.getLogEntryUnsafe(); + LOG.debug("apply trx with index=" + logEntry.getIndex()); + updateLastAppliedTermIndex(logEntry.getTerm(), logEntry.getIndex()); - String command = trx.getLogEntry().getStateMachineLogEntry() + String command = logEntry.getStateMachineLogEntry() .getLogData().toString(StandardCharsets.UTF_8); LOG.info("receive command: {}", command); diff --git a/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java b/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java index 18561ee65c..5a363bf081 100644 --- a/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/RetryCacheTests.java @@ -35,11 +35,12 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.util.Arrays; +import java.util.Collections; import java.util.concurrent.TimeUnit; import static java.util.Arrays.asList; @@ -84,12 +85,13 @@ void runTestBasicRetry(CLUSTER cluster) throws Exception { } public static void assertReply(RaftClientReply reply, RaftClient client, long callId) { - Assert.assertEquals(client.getId(), reply.getClientId()); - Assert.assertEquals(callId, reply.getCallId()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertEquals(client.getId(), reply.getClientId()); + Assertions.assertEquals(callId, reply.getCallId()); + Assertions.assertTrue(reply.isSuccess()); } - public void assertServer(MiniRaftCluster cluster, ClientId clientId, long callId, long oldLastApplied) throws Exception { + public void assertServer(MiniRaftCluster cluster, ClientId clientId, long callId, long oldLastApplied) + throws Exception { final long leaderApplied = cluster.getLeader().getInfo().getLastAppliedIndex(); // make sure retry cache has the entry for (RaftServer.Division server : cluster.iterateDivisions()) { @@ -97,10 +99,10 @@ public void assertServer(MiniRaftCluster cluster, ClientId clientId, long callId if (server.getInfo().getLastAppliedIndex() < leaderApplied) { Thread.sleep(1000); } - Assert.assertEquals(2, server.getRetryCache().getStatistics().size()); - Assert.assertNotNull(RetryCacheTestUtil.get(server, clientId, callId)); + Assertions.assertEquals(2, server.getRetryCache().getStatistics().size()); + Assertions.assertNotNull(RetryCacheTestUtil.get(server, clientId, callId)); // make sure there is only one log entry committed - Assert.assertEquals(1, count(server.getRaftLog(), oldLastApplied + 1)); + Assertions.assertEquals(1, count(server.getRaftLog(), oldLastApplied + 1)); } } @@ -136,19 +138,18 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception { final long oldLastApplied = cluster.getLeader().getInfo().getLastAppliedIndex(); // trigger the reconfiguration, make sure the original leader is kicked out - PeerChanges change = cluster.addNewPeers(2, true); - RaftPeer[] allPeers = cluster.removePeers(2, true, - asList(change.newPeers)).allPeersInNewConf; + final PeerChanges change = cluster.removePeers(2, true, Collections.emptyList()); + final RaftPeer[] allPeers = change.allPeersInNewConf; // trigger setConfiguration RaftServerTestUtil.runWithMinorityPeers(cluster, Arrays.asList(allPeers), peers -> cluster.setConfiguration(peers.toArray(RaftPeer.emptyArray()))); final RaftPeerId newLeaderId = JavaUtils.attemptRepeatedly(() -> { final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId(); - Assert.assertNotEquals(leaderId, id); + Assertions.assertNotEquals(leaderId, id); return id; }, 10, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS), "wait for a leader different than " + leaderId, LOG); - Assert.assertNotEquals(leaderId, newLeaderId); + Assertions.assertNotEquals(leaderId, newLeaderId); // same clientId and callId in the request r = cluster.newRaftClientRequest(client.getId(), newLeaderId, callId, new SimpleMessage("message")); @@ -164,7 +165,7 @@ void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception { } // check the new leader and make sure the retry did not get committed - Assert.assertEquals(0, count(cluster.getLeader().getRaftLog(), oldLastApplied + 1)); + Assertions.assertEquals(0, count(cluster.getLeader().getRaftLog(), oldLastApplied + 1)); } } } diff --git a/ratis-server/src/test/java/org/apache/ratis/WatchRequestTests.java b/ratis-server/src/test/java/org/apache/ratis/WatchRequestTests.java index 1f19f9d1c2..b842ee9db5 100644 --- a/ratis-server/src/test/java/org/apache/ratis/WatchRequestTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/WatchRequestTests.java @@ -31,8 +31,10 @@ import org.apache.ratis.retry.RetryPolicy; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.RaftServerConfigKeys.Watch; import org.apache.ratis.server.impl.MiniRaftCluster; import org.apache.ratis.server.impl.RaftServerTestUtil; +import org.apache.ratis.server.metrics.RaftServerMetricsImpl; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.Slf4jUtils; @@ -139,26 +141,26 @@ public String toString() { } } - static void runTest(CheckedConsumer testCase, MiniRaftCluster cluster, Logger LOG) + static void runTest(CheckedConsumer testCase, MiniRaftCluster cluster, Logger log) throws Exception { try(final RaftClient client = cluster.createClient(RaftTestUtil.waitForLeader(cluster).getId())) { final int[] numMessages = {1, 10, 20}; for(int n : numMessages) { - final TestParameters p = new TestParameters(n, client, cluster, LOG); - LOG.info("{}) {}, {}", n, p, cluster.printServers()); + final TestParameters p = new TestParameters(n, client, cluster, log); + log.info("{}) {}, {}", n, p, cluster.printServers()); testCase.accept(p); } } } static void runSingleTest(CheckedConsumer testCase, - MiniRaftCluster cluster, Logger LOG) + MiniRaftCluster cluster, Logger log) throws Exception { try(final RaftClient client = cluster.createClient(RaftTestUtil.waitForLeader(cluster).getId())) { final int[] numMessages = {1}; for(int n : numMessages) { - final TestParameters p = new TestParameters(n, client, cluster, LOG); - LOG.info("{}) {}, {}", n, p, cluster.printServers()); + final TestParameters p = new TestParameters(n, client, cluster, log); + log.info("{}) {}, {}", n, p, cluster.printServers()); testCase.accept(p); } } @@ -174,7 +176,8 @@ static class WatchReplies { WatchReplies(long logIndex, CompletableFuture majority, CompletableFuture all, - CompletableFuture majorityCommitted, CompletableFuture allCommitted, Logger log) { + CompletableFuture majorityCommitted, CompletableFuture allCommitted, + Logger log) { this.logIndex = logIndex; this.majority = majority; this.all = all; @@ -216,19 +219,19 @@ RaftClientReply get(CompletableFuture f, String name) throws Ex } static void runTestWatchRequestAsync(TestParameters p) throws Exception { - final Logger LOG = p.log; + final Logger log = p.log; final MiniRaftCluster cluster = p.cluster; final int numMessages = p.numMessages; // blockStartTransaction of the leader so that no transaction can be committed MAJORITY final RaftServer.Division leader = cluster.getLeader(); - LOG.info("block leader {}", leader.getId()); + log.info("block leader {}", leader.getId()); SimpleStateMachine4Testing.get(leader).blockStartTransaction(); // blockFlushStateMachineData a follower so that no transaction can be ALL_COMMITTED final List followers = cluster.getFollowers(); final RaftServer.Division blockedFollower = followers.get(ThreadLocalRandom.current().nextInt(followers.size())); - LOG.info("block follower {}", blockedFollower.getId()); + log.info("block follower {}", blockedFollower.getId()); SimpleStateMachine4Testing.get(blockedFollower).blockFlushStateMachineData(); // send a message @@ -247,9 +250,9 @@ static void runTestWatchRequestAsync(TestParameters p) throws Exception { // unblock leader so that the transaction can be committed. SimpleStateMachine4Testing.get(leader).unblockStartTransaction(); - LOG.info("unblock leader {}", leader.getId()); + log.info("unblock leader {}", leader.getId()); - checkMajority(replies, watches, LOG); + checkMajority(replies, watches, log); Assert.assertEquals(numMessages, watches.size()); @@ -259,16 +262,16 @@ static void runTestWatchRequestAsync(TestParameters p) throws Exception { assertNotDone(watches.stream().map(CompletableFuture::join).map(w -> w.allCommitted)); // unblock follower so that the transaction can be replicated and committed to all. - LOG.info("unblock follower {}", blockedFollower.getId()); + log.info("unblock follower {}", blockedFollower.getId()); SimpleStateMachine4Testing.get(blockedFollower).unblockFlushStateMachineData(); - checkAll(watches, LOG); + checkAll(watches, log); } static void checkMajority(List> replies, - List> watches, Logger LOG) throws Exception { + List> watches, Logger log) throws Exception { for(int i = 0; i < replies.size(); i++) { final RaftClientReply reply = replies.get(i).get(GET_TIMEOUT_SECOND, TimeUnit.SECONDS); - LOG.info("checkMajority {}: receive {}", i, reply); + log.info("checkMajority {}: receive {}", i, reply); final long logIndex = reply.getLogIndex(); Assert.assertTrue(reply.isSuccess()); @@ -294,11 +297,11 @@ static void checkMajority(List> replies, } } - static void checkAll(List> watches, Logger LOG) throws Exception { + static void checkAll(List> watches, Logger log) throws Exception { for(int i = 0; i < watches.size(); i++) { final WatchReplies watchReplies = watches.get(i).get(GET_TIMEOUT_SECOND, TimeUnit.SECONDS); final long logIndex = watchReplies.logIndex; - LOG.info("checkAll {}: logIndex={}", i, logIndex); + log.info("checkAll {}: logIndex={}", i, logIndex); final RaftClientReply watchAllReply = watchReplies.getAll(); final RaftClientReply watchAllCommittedReply = watchReplies.getAllCommitted(); @@ -334,14 +337,14 @@ public void testWatchRequestAsyncChangeLeader() throws Exception { } static void runTestWatchRequestAsyncChangeLeader(TestParameters p) throws Exception { - final Logger LOG = p.log; + final Logger log = p.log; final MiniRaftCluster cluster = p.cluster; final int numMessages = p.numMessages; // blockFlushStateMachineData a follower so that no transaction can be ALL_COMMITTED final List followers = cluster.getFollowers(); final RaftServer.Division blockedFollower = followers.get(ThreadLocalRandom.current().nextInt(followers.size())); - LOG.info("block follower {}", blockedFollower.getId()); + log.info("block follower {}", blockedFollower.getId()); SimpleStateMachine4Testing.get(blockedFollower).blockFlushStateMachineData(); final List> replies = new ArrayList<>(); @@ -353,7 +356,7 @@ static void runTestWatchRequestAsyncChangeLeader(TestParameters p) throws Except Assert.assertEquals(numMessages, watches.size()); // since only one follower is blocked commit, requests can be committed MAJORITY and ALL but not ALL_COMMITTED. - checkMajority(replies, watches, LOG); + checkMajority(replies, watches, log); TimeUnit.SECONDS.sleep(1); assertNotDone(watches.stream().map(CompletableFuture::join).map(w -> w.allCommitted)); @@ -363,8 +366,8 @@ static void runTestWatchRequestAsyncChangeLeader(TestParameters p) throws Except // unblock follower so that the transaction can be replicated and committed to all. SimpleStateMachine4Testing.get(blockedFollower).unblockFlushStateMachineData(); - LOG.info("unblock follower {}", blockedFollower.getId()); - checkAll(watches, LOG); + log.info("unblock follower {}", blockedFollower.getId()); + checkAll(watches, log); } @Test @@ -382,7 +385,7 @@ public void testWatchRequestTimeout() throws Exception { } static void runTestWatchRequestTimeout(TestParameters p) throws Exception { - final Logger LOG = p.log; + final Logger log = p.log; final MiniRaftCluster cluster = p.cluster; final int numMessages = p.numMessages; @@ -392,13 +395,13 @@ static void runTestWatchRequestTimeout(TestParameters p) throws Exception { // blockStartTransaction of the leader so that no transaction can be committed MAJORITY final RaftServer.Division leader = cluster.getLeader(); - LOG.info("block leader {}", leader.getId()); + log.info("block leader {}", leader.getId()); SimpleStateMachine4Testing.get(leader).blockStartTransaction(); // blockFlushStateMachineData a follower so that no transaction can be ALL_COMMITTED final List followers = cluster.getFollowers(); final RaftServer.Division blockedFollower = followers.get(ThreadLocalRandom.current().nextInt(followers.size())); - LOG.info("block follower {}", blockedFollower.getId()); + log.info("block follower {}", blockedFollower.getId()); SimpleStateMachine4Testing.get(blockedFollower).blockFlushStateMachineData(); // send a message @@ -417,13 +420,13 @@ static void runTestWatchRequestTimeout(TestParameters p) throws Exception { // unblock leader so that the transaction can be committed. SimpleStateMachine4Testing.get(leader).unblockStartTransaction(); - LOG.info("unblock leader {}", leader.getId()); + log.info("unblock leader {}", leader.getId()); - checkMajority(replies, watches, LOG); - checkTimeout(replies, watches, LOG); + checkMajority(replies, watches, log); + checkTimeout(replies, watches, log); SimpleStateMachine4Testing.get(blockedFollower).unblockFlushStateMachineData(); - LOG.info("unblock follower {}", blockedFollower.getId()); + log.info("unblock follower {}", blockedFollower.getId()); } @Test @@ -444,7 +447,7 @@ public void testWatchRequestClientTimeout() throws Exception { } static void runTestWatchRequestClientTimeout(TestParameters p) throws Exception { - final Logger LOG = p.log; + final Logger log = p.log; CompletableFuture watchReply; // watch 1000 which will never be committed @@ -457,7 +460,7 @@ static void runTestWatchRequestClientTimeout(TestParameters p) throws Exception watchReply.get(); fail("runTestWatchRequestClientTimeout failed"); } catch (Exception ex) { - LOG.error("error occurred", ex); + log.error("error occurred", ex); Assert.assertTrue(ex.getCause().getClass() == AlreadyClosedException.class || ex.getCause().getClass() == RaftRetryFailureException.class); if (ex.getCause() != null) { @@ -469,11 +472,64 @@ static void runTestWatchRequestClientTimeout(TestParameters p) throws Exception } } + @Test + public void testWatchMetrics() throws Exception { + final RaftProperties p = getProperties(); + RaftServerConfigKeys.Watch.setElementLimit(p, 10); + RaftServerConfigKeys.Watch.setTimeout(p, TimeDuration.valueOf(2, TimeUnit.SECONDS)); + try { + runWithNewCluster(NUM_SERVERS, + cluster -> runSingleTest(WatchRequestTests::runTestWatchMetrics, cluster, LOG)); + } finally { + RaftServerConfigKeys.Watch.setElementLimit(p, Watch.ELEMENT_LIMIT_DEFAULT); + RaftServerConfigKeys.Watch.setTimeout(p, RaftServerConfigKeys.Watch.TIMEOUT_DEFAULT); + } + } + + static RaftServerMetricsImpl getRaftServerMetrics(RaftServer.Division division) { + return (RaftServerMetricsImpl) division.getRaftServerMetrics(); + } + + static void runTestWatchMetrics(TestParameters p) throws Exception { + final MiniRaftCluster cluster = p.cluster; + + List clients = new ArrayList<>(); + + final ReplicationLevel replicationLevel = ReplicationLevel.MAJORITY; + try { + long initialWatchRequestTimeoutCount = getRaftServerMetrics(cluster.getLeader()) + .getNumWatchRequestsTimeout(replicationLevel).getCount(); + long initialLimitHit = getRaftServerMetrics(cluster.getLeader()) + .getNumWatchRequestQueueLimitHits(replicationLevel).getCount(); + + int uncommittedBaseIndex = 10000; + // Logs with indices 10001 - 10011 will never be committed, so it should fail with NotReplicatedException + for (int i = 1; i <= 11; i++) { + RaftClient client = cluster.createClient(cluster.getLeader().getId(), RetryPolicies.noRetry()); + clients.add(client); + client.async().watch(uncommittedBaseIndex + i, replicationLevel); + } + + // All the watch timeout for each unique index should increment the metric + RaftTestUtil.waitFor(() -> getRaftServerMetrics(cluster.getLeader()) + .getNumWatchRequestsTimeout(replicationLevel).getCount() == initialWatchRequestTimeoutCount + 10, + 300, 5000); + // There are 11 pending watch request, but the pending watch request limit is 10 + RaftTestUtil.waitFor(() -> getRaftServerMetrics(cluster.getLeader()) + .getNumWatchRequestQueueLimitHits(replicationLevel).getCount() == + initialLimitHit + 1, 300, 5000); + } finally { + for(RaftClient client : clients) { + client.close(); + } + } + } + static void checkTimeout(List> replies, - List> watches, Logger LOG) throws Exception { + List> watches, Logger log) throws Exception { for(int i = 0; i < replies.size(); i++) { final RaftClientReply reply = replies.get(i).get(GET_TIMEOUT_SECOND, TimeUnit.SECONDS); - LOG.info("checkTimeout {}: receive {}", i, reply); + log.info("checkTimeout {}: receive {}", i, reply); final long logIndex = reply.getLogIndex(); Assert.assertTrue(reply.isSuccess()); @@ -503,5 +559,6 @@ static void assertNotReplicatedException(long logIndex, ReplicationLevel replica Assert.assertNotNull(nre); Assert.assertEquals(logIndex, nre.getLogIndex()); Assert.assertEquals(replication, nre.getRequiredReplication()); + Assert.assertNotNull(nre.getCommitInfos()); } } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/BlockRequestHandlingInjection.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/BlockRequestHandlingInjection.java index 3487aa472a..9af62bc430 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/BlockRequestHandlingInjection.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/BlockRequestHandlingInjection.java @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; /** Inject code to block a server from handling incoming requests. */ -public class BlockRequestHandlingInjection implements CodeInjectionForTesting.Code { +public final class BlockRequestHandlingInjection implements CodeInjectionForTesting.Code { private static final BlockRequestHandlingInjection INSTANCE = new BlockRequestHandlingInjection(); diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java index 311a2150d6..a06336f912 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/GroupManagementBaseTest.java @@ -54,6 +54,8 @@ import java.util.Random; import java.util.stream.Collectors; +import static org.apache.ratis.RaftTestUtil.getPeersWithPriority; + public abstract class GroupManagementBaseTest extends BaseTest { static final Logger LOG = LoggerFactory.getLogger(GroupManagementBaseTest.class); @@ -63,20 +65,20 @@ public abstract class GroupManagementBaseTest extends BaseTest { Slf4jUtils.setLogLevel(RaftClient.LOG, Level.DEBUG); } - static final RaftProperties prop = new RaftProperties(); + static final RaftProperties PROP = new RaftProperties(); static { // avoid flaky behaviour in CI environment - RaftServerConfigKeys.Rpc.setTimeoutMin(prop, TimeDuration.valueOf(1500, TimeUnit.MILLISECONDS)); - RaftServerConfigKeys.Rpc.setTimeoutMax(prop, TimeDuration.valueOf(2000, TimeUnit.MILLISECONDS)); + RaftServerConfigKeys.Rpc.setTimeoutMin(PROP, TimeDuration.valueOf(1500, TimeUnit.MILLISECONDS)); + RaftServerConfigKeys.Rpc.setTimeoutMax(PROP, TimeDuration.valueOf(2000, TimeUnit.MILLISECONDS)); // it takes 5s+ to finish the blocking group add call - RaftClientConfigKeys.Rpc.setRequestTimeout(prop, TimeDuration.valueOf(12, TimeUnit.SECONDS)); + RaftClientConfigKeys.Rpc.setRequestTimeout(PROP, TimeDuration.valueOf(12, TimeUnit.SECONDS)); } public abstract MiniRaftCluster.Factory getClusterFactory(); public MiniRaftCluster getCluster(int peerNum) { - return getClusterFactory().newCluster(peerNum, prop); + return getClusterFactory().newCluster(peerNum, PROP); } @Test diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java index 6453e8e944..f35626894f 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/LeaderElectionTests.java @@ -21,14 +21,15 @@ import org.apache.ratis.RaftTestUtil; import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; -import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl; import org.apache.ratis.metrics.impl.DefaultTimekeeperImpl; +import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl; import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.SetConfigurationRequest; import org.apache.ratis.protocol.exceptions.LeaderSteppingDownException; import org.apache.ratis.protocol.exceptions.TransferLeadershipException; import org.apache.ratis.server.DivisionInfo; @@ -36,17 +37,19 @@ import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.metrics.LeaderElectionMetrics; import org.apache.ratis.server.protocol.TermIndex; -import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtils; -import org.apache.ratis.util.ExitUtils; +import org.apache.ratis.test.tag.Flaky; +import org.apache.ratis.thirdparty.com.codahale.metrics.Timer; +import org.apache.ratis.util.CodeInjectionForTesting; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; import org.apache.ratis.util.function.CheckedBiConsumer; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; +import org.slf4j.event.Level; import java.io.IOException; import java.util.ArrayList; @@ -60,21 +63,19 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.apache.ratis.RaftTestUtil.getPeersWithPriority; import static org.apache.ratis.RaftTestUtil.waitForLeader; import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LAST_LEADER_ELECTION_ELAPSED_TIME; import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LEADER_ELECTION_COUNT_METRIC; -import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LEADER_ELECTION_TIME_TAKEN; import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LEADER_ELECTION_TIMEOUT_COUNT_METRIC; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assert.assertNotNull; +import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LEADER_ELECTION_TIME_TAKEN; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.apache.ratis.thirdparty.com.codahale.metrics.Timer; -import org.slf4j.event.Level; - public abstract class LeaderElectionTests extends BaseTest implements MiniRaftCluster.Factory.Get { @@ -86,31 +87,105 @@ public abstract class LeaderElectionTests @Test public void testBasicLeaderElection() throws Exception { LOG.info("Running testBasicLeaderElection"); - final MiniRaftCluster cluster = newCluster(5); - cluster.start(); + runWithNewCluster(5, this::runTestBasicLeaderElection); + } + + void runTestBasicLeaderElection(MiniRaftCluster cluster) throws Exception { RaftTestUtil.waitAndKillLeader(cluster); RaftTestUtil.waitAndKillLeader(cluster); RaftTestUtil.waitAndKillLeader(cluster); testFailureCase("waitForLeader after killed a majority of servers", () -> RaftTestUtil.waitForLeader(cluster, null, false), IllegalStateException.class); - cluster.shutdown(); + } + + static class SleepCode implements CodeInjectionForTesting.Code { + private final long sleepMs; + + SleepCode(long sleepMs) { + this.sleepMs = sleepMs; + } + + @Override + public boolean execute(Object localId, Object remoteId, Object... args) { + try { + LOG.info("{}: Simulate RaftServer startup blocking", localId); + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + } + } + + @Test + public void testWaitServerReady() throws Exception { + final int sleepMs = 1000 + ThreadLocalRandom.current().nextInt(1000); + LOG.info("Running testWaitServerReady, sleep = {}ms", sleepMs); + CodeInjectionForTesting.put(RaftServerImpl.START_COMPLETE, new SleepCode(sleepMs)); + final Timestamp startTime = Timestamp.currentTime(); + runWithNewCluster(1, c -> runTestWaitServerReady(c, sleepMs, startTime)); + } + + void runTestWaitServerReady(MiniRaftCluster cluster, int sleepMs, Timestamp startTime) throws Exception { + LOG.info("Cluster started at {}ms", startTime.elapsedTimeMs()); + final RaftGroupId groupId = cluster.getGroupId(); + final RaftServerImpl server = (RaftServerImpl) cluster.getServers().iterator().next().getDivision(groupId); + final boolean isRunning = server.isRunning(); + LOG.info("{} isRunning at {}ms? {}", server.getId(), startTime.elapsedTimeMs(), isRunning); + + // Leader will be elected if the server is ready + Assertions.assertNotNull(waitForLeader(cluster), "No leader is elected."); + final long elapsedMs = startTime.elapsedTimeMs(); + // allow a small difference to tolerate system timer inaccuracy + Assertions.assertTrue(elapsedMs > sleepMs - 10, () -> "elapseMs = " + elapsedMs + " but sleepMs = " + sleepMs); + CodeInjectionForTesting.remove(RaftServerImpl.START_COMPLETE); + } + + @Test + public void testAddServerForWaitReady() throws Exception { + LOG.info("Running testAddServerForWaitReady"); + // normal startup cluster with 3 server + runWithNewCluster(3, this::runTestAddServerForWaitReady); + } + + void runTestAddServerForWaitReady(MiniRaftCluster cluster) throws Exception { + RaftTestUtil.waitForLeader(cluster); + try (RaftClient client = cluster.createClient()) { + for (int i = 0; i < 10; ++i) { + RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("message_" + i)); + Assertions.assertTrue(reply.isSuccess()); + } + // add 3 new servers and wait longer time + CodeInjectionForTesting.put(RaftServerImpl.START_COMPLETE, new SleepCode(2000)); + MiniRaftCluster.PeerChanges peerChanges = cluster.addNewPeers(2, true, false); + LOG.info("add new 3 servers"); + LOG.info(cluster.printServers()); + RaftClientReply reply = client.admin().setConfiguration(SetConfigurationRequest.Arguments.newBuilder() + .setServersInNewConf(peerChanges.newPeers) + .setMode(SetConfigurationRequest.Mode.ADD).build()); + assertTrue(reply.isSuccess()); + for (RaftServer server : cluster.getServers()) { + RaftServerProxy proxy = (RaftServerProxy) server; + proxy.getImpls().forEach(s -> { + Assertions.assertTrue(s.isRunning()); + }); + } + } + CodeInjectionForTesting.remove(RaftServerImpl.START_COMPLETE); } @Test public void testChangeLeader() throws Exception { - SegmentedRaftLogTestUtils.setRaftLogWorkerLogLevel(Level.TRACE); LOG.info("Running testChangeLeader"); - final MiniRaftCluster cluster = newCluster(3); - cluster.start(); + runWithNewCluster(3, this::runTestChangeLeader); + } + void runTestChangeLeader(MiniRaftCluster cluster) throws Exception { RaftPeerId leader = RaftTestUtil.waitForLeader(cluster).getId(); for(int i = 0; i < 10; i++) { leader = RaftTestUtil.changeLeader(cluster, leader, IllegalStateException::new); - ExitUtils.assertNotTerminated(); } - SegmentedRaftLogTestUtils.setRaftLogWorkerLogLevel(Level.INFO); - cluster.shutdown(); } @Test @@ -122,12 +197,12 @@ void runTestLostMajorityHeartbeats(CLUSTER cluster) throws Exception { final TimeDuration maxTimeout = RaftServerConfigKeys.Rpc.timeoutMax(getProperties()); final RaftServer.Division leader = waitForLeader(cluster); try { - isolate(cluster, leader.getId()); + RaftTestUtil.isolate(cluster, leader.getId()); maxTimeout.sleep(); maxTimeout.sleep(); RaftServerTestUtil.assertLostMajorityHeartbeatsRecently(leader); } finally { - deIsolate(cluster, leader.getId()); + RaftTestUtil.deIsolate(cluster, leader.getId()); } } @@ -138,18 +213,18 @@ public void testLeaderNotCountListenerForMajority() throws Exception { void runTestLeaderNotCountListenerForMajority(CLUSTER cluster) throws Exception { final RaftServer.Division leader = waitForLeader(cluster); - Assert.assertEquals(2, ((RaftConfigurationImpl)cluster.getLeader().getRaftConf()).getMajorityCount()); + Assertions.assertEquals(2, ((RaftConfigurationImpl)cluster.getLeader().getRaftConf()).getMajorityCount()); try (RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); List listeners = cluster.getListeners() .stream().map(RaftServer.Division::getPeer).collect(Collectors.toList()); - Assert.assertEquals(2, listeners.size()); + Assertions.assertEquals(2, listeners.size()); RaftClientReply reply = client.admin().setConfiguration(cluster.getPeers()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); Collection peer = leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER); - Assert.assertEquals(0, peer.size()); + Assertions.assertEquals(0, peer.size()); } - Assert.assertEquals(3, ((RaftConfigurationImpl)cluster.getLeader().getRaftConf()).getMajorityCount()); + Assertions.assertEquals(3, ((RaftConfigurationImpl)cluster.getLeader().getRaftConf()).getMajorityCount()); } @Test @@ -164,12 +239,12 @@ void runTestListenerNotStartLeaderElection(CLUSTER cluster) throws Exception { final RaftServer.Division listener = cluster.getListeners().get(0); final RaftPeerId listenerId = listener.getId(); try { - isolate(cluster, listenerId); + RaftTestUtil.isolate(cluster, listenerId); maxTimeout.sleep(); maxTimeout.sleep(); - Assert.assertEquals(RaftProtos.RaftPeerRole.LISTENER, listener.getInfo().getCurrentRole()); + Assertions.assertEquals(RaftProtos.RaftPeerRole.LISTENER, listener.getInfo().getCurrentRole()); } finally { - deIsolate(cluster, listener.getId()); + RaftTestUtil.deIsolate(cluster, listener.getId()); } } @@ -183,21 +258,19 @@ public void testTransferLeader() throws Exception { client.io().send(new RaftTestUtil.SimpleMessage("message")); List followers = cluster.getFollowers(); - Assert.assertEquals(2, followers.size()); + Assertions.assertEquals(2, followers.size()); RaftServer.Division newLeader = followers.get(0); RaftClientReply reply = client.admin().transferLeadership(newLeader.getId(), 20000); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); final RaftServer.Division currLeader = waitForLeader(cluster); - Assert.assertEquals(newLeader.getId(), currLeader.getId()); + Assertions.assertEquals(newLeader.getId(), currLeader.getId()); reply = client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertEquals(newLeader.getId().toString(), reply.getReplierId()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertEquals(newLeader.getId().toString(), reply.getReplierId()); + Assertions.assertTrue(reply.isSuccess()); } - - cluster.shutdown(); } } @@ -211,27 +284,25 @@ public void testYieldLeaderToHigherPriority() throws Exception { client.io().send(new RaftTestUtil.SimpleMessage("message")); List followers = cluster.getFollowers(); - Assert.assertEquals(2, followers.size()); + Assertions.assertEquals(2, followers.size()); RaftServer.Division newLeader = followers.get(0); List peers = cluster.getPeers(); List peersWithNewPriority = getPeersWithPriority(peers, newLeader.getPeer()); RaftClientReply reply = client.admin().setConfiguration(peersWithNewPriority.toArray(new RaftPeer[0])); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); // Wait the old leader to step down. // TODO: make it more deterministic. TimeDuration.valueOf(1, TimeUnit.SECONDS).sleep(); final RaftServer.Division currLeader = waitForLeader(cluster); - Assert.assertEquals(newLeader.getId(), currLeader.getId()); + Assertions.assertEquals(newLeader.getId(), currLeader.getId()); reply = client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertEquals(newLeader.getId().toString(), reply.getReplierId()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertEquals(newLeader.getId().toString(), reply.getReplierId()); + Assertions.assertTrue(reply.isSuccess()); } - - cluster.shutdown(); } } @@ -243,11 +314,11 @@ public void testTransferLeaderTimeout() throws Exception { final RaftServer.Division leader = waitForLeader(cluster); try (RaftClient client = cluster.createClient(leader.getId())) { List followers = cluster.getFollowers(); - Assert.assertEquals(followers.size(), 2); + Assertions.assertEquals(followers.size(), 2); RaftServer.Division newLeader = followers.get(0); // isolate new leader, so that transfer leadership will timeout - isolate(cluster, newLeader.getId()); + RaftTestUtil.isolate(cluster, newLeader.getId()); List peers = cluster.getPeers(); @@ -259,9 +330,9 @@ public void testTransferLeaderTimeout() throws Exception { client.admin().transferLeadership(newLeader.getId(), timeoutMs); } catch (TransferLeadershipException e) { long cost = System.currentTimeMillis() - start; - Assert.assertTrue(cost > timeoutMs); - Assert.assertTrue(e.getMessage().contains("Failed to transfer leadership to")); - Assert.assertTrue(e.getMessage().contains(TransferLeadership.Result.Type.TIMED_OUT.toString())); + Assertions.assertTrue(cost > timeoutMs); + Assertions.assertTrue(e.getMessage().contains("Failed to transfer leadership to")); + Assertions.assertTrue(e.getMessage().contains(TransferLeadership.Result.Type.TIMED_OUT.toString())); } return true; @@ -275,22 +346,20 @@ public void testTransferLeaderTimeout() throws Exception { try { client.io().send(new RaftTestUtil.SimpleMessage("message")); } catch (LeaderSteppingDownException e) { - Assert.assertTrue(e.getMessage().contains("is stepping down")); + Assertions.assertTrue(e.getMessage().contains("is stepping down")); } return null; }, 5, TimeDuration.ONE_SECOND, "check leader steppingDown", RaftServer.LOG); - Assert.assertTrue(transferTimeoutFuture.get()); + Assertions.assertTrue(transferTimeoutFuture.get()); // after transfer timeout, leader should accept request RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertEquals(leader.getId().toString(), reply.getReplierId()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertEquals(leader.getId().toString(), reply.getReplierId()); + Assertions.assertTrue(reply.isSuccess()); - deIsolate(cluster, newLeader.getId()); + RaftTestUtil.deIsolate(cluster, newLeader.getId()); } - - cluster.shutdown(); } } @@ -312,30 +381,34 @@ public void testEnforceLeader() throws Exception { } } - static void enforceLeader(MiniRaftCluster cluster, final String newLeader, Logger LOG) throws InterruptedException { - LOG.info(cluster.printServers()); + static void enforceLeader(MiniRaftCluster cluster, final String newLeader, Logger log) throws InterruptedException { + log.info(cluster.printServers()); for(int i = 0; !cluster.tryEnforceLeader(newLeader) && i < 10; i++) { final RaftServer.Division currLeader = cluster.getLeader(); - LOG.info("try enforcing leader to " + newLeader + " but " + + log.info("try enforcing leader to " + newLeader + " but " + (currLeader == null ? "no leader for round " + i : "new leader is " + currLeader.getId())); TimeDuration.ONE_SECOND.sleep(); } - LOG.info(cluster.printServers()); + log.info(cluster.printServers()); final RaftServer.Division leader = cluster.getLeader(); - Assert.assertEquals(newLeader, leader.getId().toString()); + Assertions.assertEquals(newLeader, leader.getId().toString()); } @Test public void testLateServerStart() throws Exception { - final int numServer = 3; LOG.info("Running testLateServerStart"); - final MiniRaftCluster cluster = newCluster(numServer); + try (final MiniRaftCluster cluster = newCluster(3)) { + runTestLateServerStart(cluster); + } + } + + void runTestLateServerStart(MiniRaftCluster cluster) throws Exception { cluster.initServers(); // start all except one servers final Iterator i = cluster.getServers().iterator(); - for(int j = 1; j < numServer; j++) { + for(int j = 1; j < cluster.getNumServers(); j++) { i.next().start(); } @@ -352,7 +425,7 @@ public void testLateServerStart() throws Exception { .orElseThrow(() -> new IllegalStateException("No leader yet")), 10, ONE_SECOND, "getLeaderId", LOG); LOG.info(cluster.printServers()); - Assert.assertEquals(leader.getId(), lastServerLeaderId); + Assertions.assertEquals(leader.getId(), lastServerLeaderId); } protected void testDisconnectLeader() throws Exception { @@ -363,32 +436,16 @@ protected void testDisconnectLeader() throws Exception { try (RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); Thread.sleep(1000); - isolate(cluster, leader.getId()); + RaftTestUtil.isolate(cluster, leader.getId()); RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertNotEquals(reply.getReplierId(), leader.getId().toString()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertNotEquals(reply.getReplierId(), leader.getId().toString()); + Assertions.assertTrue(reply.isSuccess()); } finally { - deIsolate(cluster, leader.getId()); + RaftTestUtil.deIsolate(cluster, leader.getId()); } - - cluster.shutdown(); } } - private void isolate(MiniRaftCluster cluster, RaftPeerId id) { - try { - BlockRequestHandlingInjection.getInstance().blockReplier(id.toString()); - cluster.setBlockRequestsFrom(id.toString(), true); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void deIsolate(MiniRaftCluster cluster, RaftPeerId id) { - BlockRequestHandlingInjection.getInstance().unblockReplier(id.toString()); - cluster.setBlockRequestsFrom(id.toString(), false); - } - @Test public void testAddListener() throws Exception { try (final MiniRaftCluster cluster = newCluster(3)) { @@ -397,17 +454,16 @@ public void testAddListener() throws Exception { try (RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); List servers = cluster.getPeers(); - Assert.assertEquals(servers.size(), 3); + Assertions.assertEquals(servers.size(), 3); MiniRaftCluster.PeerChanges changes = cluster.addNewPeers(1, true, false, RaftProtos.RaftPeerRole.LISTENER); RaftClientReply reply = client.admin().setConfiguration(servers, Arrays.asList(changes.newPeers)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); Collection listener = leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER); - Assert.assertEquals(1, listener.size()); - Assert.assertEquals(changes.newPeers[0].getId(), new ArrayList<>(listener).get(0).getId()); + Assertions.assertEquals(1, listener.size()); + Assertions.assertEquals(changes.newPeers[0].getId(), new ArrayList<>(listener).get(0).getId()); } - cluster.shutdown(); } } @@ -419,21 +475,20 @@ public void testAddFollowerWhenExistsListener() throws Exception { try (RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); List servers = cluster.getPeers(); - Assert.assertEquals(4, servers.size()); + Assertions.assertEquals(4, servers.size()); List listener = new ArrayList<>( leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER)); - Assert.assertEquals(1, listener.size()); + Assertions.assertEquals(1, listener.size()); MiniRaftCluster.PeerChanges changes = cluster.addNewPeers(1, true, false); ArrayList newPeers = new ArrayList<>(Arrays.asList(changes.newPeers)); newPeers.addAll(leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.FOLLOWER)); RaftClientReply reply = client.admin().setConfiguration(newPeers, listener); - Assert.assertTrue(reply.isSuccess()); - Assert.assertEquals(4, + Assertions.assertTrue(reply.isSuccess()); + Assertions.assertEquals(4, leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.FOLLOWER).size()); - Assert.assertEquals(1, + Assertions.assertEquals(1, leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER).size()); } - cluster.shutdown(); } } @@ -444,15 +499,14 @@ public void testRemoveListener() throws Exception { final RaftServer.Division leader = waitForLeader(cluster); try (RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertEquals(1, cluster.getListeners().size()); + Assertions.assertEquals(1, cluster.getListeners().size()); List servers = cluster.getFollowers().stream().map(RaftServer.Division::getPeer).collect( Collectors.toList()); servers.add(leader.getPeer()); RaftClientReply reply = client.admin().setConfiguration(servers); - Assert.assertTrue(reply.isSuccess()); - Assert.assertEquals(0, leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER).size()); + Assertions.assertTrue(reply.isSuccess()); + Assertions.assertEquals(0, leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER).size()); } - cluster.shutdown(); } } @@ -465,17 +519,16 @@ public void testChangeFollowerToListener() throws Exception { client.io().send(new RaftTestUtil.SimpleMessage("message")); List followers = cluster.getFollowers().stream().map( RaftServer.Division::getPeer).collect(Collectors.toList()); - Assert.assertEquals(2, followers.size()); + Assertions.assertEquals(2, followers.size()); List listeners = new ArrayList<>(); listeners.add(followers.get(1)); followers.remove(1); RaftClientReply reply = client.admin().setConfiguration(followers, listeners); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); Collection peer = leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER); - Assert.assertEquals(1, peer.size()); - Assert.assertEquals(listeners.get(0).getId(), new ArrayList<>(peer).get(0).getId()); + Assertions.assertEquals(1, peer.size()); + Assertions.assertEquals(listeners.get(0).getId(), new ArrayList<>(peer).get(0).getId()); } - cluster.shutdown(); } } @@ -488,21 +541,22 @@ public void testChangeListenerToFollower() throws Exception { client.io().send(new RaftTestUtil.SimpleMessage("message")); List listeners = cluster.getListeners() .stream().map(RaftServer.Division::getPeer).collect(Collectors.toList()); - Assert.assertEquals(listeners.size(), 1); + Assertions.assertEquals(listeners.size(), 1); RaftClientReply reply = client.admin().setConfiguration(cluster.getPeers()); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); Collection peer = leader.getRaftConf().getAllPeers(RaftProtos.RaftPeerRole.LISTENER); - Assert.assertEquals(0, peer.size()); + Assertions.assertEquals(0, peer.size()); } - cluster.shutdown(); } } @Test - public void testLeaderElectionMetrics() throws IOException, InterruptedException { + public void testLeaderElectionMetrics() throws Exception { + runWithNewCluster(3, this::runTestLeaderElectionMetrics); + } + + void runTestLeaderElectionMetrics(MiniRaftCluster cluster) throws Exception { Timestamp timestamp = Timestamp.currentTime(); - final MiniRaftCluster cluster = newCluster(3); - cluster.start(); final RaftServer.Division leaderServer = waitForLeader(cluster); final RatisMetricRegistryImpl ratisMetricRegistry = (RatisMetricRegistryImpl) @@ -515,7 +569,8 @@ public void testLeaderElectionMetrics() throws IOException, InterruptedException long numLeaderElectionTimeout = ratisMetricRegistry.counter(LEADER_ELECTION_TIMEOUT_COUNT_METRIC).getCount(); assertTrue(numLeaderElectionTimeout > 0); - final DefaultTimekeeperImpl timekeeper = (DefaultTimekeeperImpl) ratisMetricRegistry.timer(LEADER_ELECTION_TIME_TAKEN); + final DefaultTimekeeperImpl timekeeper = + (DefaultTimekeeperImpl) ratisMetricRegistry.timer(LEADER_ELECTION_TIME_TAKEN); final Timer timer = timekeeper.getTimer(); double meanTimeNs = timer.getSnapshot().getMean(); long elapsedNs = timestamp.elapsedTime().toLong(TimeUnit.NANOSECONDS); @@ -568,15 +623,15 @@ public void testPreVote() { assertEquals(followers.size(), 2); RaftServer.Division follower = followers.get(0); - isolate(cluster, follower.getId()); + RaftTestUtil.isolate(cluster, follower.getId()); // send message so that the isolated follower's log lag the others RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); final long savedTerm = leader.getInfo().getCurrentTerm(); LOG.info("Wait follower {} timeout and trigger pre-vote", follower.getId()); Thread.sleep(2000); - deIsolate(cluster, follower.getId()); + RaftTestUtil.deIsolate(cluster, follower.getId()); Thread.sleep(2000); // with pre-vote leader will not step down RaftServer.Division newleader = waitForLeader(cluster); @@ -586,10 +641,8 @@ public void testPreVote() { assertEquals(savedTerm, leader.getInfo().getCurrentTerm()); reply = client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } - - cluster.shutdown(); } catch (Exception e) { fail(e.getMessage()); } @@ -604,9 +657,10 @@ void runTestListenerRejectRequestVote(CLUSTER cluster) throws IOException, Inter final TermIndex lastEntry = leader.getRaftLog().getLastEntryTermIndex(); RaftServer.Division listener = cluster.getListeners().get(0); final RaftProtos.RequestVoteRequestProto r = ServerProtoUtils.toRequestVoteRequestProto( - leader.getMemberId(), listener.getId(), leader.getRaftLog().getLastEntryTermIndex().getTerm() + 1, lastEntry, true); + leader.getMemberId(), listener.getId(), + leader.getRaftLog().getLastEntryTermIndex().getTerm() + 1, lastEntry, true); RaftProtos.RequestVoteReplyProto listenerReply = listener.getRaftServer().requestVote(r); - Assert.assertFalse(listenerReply.getServerReply().getSuccess()); + Assertions.assertFalse(listenerReply.getServerReply().getSuccess()); } @@ -620,23 +674,23 @@ void runTestPauseResumeLeaderElection(CLUSTER cluster) throws IOException, Inter final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster); final RaftPeerId leaderId = leader.getId(); final List followers = cluster.getFollowers(); - Assert.assertTrue(followers.size() >= 1); + Assertions.assertTrue(followers.size() >= 1); final RaftServerImpl f1 = (RaftServerImpl)followers.get(0); try (final RaftClient client = cluster.createClient()) { pauseLeaderReply = client.getLeaderElectionManagementApi(f1.getId()).pause(); - Assert.assertTrue(pauseLeaderReply.isSuccess()); + Assertions.assertTrue(pauseLeaderReply.isSuccess()); client.io().send(new RaftTestUtil.SimpleMessage("message")); RaftServer.Division newLeader = followers.get(0); List peers = cluster.getPeers(); List peersWithNewPriority = getPeersWithPriority(peers, newLeader.getPeer()); RaftClientReply reply = client.admin().setConfiguration(peersWithNewPriority.toArray(new RaftPeer[0])); - Assert.assertTrue(reply.isSuccess()); - JavaUtils.attempt(() -> Assert.assertEquals(leaderId, leader.getId()), + Assertions.assertTrue(reply.isSuccess()); + JavaUtils.attempt(() -> Assertions.assertEquals(leaderId, leader.getId()), 20, HUNDRED_MILLIS, "check leader id", LOG); final RaftClientReply resumeLeaderReply = client.getLeaderElectionManagementApi(f1.getId()).resume(); - Assert.assertTrue(resumeLeaderReply.isSuccess()); - JavaUtils.attempt(() -> Assert.assertEquals(f1.getId(), cluster.getLeader().getId()), + Assertions.assertTrue(resumeLeaderReply.isSuccess()); + JavaUtils.attempt(() -> Assertions.assertEquals(f1.getId(), cluster.getLeader().getId()), 20, HUNDRED_MILLIS, "check new leader", LOG); } } @@ -662,21 +716,22 @@ void runTestLeaderLease(CLUSTER cluster, long leaseTimeoutMs) throws Exception { try (final RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertTrue(leader.getInfo().isLeader()); - Assert.assertTrue(leader.getInfo().isLeaderReady()); + Assertions.assertTrue(leader.getInfo().isLeader()); + Assertions.assertTrue(leader.getInfo().isLeaderReady()); RaftServerTestUtil.assertLeaderLease(leader, true); - isolate(cluster, leader.getId()); + RaftTestUtil.isolate(cluster, leader.getId()); Thread.sleep(leaseTimeoutMs); - Assert.assertTrue(leader.getInfo().isLeader()); - Assert.assertTrue(leader.getInfo().isLeaderReady()); + Assertions.assertTrue(leader.getInfo().isLeader()); + Assertions.assertTrue(leader.getInfo().isLeaderReady()); RaftServerTestUtil.assertLeaderLease(leader, false); } finally { - deIsolate(cluster, leader.getId()); + RaftTestUtil.deIsolate(cluster, leader.getId()); } } + @Flaky("RATIS-2108") @Test public void testLeaderLeaseDuringReconfiguration() throws Exception { // use a strict lease @@ -690,8 +745,8 @@ void runTestLeaderLeaseDuringReconfiguration(CLUSTER cluster, long leaseTimeoutM try (final RaftClient client = cluster.createClient(leader.getId())) { client.io().send(new RaftTestUtil.SimpleMessage("message")); - Assert.assertTrue(leader.getInfo().isLeader()); - Assert.assertTrue(leader.getInfo().isLeaderReady()); + Assertions.assertTrue(leader.getInfo().isLeader()); + Assertions.assertTrue(leader.getInfo().isLeaderReady()); RaftServerTestUtil.assertLeaderLease(leader, true); final List followers = cluster.getFollowers(); @@ -712,8 +767,8 @@ void runTestLeaderLeaseDuringReconfiguration(CLUSTER cluster, long leaseTimeoutM Thread.sleep(leaseTimeoutMs); - Assert.assertTrue(leader.getInfo().isLeader()); - Assert.assertTrue(leader.getInfo().isLeaderReady()); + Assertions.assertTrue(leader.getInfo().isLeader()); + Assertions.assertTrue(leader.getInfo().isLeaderReady()); RaftServerTestUtil.assertLeaderLease(leader, false); } finally { diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/LogMetadataTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/LogMetadataTests.java new file mode 100644 index 0000000000..a834e27144 --- /dev/null +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/LogMetadataTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ratis.server.impl; + +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.ratis.BaseTest; +import org.apache.ratis.RaftTestUtil; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.server.RaftServer; +import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.raftlog.RaftLog; +import org.junit.Assert; +import org.junit.Test; + +public abstract class LogMetadataTests + extends BaseTest + implements MiniRaftCluster.Factory.Get { + + @Test + public void testLogMetadataEnabled() throws Exception { + testLogMetadataBasicTest(true, x -> x > RaftLog.INVALID_LOG_INDEX); + } + + @Test + public void testLogMetadataDisabled() throws Exception { + testLogMetadataBasicTest(false, x -> x == RaftLog.INVALID_LOG_INDEX); + } + + public void testLogMetadataBasicTest(boolean logMetadata, Predicate checker) + throws Exception { + final RaftProperties prop = getProperties(); + RaftServerConfigKeys.Log.setLogMetadataEnabled(prop, logMetadata); + + final MiniRaftCluster cluster = newCluster(3); + try { + cluster.start(); + RaftTestUtil.waitForLeader(cluster); + final RaftServer.Division leader = cluster.getLeader(); + RaftPeerId leaderId = leader.getId(); + + cluster.getLeaderAndSendFirstMessage(true); + + // kill majority servers + for (RaftPeerId id : cluster.getGroup().getPeers().stream().map(RaftPeer::getId) + .filter(x -> !x.equals(leaderId)).collect(Collectors.toList())) { + cluster.killServer(id); + } + + // only restart one server + cluster.restartServer(leaderId, false); + + long commitIndex = cluster.getServer(leaderId).getDivision(cluster.getGroupId()).getRaftLog() + .getLastCommittedIndex(); + + Assert.assertTrue(checker.test(commitIndex)); + } finally { + cluster.shutdown(); + } + } +} diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/MiniRaftCluster.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/MiniRaftCluster.java index f6dd6121cb..08ff17a4cd 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/MiniRaftCluster.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/MiniRaftCluster.java @@ -50,6 +50,7 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.NetUtils; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedLeakDetector; import org.apache.ratis.util.ReflectionUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.function.CheckedConsumer; @@ -92,14 +93,14 @@ public abstract class MiniRaftCluster implements Closeable { TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); static final AtomicInteger THREAD_COUNT = new AtomicInteger(0); - public static abstract class Factory { + public abstract static class Factory { public interface Get { - Supplier properties = JavaUtils.memoize(RaftProperties::new); + Supplier PROPERTIES = JavaUtils.memoize(RaftProperties::new); Factory getFactory(); default RaftProperties getProperties() { - return properties.get(); + return PROPERTIES.get(); } default RaftProperties setStateMachine(Class stateMachineClass) { @@ -120,30 +121,42 @@ default void runWithNewCluster(int numServers, CheckedConsumer testCase) throws Exception { + default void runWithNewCluster(int numServers, boolean startCluster, CheckedConsumer testCase) + throws Exception { runWithNewCluster(numServers, 0, startCluster, testCase); } - default void runWithNewCluster(int numServers, int numListeners, CheckedConsumer testCase) throws Exception { + default void runWithNewCluster(int numServers, int numListeners, CheckedConsumer testCase) + throws Exception { runWithNewCluster(numServers, numListeners, true, testCase); } - default void runWithNewCluster(int numServers, int numListeners, boolean startCluster, CheckedConsumer testCase) - throws Exception { + default void runWithNewCluster(int numServers, int numListeners, boolean startCluster, + CheckedConsumer testCase) throws Exception { final StackTraceElement caller = JavaUtils.getCallerStackTraceElement(); - LOG.info("Running " + caller.getMethodName()); + LOG.info("Running {}", caller.getMethodName()); final CLUSTER cluster = newCluster(numServers, numListeners); + Throwable failed = null; try { if (startCluster) { cluster.start(); } testCase.accept(cluster); - } catch(Exception t) { + } catch(Throwable t) { LOG.info(cluster.printServers()); - LOG.error("Failed " + caller, t); + LOG.error("Failed {}", caller, t); + failed = t; throw t; } finally { - cluster.shutdown(); + try { + cluster.shutdown(); + } catch (Exception e) { + if (failed == null) { + throw e; + } else { + failed.addSuppressed(e); + } + } } } @@ -151,9 +164,10 @@ default void runWithSameCluster(int numServers, CheckedConsumer testCase) throws Exception { + default void runWithSameCluster(int numServers, int numListeners, CheckedConsumer testCase) + throws Exception { final StackTraceElement caller = JavaUtils.getCallerStackTraceElement(); - LOG.info("Running " + caller.getMethodName()); + LOG.info("Running {}", caller.getMethodName()); CLUSTER cluster = null; try { cluster = getFactory().reuseCluster(numServers, numListeners, getProperties()); @@ -162,7 +176,7 @@ default void runWithSameCluster(int numServers, int numListeners, CheckedConsume if (cluster != null) { LOG.info(cluster.printServers()); } - LOG.error("Failed " + caller, t); + LOG.error("Failed {}", caller, t); throw t; } } @@ -198,7 +212,7 @@ public CLUSTER newCluster(int numServer, int numListeners, RaftProperties prop) } } - public static abstract class RpcBase extends MiniRaftCluster { + public abstract static class RpcBase extends MiniRaftCluster { public RpcBase(String[] ids, String[] listenerIds, RaftProperties properties, Parameters parameters) { super(ids, listenerIds, properties, parameters); } @@ -314,22 +328,22 @@ public RaftProperties getProperties() { } public MiniRaftCluster initServers() { - LOG.info("servers = " + servers); + LOG.info("servers = {}", servers); if (servers.isEmpty()) { putNewServers(CollectionUtils.as(group.getPeers(), RaftPeer::getId), true, group); } return this; } - public RaftServerProxy putNewServer(RaftPeerId id, RaftGroup group, boolean format) { - final RaftServerProxy s = newRaftServer(id, group, format); + public RaftServerProxy putNewServer(RaftPeerId id, RaftGroup raftGroup, boolean format) { + final RaftServerProxy s = newRaftServer(id, raftGroup, format); peers.put(s.getId(), s.getPeer()); Preconditions.assertTrue(servers.put(id, s) == null); return s; } - private Collection putNewServers(Iterable peers, boolean format, RaftGroup raftGroup) { - return StreamSupport.stream(peers.spliterator(), false) + private Collection putNewServers(Iterable peerIds, boolean format, RaftGroup raftGroup) { + return StreamSupport.stream(peerIds.spliterator(), false) .map(id -> putNewServer(id, raftGroup, format)) .collect(Collectors.toList()); } @@ -345,7 +359,7 @@ public void start() throws IOException { startServers(servers.values()); this.timer.updateAndGet(t -> t != null? t - : JavaUtils.runRepeatedly(() -> LOG.info("TIMED-PRINT: " + printServers()), 10, 10, TimeUnit.SECONDS)); + : JavaUtils.runRepeatedly(() -> LOG.info("TIMED-PRINT: {}.", printServers()), 10, 10, TimeUnit.SECONDS)); } /** @@ -355,13 +369,14 @@ public RaftServer.Division restartServer(RaftPeerId serverId, boolean format) th return restartServer(serverId, group, format); } - public RaftServer.Division restartServer(RaftPeerId serverId, RaftGroup group, boolean format) throws IOException { + public RaftServer.Division restartServer(RaftPeerId serverId, RaftGroup raftGroup, boolean format) + throws IOException { killServer(serverId); servers.remove(serverId); - final RaftServer proxy = putNewServer(serverId, group, format); + final RaftServer proxy = putNewServer(serverId, raftGroup, format); proxy.start(); - return group == null? null: proxy.getDivision(group.getGroupId()); + return raftGroup == null? null: proxy.getDivision(raftGroup.getGroupId()); } public void restart(boolean format) throws IOException { @@ -377,8 +392,8 @@ public TimeDuration getTimeoutMax() { return RaftServerConfigKeys.Rpc.timeoutMax(properties); } - private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) { - LOG.info("newRaftServer: {}, {}, format? {}", id, group, format); + private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup raftGroup, boolean format) { + LOG.info("newRaftServer: {}, {}, format? {}", id, raftGroup, format); try { final File dir = getStorageDir(id); if (format) { @@ -387,27 +402,27 @@ private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean fo } final RaftProperties prop = new RaftProperties(properties); RaftServerConfigKeys.setStorageDir(prop, Collections.singletonList(dir)); - return ServerImplUtils.newRaftServer(id, group, + return ServerImplUtils.newRaftServer(id, raftGroup, format? RaftStorage.StartupOption.FORMAT: RaftStorage.StartupOption.RECOVER, - getStateMachineRegistry(prop), null, prop, setPropertiesAndInitParameters(id, group, prop)); + getStateMachineRegistry(prop), null, prop, setPropertiesAndInitParameters(id, raftGroup, prop)); } catch (IOException e) { throw new RuntimeException(e); } } protected abstract Parameters setPropertiesAndInitParameters( - RaftPeerId id, RaftGroup group, RaftProperties properties); + RaftPeerId id, RaftGroup raftGroup, RaftProperties raftProperties); public void setStateMachineRegistry(StateMachine.Registry stateMachineRegistry) { this.stateMachineRegistry = stateMachineRegistry; } - StateMachine.Registry getStateMachineRegistry(RaftProperties properties) { + StateMachine.Registry getStateMachineRegistry(RaftProperties raftProperties) { if (stateMachineRegistry != null) { return stateMachineRegistry; } - final Class smClass = properties.getClass( + final Class smClass = raftProperties.getClass( STATEMACHINE_CLASS_KEY, null, StateMachine.class); if (smClass == null) { return STATEMACHINE_REGISTRY_DEFAULT; @@ -423,7 +438,7 @@ StateMachine.Registry getStateMachineRegistry(RaftProperties properties) { try { final Class[] argClasses = {RaftProperties.class}; - return ReflectionUtils.newInstance(smClass, argClasses, properties); + return ReflectionUtils.newInstance(smClass, argClasses, raftProperties); } catch(RuntimeException e) { exception.addSuppressed(e); } @@ -495,8 +510,8 @@ public PeerChanges addNewPeers(String[] ids, boolean startNewPeer, return new PeerChanges(p, np, RaftPeer.emptyArray()); } - void startServers(Iterable servers) throws IOException { - for(RaftServer s : servers) { + void startServers(Iterable raftServers) throws IOException { + for(RaftServer s : raftServers) { s.start(); peers.put(s.getId(), s.getPeer()); } @@ -507,12 +522,12 @@ void startServers(Iterable servers) throws IOException { */ public PeerChanges removePeers(int number, boolean removeLeader, Collection excluded) throws InterruptedException { - Collection peers = new ArrayList<>(group.getPeers()); + Collection raftPeers = new ArrayList<>(group.getPeers()); List removedPeers = new ArrayList<>(number); if (removeLeader) { final RaftPeer leader = RaftTestUtil.waitForLeader(this).getPeer(); Preconditions.assertTrue(!excluded.contains(leader)); - peers.remove(leader); + raftPeers.remove(leader); removedPeers.add(leader); } final List followers = getFollowers(); @@ -520,18 +535,18 @@ public PeerChanges removePeers(int number, boolean removeLeader, removed < (removeLeader ? number - 1 : number); i++) { RaftPeer toRemove = followers.get(i).getPeer(); if (!excluded.contains(toRemove)) { - peers.remove(toRemove); + raftPeers.remove(toRemove); removedPeers.add(toRemove); removed++; } } - final RaftPeer[] p = peers.toArray(RaftPeer.emptyArray()); + final RaftPeer[] p = raftPeers.toArray(RaftPeer.emptyArray()); group = RaftGroup.valueOf(group.getGroupId(), p); return new PeerChanges(p, RaftPeer.emptyArray(), removedPeers.toArray(RaftPeer.emptyArray())); } public void killServer(RaftPeerId id) { - LOG.info("killServer " + id); + LOG.info("killServer {}", id); servers.get(id).close(); } @@ -727,13 +742,13 @@ public List getPeers() { return toRaftPeers(getServers()); } - RaftPeer getPeer(RaftPeerId id, RaftGroup group) { + RaftPeer getPeer(RaftPeerId id, RaftGroup raftGroup) { RaftPeer p = peers.get(id); if (p != null) { return p; } - if (group != null) { - p = group.getPeer(id); + if (raftGroup != null) { + p = raftGroup.getPeer(id); } if (p == null) { p = Optional.ofNullable(servers.get(id)).map(RaftServerProxy::getPeer).orElse(null); @@ -768,21 +783,21 @@ public RaftClient createClient(RaftPeerId leaderId, RetryPolicy retryPolicy) { return createClient(leaderId, group, retryPolicy); } - public RaftClient createClient(RaftPeerId leaderId, RaftGroup group) { - return createClient(leaderId, group, getDefaultRetryPolicy()); + public RaftClient createClient(RaftPeerId leaderId, RaftGroup raftGroup) { + return createClient(leaderId, raftGroup, getDefaultRetryPolicy()); } public RaftClient createClient(RaftPeer primaryServer) { return createClient(null, group, getDefaultRetryPolicy(), primaryServer); } - public RaftClient createClient(RaftPeerId leaderId, RaftGroup group, RetryPolicy retryPolicy) { - return createClient(leaderId, group, retryPolicy, null); + public RaftClient createClient(RaftPeerId leaderId, RaftGroup raftGroup, RetryPolicy retryPolicy) { + return createClient(leaderId, raftGroup, retryPolicy, null); } - public RaftClient createClient(RaftPeerId leaderId, RaftGroup group, RetryPolicy retryPolicy, RaftPeer primaryServer) { + public RaftClient createClient(RaftPeerId leaderId, RaftGroup raftGroup, RetryPolicy retryPolicy, RaftPeer primaryServer) { RaftClient.Builder builder = RaftClient.newBuilder() - .setRaftGroup(group) + .setRaftGroup(raftGroup) .setLeaderId(leaderId) .setProperties(properties) .setParameters(parameters) @@ -810,15 +825,15 @@ public RaftClientRequest newRaftClientRequest( public SetConfigurationRequest newSetConfigurationRequest( ClientId clientId, RaftPeerId leaderId, - RaftPeer... peers) { + RaftPeer... raftPeers) { return new SetConfigurationRequest(clientId, leaderId, getGroupId(), CallId.getDefault(), - SetConfigurationRequest.Arguments.newBuilder().setServersInNewConf(peers).build()); + SetConfigurationRequest.Arguments.newBuilder().setServersInNewConf(raftPeers).build()); } - public void setConfiguration(RaftPeer... peers) throws IOException { + public void setConfiguration(RaftPeer... raftPeers) throws IOException { try(RaftClient client = createClient()) { - LOG.info("Start changing the configuration: {}", Arrays.asList(peers)); - final RaftClientReply reply = client.admin().setConfiguration(peers); + LOG.info("Start changing the configuration: {}", Arrays.asList(raftPeers)); + final RaftClientReply reply = client.admin().setConfiguration(raftPeers); Preconditions.assertTrue(reply.isSuccess()); } } @@ -842,18 +857,42 @@ public void shutdown() { final ExecutorService executor = Executors.newFixedThreadPool(servers.size(), (t) -> Daemon.newBuilder().setName("MiniRaftCluster-" + THREAD_COUNT.incrementAndGet()).setRunnable(t).build()); getServers().forEach(proxy -> executor.submit(() -> JavaUtils.runAsUnchecked(proxy::close))); - try { - executor.shutdown(); - // just wait for a few seconds - executor.awaitTermination(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { + final int maxRetries = 30; + final TimeDuration retrySleep = TimeDuration.ONE_SECOND; + executor.shutdown(); + boolean terminated = false; + + for(int i = 0; i < maxRetries && !terminated; ) { + try { + terminated = executor.awaitTermination(retrySleep.getDuration(), retrySleep.getUnit()); + if (!terminated) { + i++; + if (i < maxRetries) { + LOG.warn("{}/{}: Not yet able to shutdown executor, will wait again ...", i, maxRetries); + } else { + LOG.error("Failed to shutdown executor, some servers may be still running:\n{}", printServers()); + } + } + } catch (InterruptedException e) { LOG.warn("shutdown interrupted", e); - Thread.currentThread().interrupt(); + } } Optional.ofNullable(timer.get()).ifPresent(Timer::cancel); ExitUtils.assertNotTerminated(); LOG.info("{} shutdown completed", JavaUtils.getClassSimpleName(getClass())); + + // GC to ensure leak detection work. + try { + RaftTestUtil.gc(); + } catch (InterruptedException e) { + LOG.warn("gc interrupted.", e); + } + try { + ReferenceCountedLeakDetector.getLeakDetector().assertNoLeaks(maxRetries, retrySleep); + } catch (InterruptedException e) { + LOG.warn("LeakDetector interrupted.", e); + } } /** diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java index 3b8e206deb..843c271b1c 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftReconfigurationBaseTest.java @@ -51,7 +51,10 @@ import org.slf4j.event.Level; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -70,8 +73,8 @@ public abstract class RaftReconfigurationBaseTest logSyncDelay.setDelayMs(peer.getId().toString(), 1000)); + .forEach(peer -> LOG_SYNC_DELAY.setDelayMs(peer.getId().toString(), 1000)); final CountDownLatch latch = new CountDownLatch(1); final RaftPeer[] peersInRequest2 = cluster.getPeers().toArray(new RaftPeer[0]); @@ -691,7 +694,7 @@ void runTestOverlappedSetConfRequests(CLUSTER cluster) throws Exception { Assert.assertTrue(confChanged.get()); Assert.assertTrue(caughtException.get()); } finally { - logSyncDelay.clear(); + LOG_SYNC_DELAY.clear(); } } @@ -792,7 +795,7 @@ public void testLeaderNotReadyException() throws Exception { try { // delay 1s for each logSync call cluster.getServers().forEach( - peer -> leaderPlaceHolderDelay.setDelayMs(peer.getId().toString(), 2000)); + peer -> LEADER_PLACE_HOLDER_DELAY.setDelayMs(peer.getId().toString(), 2000)); cluster.start(); AtomicBoolean caughtNotReady = new AtomicBoolean(false); @@ -833,7 +836,7 @@ public void testLeaderNotReadyException() throws Exception { Assert.assertTrue(success.get()); Assert.assertTrue(caughtNotReady.get()); } finally { - leaderPlaceHolderDelay.clear(); + LEADER_PLACE_HOLDER_DELAY.clear(); cluster.shutdown(); } } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java index 2927ec3495..0054e16018 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/RaftServerTestUtil.java @@ -31,9 +31,12 @@ import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerRpc; import org.apache.ratis.server.leader.LogAppender; +import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog; import org.apache.ratis.server.storage.RaftStorage; +import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.MemoizedSupplier; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.function.CheckedConsumer; @@ -49,14 +52,18 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -public class RaftServerTestUtil { +public final class RaftServerTestUtil { static final Logger LOG = LoggerFactory.getLogger(RaftServerTestUtil.class); + private RaftServerTestUtil() { + } + public static final RaftGroupMemberId TEST_MEMBER_ID = RaftGroupMemberId.valueOf( RaftPeerId.valueOf("test"), RaftGroupId.emptyGroupId()); @@ -138,6 +145,15 @@ public static ConfigurationManager getConfigurationManager(RaftServer.Division s return (ConfigurationManager) RaftTestUtil.getDeclaredField(getState(server), "configurationManager"); } + public static Logger getTransactionContextLog() { + return TransactionManager.LOG; + } + + public static Map> getTransactionContextMap( + RaftServer.Division server) { + return ((RaftServerImpl)server).getTransactionContextMapForTesting(); + } + public static RaftConfiguration newRaftConfiguration(Collection peers) { return RaftConfigurationImpl.newBuilder().setConf(peers).build(); } @@ -235,7 +251,8 @@ private static Collection parseMinorityPeers(MiniRaftCluster cluster, } // All new peers has been added. Handle the removed peers. - List peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)).collect(Collectors.toList()); + List peersToRemove = peers.stream().filter(peer -> !peersInNewConf.contains(peer)) + .collect(Collectors.toList()); if (!peersToRemove.isEmpty()) { return peersInNewConf; } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java index f59958a943..439245b496 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/RetryCacheTestUtil.java @@ -36,7 +36,11 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class RetryCacheTestUtil { +public final class RetryCacheTestUtil { + + private RetryCacheTestUtil() { + } + public static RetryCache createRetryCache(){ return new RetryCacheImpl(TimeDuration.valueOf(60, TimeUnit.SECONDS), null); } @@ -80,7 +84,11 @@ public static SegmentedRaftLog newSegmentedRaftLog(RaftGroupMemberId memberId, R final RaftServerImpl server = mock(RaftServerImpl.class); when(server.getRetryCache()).thenReturn((RetryCacheImpl) retryCache); when(server.getMemberId()).thenReturn(memberId); + + final TransactionManager transactionManager = new TransactionManager(memberId.getPeerId()); + when(server.getTransactionManager()).thenReturn(transactionManager); doCallRealMethod().when(server).notifyTruncatedLogEntry(any(LogEntryProto.class)); + return SegmentedRaftLog.newBuilder() .setMemberId(memberId) .setServer(server) diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java index 28f8e6ace9..23578f01a1 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/StateMachineShutdownTests.java @@ -28,47 +28,112 @@ import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.TransactionContext; -import org.junit.Assert; -import org.junit.Test; - +import org.junit.*; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; - +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; public abstract class StateMachineShutdownTests extends BaseTest implements MiniRaftCluster.Factory.Get { - + public static Logger LOG = LoggerFactory.getLogger(StateMachineUpdater.class); + private static MockedStatic mocked; protected static class StateMachineWithConditionalWait extends SimpleStateMachine4Testing { + boolean unblockAllTxns = false; + final Set blockTxns = ConcurrentHashMap.newKeySet(); + private final ExecutorService executor = Executors.newFixedThreadPool(10); + public static Map>> futures = new ConcurrentHashMap<>(); + public static Map numTxns = new ConcurrentHashMap<>(); + private final Map appliedTxns = new ConcurrentHashMap<>(); + + private synchronized void updateTxns() { + long appliedIndex = this.getLastAppliedTermIndex().getIndex() + 1; + Long appliedTerm = null; + while (appliedTxns.containsKey(appliedIndex)) { + appliedTerm = appliedTxns.remove(appliedIndex); + appliedIndex += 1; + } + if (appliedTerm != null) { + updateLastAppliedTermIndex(appliedTerm, appliedIndex - 1); + } + } - private final Long objectToWait = 0L; - volatile boolean blockOnApply = true; + @Override + public void notifyTermIndexUpdated(long term, long index) { + appliedTxns.put(index, term); + updateTxns(); + } @Override public CompletableFuture applyTransaction(TransactionContext trx) { - if (blockOnApply) { - synchronized (objectToWait) { - try { - objectToWait.wait(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(); + final RaftProtos.LogEntryProto entry = trx.getLogEntryUnsafe(); + + CompletableFuture future = new CompletableFuture<>(); + futures.computeIfAbsent(Thread.currentThread().getId(), k -> new HashSet<>()).add(future); + executor.submit(() -> { + synchronized (blockTxns) { + if (!unblockAllTxns) { + blockTxns.add(entry.getIndex()); + } + while (!unblockAllTxns && blockTxns.contains(entry.getIndex())) { + try { + blockTxns.wait(10000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } + numTxns.computeIfAbsent(getId(), (k) -> new AtomicLong()).incrementAndGet(); + appliedTxns.put(entry.getIndex(), entry.getTerm()); + updateTxns(); + future.complete(new RaftTestUtil.SimpleMessage("done")); + }); + return future; + } + + public void unBlockApplyTxn(long txnId) { + synchronized (blockTxns) { + blockTxns.remove(txnId); + blockTxns.notifyAll(); } - RaftProtos.LogEntryProto entry = trx.getLogEntry(); - updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); - return CompletableFuture.completedFuture(new RaftTestUtil.SimpleMessage("done")); } - public void unBlockApplyTxn() { - blockOnApply = false; - synchronized (objectToWait) { - objectToWait.notifyAll(); + public void unblockAllTxns() { + unblockAllTxns = true; + synchronized (blockTxns) { + for (Long txnId : blockTxns) { + blockTxns.remove(txnId); + } + blockTxns.notifyAll(); } } } + @Before + public void setup() { + mocked = Mockito.mockStatic(CompletableFuture.class, Mockito.CALLS_REAL_METHODS); + } + + @After + public void tearDownClass() { + if (mocked != null) { + mocked.close(); + } + + } + @Test public void testStateMachineShutdownWaitsForApplyTxn() throws Exception { final RaftProperties prop = getProperties(); @@ -82,10 +147,9 @@ public void testStateMachineShutdownWaitsForApplyTxn() throws Exception { //Unblock leader and one follower ((StateMachineWithConditionalWait)leader.getStateMachine()) - .unBlockApplyTxn(); + .unblockAllTxns(); ((StateMachineWithConditionalWait)cluster. - getFollowers().get(0).getStateMachine()).unBlockApplyTxn(); - + getFollowers().get(0).getStateMachine()).unblockAllTxns(); cluster.getLeaderAndSendFirstMessage(true); try (final RaftClient client = cluster.createClient(leaderId)) { @@ -107,16 +171,30 @@ public void testStateMachineShutdownWaitsForApplyTxn() throws Exception { final Thread t = new Thread(secondFollower::close); t.start(); - // The second follower should still be blocked in apply transaction - Assert.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < logIndex); + // Now unblock the second follower - ((StateMachineWithConditionalWait) secondFollower.getStateMachine()) - .unBlockApplyTxn(); + long minIndex = ((StateMachineWithConditionalWait) secondFollower.getStateMachine()).blockTxns.stream() + .min(Comparator.naturalOrder()).get(); + Assert.assertEquals(2, StateMachineWithConditionalWait.numTxns.values().stream() + .filter(val -> val.get() == 3).count()); + // The second follower should still be blocked in apply transaction + Assert.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < minIndex); + for (long index : ((StateMachineWithConditionalWait) secondFollower.getStateMachine()).blockTxns) { + if (minIndex != index) { + ((StateMachineWithConditionalWait) secondFollower.getStateMachine()).unBlockApplyTxn(index); + } + } + Assert.assertEquals(2, StateMachineWithConditionalWait.numTxns.values().stream() + .filter(val -> val.get() == 3).count()); + Assert.assertTrue(secondFollower.getInfo().getLastAppliedIndex() < minIndex); + ((StateMachineWithConditionalWait) secondFollower.getStateMachine()).unBlockApplyTxn(minIndex); // Now wait for the thread t.join(5000); Assert.assertEquals(logIndex, secondFollower.getInfo().getLastAppliedIndex()); + Assert.assertEquals(3, StateMachineWithConditionalWait.numTxns.values().stream() + .filter(val -> val.get() == 3).count()); cluster.shutdown(); } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/TestLogAppenderMetrics.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/TestLogAppenderMetrics.java index 5c78db42f6..6a46734689 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/TestLogAppenderMetrics.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/TestLogAppenderMetrics.java @@ -27,11 +27,11 @@ import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.metrics.LogAppenderMetrics; import org.apache.ratis.util.Timestamp; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; import org.apache.ratis.thirdparty.com.codahale.metrics.Gauge; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestLogAppenderMetrics { @@ -39,7 +39,7 @@ public class TestLogAppenderMetrics { private RaftPeerId raftPeerId; private MyFollowerInfo followerInfo; - @Before + @BeforeEach public void setup() { RaftGroupId raftGroupId = RaftGroupId.randomId(); raftPeerId = RaftPeerId.valueOf("TestId"); @@ -55,19 +55,19 @@ public void setup() { public void testLogAppenderGauges() { Gauge nextIndex = ratisMetricRegistry.getGauges((s, metric) -> s.contains(String.format(FOLLOWER_NEXT_INDEX, raftPeerId.toString()))).values().iterator().next(); - Assert.assertEquals(100L, nextIndex.getValue()); + Assertions.assertEquals(100L, nextIndex.getValue()); Gauge matchIndex = ratisMetricRegistry.getGauges((s, metric) -> s.contains(String.format(FOLLOWER_MATCH_INDEX, raftPeerId.toString()))).values().iterator().next(); - Assert.assertEquals(0L, matchIndex.getValue()); + Assertions.assertEquals(0L, matchIndex.getValue()); Gauge rpcTime = ratisMetricRegistry.getGauges((s, metric) -> s.contains(String.format(FOLLOWER_RPC_RESP_TIME, raftPeerId.toString()))).values().iterator().next(); - Assert.assertTrue(((Long) rpcTime.getValue()) > 0); + Assertions.assertTrue(((Long) rpcTime.getValue()) > 0); followerInfo.updateNextIndex(200L); followerInfo.updateMatchIndex(100L); followerInfo.updateLastRpcResponseTime(); - Assert.assertEquals(200L, nextIndex.getValue()); - Assert.assertEquals(100L, matchIndex.getValue()); - Assert.assertNotNull(rpcTime.getValue()); + Assertions.assertEquals(200L, nextIndex.getValue()); + Assertions.assertEquals(100L, matchIndex.getValue()); + Assertions.assertNotNull(rpcTime.getValue()); } private static class MyFollowerInfo { @@ -83,16 +83,16 @@ long getNextIndex() { return nextIndex; } - void updateNextIndex(long nextIndex) { - this.nextIndex = nextIndex; + void updateNextIndex(long index) { + this.nextIndex = index; } long getMatchIndex() { return matchIndex; } - void updateMatchIndex(long matchIndex) { - this.matchIndex = matchIndex; + void updateMatchIndex(long index) { + this.matchIndex = index; } Timestamp getLastRpcTime() { diff --git a/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRetryCacheMetrics.java b/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRetryCacheMetrics.java index b25a50bf97..0d779a2dc0 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRetryCacheMetrics.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/impl/TestRetryCacheMetrics.java @@ -19,7 +19,7 @@ package org.apache.ratis.server.impl; import static org.apache.ratis.server.metrics.RaftServerMetricsImpl.*; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl; import org.apache.ratis.thirdparty.com.codahale.metrics.Gauge; @@ -30,9 +30,9 @@ import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.metrics.RaftServerMetricsImpl; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import java.util.Map; @@ -43,7 +43,7 @@ public class TestRetryCacheMetrics { private static RatisMetricRegistryImpl ratisMetricRegistry; private static RetryCacheImpl retryCache; - @BeforeClass + @BeforeAll public static void setUp() { RaftGroupId raftGroupId = RaftGroupId.randomId(); RaftPeerId raftPeerId = RaftPeerId.valueOf("TestId"); @@ -56,7 +56,7 @@ public static void setUp() { ratisMetricRegistry = (RatisMetricRegistryImpl) raftServerMetrics.getRegistry(); } - @After + @AfterEach public void tearDown() { retryCache.close(); checkEntryCount(0); diff --git a/ratis-server/src/test/java/org/apache/ratis/server/metrics/TestLeaderElectionMetrics.java b/ratis-server/src/test/java/org/apache/ratis/server/metrics/TestLeaderElectionMetrics.java index a39612c57e..bba5d92316 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/metrics/TestLeaderElectionMetrics.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/metrics/TestLeaderElectionMetrics.java @@ -20,8 +20,8 @@ import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LAST_LEADER_ELECTION_ELAPSED_TIME; import static org.apache.ratis.server.metrics.LeaderElectionMetrics.LEADER_ELECTION_TIMEOUT_COUNT_METRIC; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl; import org.apache.ratis.thirdparty.com.codahale.metrics.Gauge; @@ -29,8 +29,8 @@ import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeerId; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import java.util.SortedMap; @@ -42,7 +42,7 @@ public class TestLeaderElectionMetrics extends BaseTest { private static LeaderElectionMetrics leaderElectionMetrics; private static RatisMetricRegistryImpl ratisMetricRegistry; - @BeforeClass + @BeforeAll public static void setUp() { RaftGroupId raftGroupId = RaftGroupId.randomId(); RaftPeerId raftPeerId = RaftPeerId.valueOf("TestId"); @@ -58,14 +58,14 @@ public void testOnLeaderElectionCompletion() throws Exception { (s, metric) -> s.contains(LAST_LEADER_ELECTION_ELAPSED_TIME)); LOG.info("{} gauges: {}", LAST_LEADER_ELECTION_ELAPSED_TIME, gauges); final Long leaderElectionLatency = (Long)gauges.values().iterator().next().getValue(); - assertTrue("leaderElectionLatency = " + leaderElectionLatency, leaderElectionLatency >= 0L); + assertTrue(leaderElectionLatency >= 0L, "leaderElectionLatency = " + leaderElectionLatency); } @Test public void testOnLeaderElectionTimeout() throws Exception { long numLeaderElectionTimeouts = ratisMetricRegistry.counter( LEADER_ELECTION_TIMEOUT_COUNT_METRIC).getCount(); - assertTrue(numLeaderElectionTimeouts == 0); + assertEquals(0, numLeaderElectionTimeouts); leaderElectionMetrics.onLeaderElectionTimeout(); numLeaderElectionTimeouts = ratisMetricRegistry.counter(LEADER_ELECTION_TIMEOUT_COUNT_METRIC).getCount(); assertEquals(1, numLeaderElectionTimeouts); diff --git a/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java b/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java index e242eddf59..473aa0a46c 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/raftlog/segmented/SegmentedRaftLogTestUtils.java @@ -17,25 +17,33 @@ */ package org.apache.ratis.server.raftlog.segmented; +import org.apache.ratis.server.RaftServer; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.Slf4jUtils; import org.slf4j.event.Level; import java.io.File; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; public interface SegmentedRaftLogTestUtils { SizeInBytes MAX_OP_SIZE = SizeInBytes.valueOf("32MB"); static SegmentedRaftLogInputStream newSegmentedRaftLogInputStream(File log, long startIndex, long endIndex, boolean isOpen) { - return new SegmentedRaftLogInputStream(log, startIndex, endIndex, isOpen, MAX_OP_SIZE, null); + final LogSegmentStartEnd startEnd = LogSegmentStartEnd.valueOf(startIndex, endIndex, isOpen); + return new SegmentedRaftLogInputStream(log, startEnd, MAX_OP_SIZE, null); } static void setRaftLogWorkerLogLevel(Level level) { Slf4jUtils.setLogLevel(SegmentedRaftLogWorker.LOG, level); } - static String getLogFlushTimeMetric(String memberId) { - return SegmentedRaftLogWorker.class.getName() + "." + memberId + ".flush-time"; + static List getOpenLogFiles(RaftServer.Division server) throws Exception { + return LogSegmentPath.getLogSegmentPaths(server.getRaftStorage()).stream() + .filter(p -> p.getStartEnd().isOpen()) + .map(LogSegmentPath::getPath) + .collect(Collectors.toList()); } } diff --git a/ratis-server/src/test/java/org/apache/ratis/server/simulation/MiniRaftClusterWithSimulatedRpc.java b/ratis-server/src/test/java/org/apache/ratis/server/simulation/MiniRaftClusterWithSimulatedRpc.java index d0e7e9f5c8..922796cb80 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/simulation/MiniRaftClusterWithSimulatedRpc.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/simulation/MiniRaftClusterWithSimulatedRpc.java @@ -34,7 +34,7 @@ import static org.apache.ratis.conf.ConfUtils.requireMin; -public class MiniRaftClusterWithSimulatedRpc extends MiniRaftCluster { +public final class MiniRaftClusterWithSimulatedRpc extends MiniRaftCluster { static final Logger LOG = LoggerFactory.getLogger(MiniRaftClusterWithSimulatedRpc.class); public static final Factory FACTORY diff --git a/ratis-server/src/test/java/org/apache/ratis/server/simulation/RaftServerReply.java b/ratis-server/src/test/java/org/apache/ratis/server/simulation/RaftServerReply.java index 5d9d871f8c..4ad9268e33 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/simulation/RaftServerReply.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/simulation/RaftServerReply.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.server.simulation; -import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftRpcMessage; import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto; @@ -94,11 +93,6 @@ StartLeaderElectionReplyProto getStartLeaderElection() { return startLeaderElection; } - @Override - public boolean isRequest() { - return false; - } - @Override public String getRequestorId() { if (isAppendEntries()) { diff --git a/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRequestReply.java b/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRequestReply.java index 0874e7466c..1b7da951c1 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRequestReply.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRequestReply.java @@ -27,7 +27,11 @@ import java.io.IOException; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; diff --git a/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRpc.java b/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRpc.java index 0399b414bf..e570c35af0 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRpc.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/simulation/SimulatedRpc.java @@ -28,9 +28,13 @@ import java.util.Objects; -class SimulatedRpc implements RpcType { +public class SimulatedRpc implements RpcType { static final SimulatedRpc INSTANCE = new SimulatedRpc(); + public static SimulatedRpc get() { + return INSTANCE; + } + @Override public String name() { return getClass().getName(); diff --git a/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java b/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java index bb4f6a076c..ee30bd2c79 100644 --- a/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java +++ b/ratis-server/src/test/java/org/apache/ratis/server/storage/RaftStorageTestUtils.java @@ -21,12 +21,15 @@ import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.RATIS_LOG_WORKER_METRICS; import org.apache.ratis.metrics.RatisMetrics; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.raftlog.LogProtoUtils; +import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.server.raftlog.RaftLogBase; import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.util.AutoCloseableLock; +import org.apache.ratis.util.ReferenceCountedObject; import java.io.File; import java.io.IOException; @@ -72,11 +75,22 @@ static void printLog(RaftLogBase log, Consumer println) { b.append(i == committed? 'c': ' '); b.append(String.format("%3d: ", i)); try { - b.append(LogProtoUtils.toLogEntryString(log.get(i))); + b.append(LogProtoUtils.toLogEntryString(getLogUnsafe(log, i))); } catch (RaftLogIOException e) { b.append(e); } println.accept(b.toString()); } } + + static LogEntryProto getLogUnsafe(RaftLog log, long index) throws RaftLogIOException { + ReferenceCountedObject ref = log.retainLog(index); + try { + return ref != null ? ref.get() : null; + } finally { + if (ref != null) { + ref.release(); + } + } + } } diff --git a/ratis-server/src/test/java/org/apache/ratis/statemachine/RaftSnapshotBaseTest.java b/ratis-server/src/test/java/org/apache/ratis/statemachine/RaftSnapshotBaseTest.java index fe1a97ddca..bbe0284906 100644 --- a/ratis-server/src/test/java/org/apache/ratis/statemachine/RaftSnapshotBaseTest.java +++ b/ratis-server/src/test/java/org/apache/ratis/statemachine/RaftSnapshotBaseTest.java @@ -21,6 +21,7 @@ import static org.apache.ratis.server.impl.StateMachineMetrics.RATIS_STATEMACHINE_METRICS_DESC; import static org.apache.ratis.server.impl.StateMachineMetrics.STATEMACHINE_TAKE_SNAPSHOT_TIMER; import static org.apache.ratis.metrics.RatisMetrics.RATIS_APPLICATION_NAME_METRICS; +import static org.apache.ratis.server.storage.RaftStorageTestUtils.getLogUnsafe; import org.apache.ratis.BaseTest; import org.apache.ratis.metrics.LongCounter; @@ -49,10 +50,10 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.Slf4jUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -95,17 +96,17 @@ public static void assertLeaderContent(MiniRaftCluster cluster) throws Exception public static void assertLogContent(RaftServer.Division server, boolean isLeader) throws Exception { final RaftLog log = server.getRaftLog(); final long lastIndex = log.getLastEntryTermIndex().getIndex(); - final LogEntryProto e = log.get(lastIndex); - Assert.assertTrue(e.hasMetadataEntry()); + final LogEntryProto e = getLogUnsafe(log, lastIndex); + Assertions.assertTrue(e.hasMetadataEntry()); JavaUtils.attemptRepeatedly(() -> { - Assert.assertEquals(log.getLastCommittedIndex() - 1, e.getMetadataEntry().getCommitIndex()); + Assertions.assertEquals(log.getLastCommittedIndex() - 1, e.getMetadataEntry().getCommitIndex()); return null; }, 50, BaseTest.HUNDRED_MILLIS, "CheckMetadataEntry", LOG); SimpleStateMachine4Testing simpleStateMachine = SimpleStateMachine4Testing.get(server); if (isLeader) { - Assert.assertTrue("Not notified as a leader", simpleStateMachine.isNotifiedAsLeader()); + Assertions.assertTrue(simpleStateMachine.isNotifiedAsLeader(), "Not notified as a leader"); } final LogEntryProto[] entries = simpleStateMachine.getContent(); long message = 0; @@ -113,7 +114,7 @@ public static void assertLogContent(RaftServer.Division server, boolean isLeader LOG.info("{}) {} {}", i, message, entries[i].toString().replace("\n", ", ")); if (entries[i].hasStateMachineLogEntry()) { final SimpleMessage m = new SimpleMessage("m" + message++); - Assert.assertArrayEquals(m.getContent().toByteArray(), + Assertions.assertArrayEquals(m.getContent().toByteArray(), entries[i].getStateMachineLogEntry().getLogData().toByteArray()); } } @@ -123,7 +124,7 @@ public static void assertLogContent(RaftServer.Division server, boolean isLeader public abstract MiniRaftCluster.Factory getFactory(); - @Before + @BeforeEach public void setup() throws IOException { final RaftProperties prop = new RaftProperties(); prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, @@ -135,7 +136,7 @@ public void setup() throws IOException { cluster.start(); } - @After + @AfterEach public void tearDown() { if (cluster != null) { cluster.shutdown(); @@ -155,7 +156,7 @@ public void testRestartPeer() throws Exception { try(final RaftClient client = cluster.createClient(leaderId)) { for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) { RaftClientReply reply = client.io().send(new SimpleMessage("m" + i)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } @@ -164,7 +165,7 @@ public void testRestartPeer() throws Exception { // wait for the snapshot to be done final List snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex); JavaUtils.attemptRepeatedly(() -> { - Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists)); + Assertions.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists)); return null; }, 10, ONE_SECOND, "snapshotFile.exist", LOG); @@ -202,7 +203,7 @@ public void testBasicInstallSnapshot() throws Exception { try(final RaftClient client = cluster.createClient(leaderId)) { for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) { RaftClientReply reply = client.io().send(new SimpleMessage("m" + i)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } @@ -211,7 +212,7 @@ public void testBasicInstallSnapshot() throws Exception { LOG.info("nextIndex = {}", nextIndex); final List snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex); JavaUtils.attemptRepeatedly(() -> { - Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists)); + Assertions.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists)); return null; }, 10, ONE_SECOND, "snapshotFile.exist", LOG); verifyTakeSnapshotMetric(cluster.getLeader()); @@ -233,7 +234,7 @@ public void testBasicInstallSnapshot() throws Exception { // generate some more traffic try(final RaftClient client = cluster.createClient(cluster.getLeader().getId())) { - Assert.assertTrue(client.io().send(new SimpleMessage("m" + i)).isSuccess()); + Assertions.assertTrue(client.io().send(new SimpleMessage("m" + i)).isSuccess()); } // add two more peers @@ -247,7 +248,7 @@ public void testBasicInstallSnapshot() throws Exception { for (String newPeer : newPeers) { final RaftServer.Division s = cluster.getDivision(RaftPeerId.valueOf(newPeer)); SimpleStateMachine4Testing simpleStateMachine = SimpleStateMachine4Testing.get(s); - Assert.assertSame(LifeCycle.State.RUNNING, simpleStateMachine.getLifeCycleState()); + Assertions.assertSame(LifeCycle.State.RUNNING, simpleStateMachine.getLifeCycleState()); } // Verify installSnapshot counter on leader before restart. @@ -262,7 +263,7 @@ public void testBasicInstallSnapshot() throws Exception { assertLeaderContent(cluster); // verify that snapshot was taken when stopping the server - Assert.assertTrue(count < timer.getCount()); + Assertions.assertTrue(count < timer.getCount()); } finally { cluster.shutdown(); } @@ -283,7 +284,7 @@ public void testInstallSnapshotDuringBootstrap() throws Exception { try(final RaftClient client = cluster.createClient(leaderId)) { for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) { RaftClientReply reply = client.io().send(new SimpleMessage("m" + i)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } @@ -292,7 +293,7 @@ public void testInstallSnapshotDuringBootstrap() throws Exception { LOG.info("nextIndex = {}", nextIndex); final List snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex); JavaUtils.attemptRepeatedly(() -> { - Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists)); + Assertions.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists)); return null; }, 10, ONE_SECOND, "snapshotFile.exist", LOG); verifyTakeSnapshotMetric(cluster.getLeader()); @@ -310,7 +311,7 @@ public void testInstallSnapshotDuringBootstrap() throws Exception { for (String newPeer : newPeers) { final RaftServer.Division s = cluster.getDivision(RaftPeerId.valueOf(newPeer)); SimpleStateMachine4Testing simpleStateMachine = SimpleStateMachine4Testing.get(s); - Assert.assertSame(LifeCycle.State.RUNNING, simpleStateMachine.getLifeCycleState()); + Assertions.assertSame(LifeCycle.State.RUNNING, simpleStateMachine.getLifeCycleState()); } // Verify installSnapshot counter on leader @@ -324,13 +325,13 @@ public void testInstallSnapshotDuringBootstrap() throws Exception { protected void verifyInstallSnapshotMetric(RaftServer.Division leader) { final LongCounter installSnapshotCounter = ((RaftServerMetricsImpl)leader.getRaftServerMetrics()) .getNumInstallSnapshot(); - Assert.assertNotNull(installSnapshotCounter); - Assert.assertTrue(installSnapshotCounter.getCount() >= 1); + Assertions.assertNotNull(installSnapshotCounter); + Assertions.assertTrue(installSnapshotCounter.getCount() >= 1); } private static void verifyTakeSnapshotMetric(RaftServer.Division leader) { Timer timer = getTakeSnapshotTimer(leader); - Assert.assertTrue(timer.getCount() > 0); + Assertions.assertTrue(timer.getCount() > 0); } private static Timer getTakeSnapshotTimer(RaftServer.Division leader) { @@ -338,9 +339,9 @@ private static Timer getTakeSnapshotTimer(RaftServer.Division leader) { RATIS_APPLICATION_NAME_METRICS, RATIS_STATEMACHINE_METRICS, RATIS_STATEMACHINE_METRICS_DESC); Optional opt = MetricRegistries.global().get(info); - Assert.assertTrue(opt.isPresent()); + Assertions.assertTrue(opt.isPresent()); RatisMetricRegistry metricRegistry = opt.get(); - Assert.assertNotNull(metricRegistry); + Assertions.assertNotNull(metricRegistry); return ((DefaultTimekeeperImpl)metricRegistry.timer(STATEMACHINE_TAKE_SNAPSHOT_TIMER)).getTimer(); } } diff --git a/ratis-server/src/test/java/org/apache/ratis/statemachine/impl/SimpleStateMachine4Testing.java b/ratis-server/src/test/java/org/apache/ratis/statemachine/impl/SimpleStateMachine4Testing.java index 312c9508d3..74bc0c5355 100644 --- a/ratis-server/src/test/java/org/apache/ratis/statemachine/impl/SimpleStateMachine4Testing.java +++ b/ratis-server/src/test/java/org/apache/ratis/statemachine/impl/SimpleStateMachine4Testing.java @@ -47,6 +47,7 @@ import org.apache.ratis.util.LifeCycle; import org.apache.ratis.util.MD5FileUtil; import org.apache.ratis.util.Preconditions; +import org.apache.ratis.util.ReferenceCountedObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -208,12 +209,12 @@ private void put(LogEntryProto entry) { } @Override - public synchronized void initialize(RaftServer server, RaftGroupId groupId, + public synchronized void initialize(RaftServer server, RaftGroupId raftGroupId, RaftStorage raftStorage) throws IOException { LOG.info("Initializing " + this); - this.groupId = groupId; + this.groupId = raftGroupId; getLifeCycle().startAndTransition(() -> { - super.initialize(server, groupId, raftStorage); + super.initialize(server, raftGroupId, raftStorage); storage.init(raftStorage); loadSnapshot(storage.getLatestSnapshot()); @@ -244,10 +245,11 @@ public synchronized void reinitialize() throws IOException { @Override public CompletableFuture applyTransaction(TransactionContext trx) { blocking.await(Blocking.Type.APPLY_TRANSACTION); - LogEntryProto entry = Objects.requireNonNull(trx.getLogEntry()); + ReferenceCountedObject entryRef = Objects.requireNonNull(trx.getLogEntryRef()); + LogEntryProto entry = entryRef.get(); LOG.info("applyTransaction for log index {}", entry.getIndex()); - put(entry); + put(LogProtoUtils.copy(entry)); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); final SimpleMessage m = new SimpleMessage(entry.getIndex() + " OK"); @@ -328,10 +330,10 @@ public CompletableFuture query(Message request) { final String string = request.getContent().toStringUtf8(); Exception exception; try { - LOG.info("query " + string); + LOG.info("query {}, all available: {}", string, dataMap.keySet()); final LogEntryProto entry = dataMap.get(string); if (entry != null) { - return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString())); + return CompletableFuture.completedFuture(Message.valueOf(entry)); } exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string); } catch (Exception e) { @@ -355,7 +357,8 @@ public TransactionContext startTransaction(RaftClientRequest request) { } @Override - public CompletableFuture write(LogEntryProto entry) { + public CompletableFuture write(ReferenceCountedObject entry, TransactionContext context) { + Preconditions.assertTrue(entry.get() != null); return blocking.getFuture(Blocking.Type.WRITE_STATE_MACHINE_DATA); } @@ -376,6 +379,8 @@ public void close() { running = false; checkpointer.interrupt(); }); + indexMap.clear(); + dataMap.clear(); } public LogEntryProto[] getContent() { diff --git a/ratis-shell/pom.xml b/ratis-shell/pom.xml index 78ab6679fd..41c2a0ec33 100644 --- a/ratis-shell/pom.xml +++ b/ratis-shell/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-shell Apache Ratis Shell + + + true + + ratis-client diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/CliUtils.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/CliUtils.java new file mode 100644 index 0000000000..1cecc665c6 --- /dev/null +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/CliUtils.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.shell.cli; + +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.protocol.GroupInfoReply; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.exceptions.RaftException; +import org.apache.ratis.util.function.CheckedFunction; + +import java.io.IOException; +import java.io.PrintStream; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Utilities for command line interface. + */ +public final class CliUtils { + private CliUtils() { + // prevent instantiation + } + + /** @return {@link RaftPeerId} from the given address. */ + public static RaftPeerId getPeerId(InetSocketAddress address) { + return getPeerId(address.getHostString(), address.getPort()); + } + + /** @return {@link RaftPeerId} from the given host and port. */ + public static RaftPeerId getPeerId(String host, int port) { + return RaftPeerId.getRaftPeerId(host + "_" + port); + } + + /** + * Apply the given function to the given parameter a list. + * + * @param list the input parameter list + * @param function the function to be applied + * @param parameter type + * @param return value type + * @param the exception type thrown by the given function. + * @return the first non-null value returned by the given function applied to the given list. + */ + private static RETURN applyFunctionReturnFirstNonNull( + Collection list, CheckedFunction function, PrintStream out) { + for (PARAMETER parameter : list) { + try { + RETURN ret = function.apply(parameter); + if (ret != null) { + return ret; + } + } catch (Throwable e) { + e.printStackTrace(out); + } + } + return null; + } + + /** Parse the given string as a list of {@link RaftPeer}. */ + public static List parseRaftPeers(String peers) { + List addresses = new ArrayList<>(); + String[] peersArray = peers.split(","); + for (String peer : peersArray) { + addresses.add(parseInetSocketAddress(peer)); + } + + return addresses.stream() + .map(addr -> RaftPeer.newBuilder().setId(getPeerId(addr)).setAddress(addr).build()) + .collect(Collectors.toList()); + } + + /** Parse the given string as a {@link RaftGroupId}. */ + public static RaftGroupId parseRaftGroupId(String groupId) { + return groupId != null && groupId.isEmpty() ? RaftGroupId.valueOf(UUID.fromString(groupId)) : null; + } + + /** + * Get the group id from the given peers if the given group id is null. + * + * @param client for communicating to the peers. + * @param peers the peers of the group. + * @param groupId the given group id, if there is any. + * @param err for printing error messages. + * @return the group id from the given peers if the given group id is null; + * otherwise, return the given group id. + */ + public static RaftGroupId getGroupId(RaftClient client, List peers, RaftGroupId groupId, + PrintStream err) throws IOException { + if (groupId != null) { + return groupId; + } + + final List groupIds = applyFunctionReturnFirstNonNull(peers, + p -> client.getGroupManagementApi(p.getId()).list().getGroupIds(), err); + + if (groupIds == null) { + final String message = "Failed to get group ID from " + peers; + err.println("Failed to get group ID from " + peers); + throw new IOException(message); + } else if (groupIds.size() == 1) { + return groupIds.get(0); + } else { + String message = "Unexpected multiple group IDs " + groupIds + + ". In such case, the target group ID must be specified."; + err.println(message); + throw new IOException(message); + } + } + + /** + * Get the group info from the given peers. + * + * @param client for communicating to the peers. + * @param peers the peers of the group. + * @param groupId the target group + * @param err for printing error messages. + * @return the group info + */ + public static GroupInfoReply getGroupInfo(RaftClient client, List peers, RaftGroupId groupId, + PrintStream err) throws IOException { + GroupInfoReply groupInfoReply = applyFunctionReturnFirstNonNull(peers, + p -> client.getGroupManagementApi((p.getId())).info(groupId), err); + checkReply(groupInfoReply, () -> "Failed to get group info for " + groupId.getUuid() + + " from " + peers, err); + return groupInfoReply; + } + + /** Check if the given reply is success. */ + public static void checkReply(RaftClientReply reply, Supplier message, PrintStream printStream) + throws IOException { + if (reply == null || !reply.isSuccess()) { + final RaftException e = Optional.ofNullable(reply) + .map(RaftClientReply::getException) + .orElseGet(() -> new RaftException("Reply: " + reply)); + printStream.println(message.get()); + throw new IOException(message.get(), e); + } + } + + /** Parse the given string as a {@link InetSocketAddress}. */ + public static InetSocketAddress parseInetSocketAddress(String address) { + try { + final String[] hostPortPair = address.split(":"); + if (hostPortPair.length < 2) { + throw new IllegalArgumentException("Unexpected address format ."); + } + return new InetSocketAddress(hostPortPair[0], Integer.parseInt(hostPortPair[1])); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse the server address parameter \"" + address + "\".", e); + } + } + +} diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/RaftUtils.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/RaftUtils.java deleted file mode 100644 index 1239fc56c4..0000000000 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/RaftUtils.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ratis.shell.cli; - -import org.apache.ratis.client.RaftClient; -import org.apache.ratis.client.RaftClientConfigKeys; -import org.apache.ratis.conf.RaftProperties; -import org.apache.ratis.protocol.RaftGroup; -import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.retry.ExponentialBackoffRetry; -import org.apache.ratis.util.TimeDuration; - -import java.net.InetSocketAddress; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -/** - * Helper class for raft operations. - */ -public final class RaftUtils { - - private RaftUtils() { - // prevent instantiation - } - - /** - * Gets the raft peer id. - * - * @param address the address of the server - * @return the raft peer id - */ - public static RaftPeerId getPeerId(InetSocketAddress address) { - return getPeerId(address.getHostString(), address.getPort()); - } - - /** - * Gets the raft peer id. - * - * @param host the hostname of the server - * @param port the port of the server - * @return the raft peer id - */ - public static RaftPeerId getPeerId(String host, int port) { - return RaftPeerId.getRaftPeerId(host + "_" + port); - } - - /** - * Create a raft client to communicate to ratis server. - * @param raftGroup the raft group - * @return return a raft client - */ - public static RaftClient createClient(RaftGroup raftGroup) { - RaftProperties properties = new RaftProperties(); - RaftClientConfigKeys.Rpc.setRequestTimeout(properties, - TimeDuration.valueOf(15, TimeUnit.SECONDS)); - - // Since ratis-shell support GENERIC_COMMAND_OPTIONS, here we should - // merge these options to raft properties to make it work. - final Properties sys = System.getProperties(); - sys.stringPropertyNames().forEach(key -> properties.set(key, sys.getProperty(key))); - - ExponentialBackoffRetry retryPolicy = ExponentialBackoffRetry.newBuilder() - .setBaseSleepTime(TimeDuration.valueOf(1000, TimeUnit.MILLISECONDS)) - .setMaxAttempts(10) - .setMaxSleepTime( - TimeDuration.valueOf(100_000, TimeUnit.MILLISECONDS)) - .build(); - return RaftClient.newBuilder() - .setRaftGroup(raftGroup) - .setProperties(properties) - .setRetryPolicy(retryPolicy) - .build(); - } -} diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/RatisShell.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/RatisShell.java index 2e53e31912..cdad683011 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/RatisShell.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/RatisShell.java @@ -17,6 +17,9 @@ */ package org.apache.ratis.shell.cli.sh; +import org.apache.ratis.conf.Parameters; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.retry.RetryPolicy; import org.apache.ratis.shell.cli.AbstractShell; import org.apache.ratis.shell.cli.Command; import org.apache.ratis.shell.cli.sh.command.AbstractParentCommand; @@ -60,7 +63,11 @@ public static void main(String[] args) { } public RatisShell(PrintStream out) { - super(new Context(out)); + this(new Context(out)); + } + + private RatisShell(Context context) { + super(context); } @Override @@ -73,4 +80,39 @@ protected Map loadCommands(Context context) { return allParentCommands(context).stream() .collect(Collectors.toMap(Command::getCommandName, Function.identity())); } + + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + private PrintStream printStream = System.out; + private RetryPolicy retryPolicy; + private RaftProperties properties; + private Parameters parameters; + + public Builder setPrintStream(PrintStream printStream) { + this.printStream = printStream; + return this; + } + + public Builder setRetryPolicy(RetryPolicy retryPolicy) { + this.retryPolicy = retryPolicy; + return this; + } + + public Builder setProperties(RaftProperties properties) { + this.properties = properties; + return this; + } + + public Builder setParameters(Parameters parameters) { + this.parameters = parameters; + return this; + } + + public RatisShell build() { + return new RatisShell(new Context(printStream, false, retryPolicy, properties, parameters)); + } + } } diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java index 20a52a80f8..e1d7f8e0b6 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractCommand.java @@ -20,40 +20,31 @@ import org.apache.ratis.shell.cli.Command; import java.io.PrintStream; -import java.net.InetSocketAddress; /** * The base class for all the ratis shell {@link Command} classes. */ public abstract class AbstractCommand implements Command { - private final PrintStream printStream; + private final Context context; protected AbstractCommand(Context context) { - printStream = context.getPrintStream(); + this.context = context; } - public static InetSocketAddress parseInetSocketAddress(String address) { - try { - final String[] hostPortPair = address.split(":"); - if (hostPortPair.length < 2) { - throw new IllegalArgumentException("Unexpected address format ."); - } - return new InetSocketAddress(hostPortPair[0], Integer.parseInt(hostPortPair[1])); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse the server address parameter \"" + address + "\".", e); - } + protected Context getContext() { + return context; } protected PrintStream getPrintStream() { - return printStream; + return getContext().getPrintStream(); } protected void printf(String format, Object... args) { - printStream.printf(format, args); + getPrintStream().printf(format, args); } protected void println(Object message) { - printStream.println(message); + getPrintStream().println(message); } } diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java index 1888c0e0ea..aea1f7c4b3 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/AbstractRatisCommand.java @@ -18,9 +18,13 @@ package org.apache.ratis.shell.cli.sh.command; import org.apache.commons.cli.Option; -import org.apache.ratis.protocol.*; -import org.apache.ratis.protocol.exceptions.RaftException; -import org.apache.ratis.shell.cli.RaftUtils; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.GroupInfoReply; +import org.apache.ratis.shell.cli.CliUtils; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.ratis.client.RaftClient; @@ -30,11 +34,14 @@ import org.apache.ratis.proto.RaftProtos.RaftPeerRole; import org.apache.ratis.proto.RaftProtos.RoleInfoProto; import org.apache.ratis.util.ProtoUtils; -import org.apache.ratis.util.function.CheckedFunction; import java.io.IOException; +import java.io.PrintStream; import java.net.InetSocketAddress; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -46,32 +53,6 @@ public abstract class AbstractRatisCommand extends AbstractCommand { public static final String PEER_OPTION_NAME = "peers"; public static final String GROUPID_OPTION_NAME = "groupid"; - public static final RaftGroupId DEFAULT_RAFT_GROUP_ID = RaftGroupId.randomId(); - - /** - * Execute a given function with input parameter from the members of a list. - * - * @param list the input parameters - * @param function the function to be executed - * @param parameter type - * @param return value type - * @param the exception type thrown by the given function. - * @return the value returned by the given function. - */ - public static K run(Collection list, CheckedFunction function) { - for (T t : list) { - try { - K ret = function.apply(t); - if (ret != null) { - return ret; - } - } catch (Throwable e) { - e.printStackTrace(); - } - } - return null; - } - private RaftGroup raftGroup; private GroupInfoReply groupInfoReply; @@ -81,51 +62,22 @@ protected AbstractRatisCommand(Context context) { @Override public int run(CommandLine cl) throws IOException { - List addresses = new ArrayList<>(); - String peersStr = cl.getOptionValue(PEER_OPTION_NAME); - String[] peersArray = peersStr.split(","); - for (String peer : peersArray) { - addresses.add(parseInetSocketAddress(peer)); - } - - final RaftGroupId raftGroupIdFromConfig = cl.hasOption(GROUPID_OPTION_NAME)? - RaftGroupId.valueOf(UUID.fromString(cl.getOptionValue(GROUPID_OPTION_NAME))) - : DEFAULT_RAFT_GROUP_ID; - - List peers = addresses.stream() - .map(addr -> RaftPeer.newBuilder() - .setId(RaftUtils.getPeerId(addr)) - .setAddress(addr) - .build() - ).collect(Collectors.toList()); - raftGroup = RaftGroup.valueOf(raftGroupIdFromConfig, peers); - try (final RaftClient client = RaftUtils.createClient(raftGroup)) { - final RaftGroupId remoteGroupId; - if (raftGroupIdFromConfig != DEFAULT_RAFT_GROUP_ID) { - remoteGroupId = raftGroupIdFromConfig; - } else { - final List groupIds = run(peers, - p -> client.getGroupManagementApi((p.getId())).list().getGroupIds()); - - if (groupIds == null) { - println("Failed to get group ID from " + peers); - return -1; - } else if (groupIds.size() == 1) { - remoteGroupId = groupIds.get(0); - } else { - println("There are more than one groups, you should specific one. " + groupIds); - return -2; - } - } - - groupInfoReply = run(peers, p -> client.getGroupManagementApi((p.getId())).info(remoteGroupId)); - processReply(groupInfoReply, - () -> "Failed to get group info for group id " + remoteGroupId.getUuid() + " from " + peers); + final List peers = CliUtils.parseRaftPeers(cl.getOptionValue(PEER_OPTION_NAME)); + final RaftGroupId groupIdSpecified = CliUtils.parseRaftGroupId(cl.getOptionValue(GROUPID_OPTION_NAME)); + raftGroup = RaftGroup.valueOf(groupIdSpecified != null? groupIdSpecified: RaftGroupId.randomId(), peers); + PrintStream printStream = getPrintStream(); + try (final RaftClient client = newRaftClient()) { + final RaftGroupId remoteGroupId = CliUtils.getGroupId(client, peers, groupIdSpecified, printStream); + groupInfoReply = CliUtils.getGroupInfo(client, peers, remoteGroupId, printStream); raftGroup = groupInfoReply.getGroup(); } return 0; } + protected RaftClient newRaftClient() { + return getContext().newRaftClient(getRaftGroup()); + } + @Override public Options getOptions() { return new Options() @@ -168,14 +120,7 @@ protected RaftPeerProto getLeader(RoleInfoProto roleInfo) { } protected void processReply(RaftClientReply reply, Supplier messageSupplier) throws IOException { - if (reply == null || !reply.isSuccess()) { - final RaftException e = Optional.ofNullable(reply) - .map(RaftClientReply::getException) - .orElseGet(() -> new RaftException("Reply: " + reply)); - final String message = messageSupplier.get(); - printf("%s. Error: %s%n", message, e); - throw new IOException(message, e); - } + CliUtils.checkReply(reply, messageSupplier, getPrintStream()); } protected List getIds(String[] optionValues, BiConsumer consumer) { @@ -184,8 +129,8 @@ protected List getIds(String[] optionValues, BiConsumer ids = new ArrayList<>(); for (String address : optionValues) { - final InetSocketAddress serverAddress = parseInetSocketAddress(address); - final RaftPeerId peerId = RaftUtils.getPeerId(serverAddress); + final InetSocketAddress serverAddress = CliUtils.parseInetSocketAddress(address); + final RaftPeerId peerId = CliUtils.getPeerId(serverAddress); consumer.accept(peerId, serverAddress); ids.add(peerId); } diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/Context.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/Context.java index bae98dc0b5..6f80256d44 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/Context.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/command/Context.java @@ -17,27 +17,59 @@ */ package org.apache.ratis.shell.cli.sh.command; +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.client.RaftClientConfigKeys; +import org.apache.ratis.conf.Parameters; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.retry.ExponentialBackoffRetry; +import org.apache.ratis.retry.RetryPolicy; import org.apache.ratis.thirdparty.com.google.common.io.Closer; +import org.apache.ratis.util.TimeDuration; import java.io.Closeable; import java.io.IOException; import java.io.PrintStream; import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.TimeUnit; /** * A context for ratis-shell. */ public final class Context implements Closeable { + private static final TimeDuration DEFAULT_REQUEST_TIMEOUT = TimeDuration.valueOf(15, TimeUnit.SECONDS); + private static final RetryPolicy DEFAULT_RETRY_POLICY = ExponentialBackoffRetry.newBuilder() + .setBaseSleepTime(TimeDuration.valueOf(1000, TimeUnit.MILLISECONDS)) + .setMaxAttempts(10) + .setMaxSleepTime(TimeDuration.valueOf(100_000, TimeUnit.MILLISECONDS)) + .build(); + private final PrintStream mPrintStream; private final Closer mCloser; + private final boolean cli; + private final RetryPolicy retryPolicy; + private final RaftProperties properties; + private final Parameters parameters; + /** * Build a context. * @param printStream the print stream */ public Context(PrintStream printStream) { + this(printStream, true, DEFAULT_RETRY_POLICY, new RaftProperties(), null); + } + + public Context(PrintStream printStream, boolean cli, RetryPolicy retryPolicy, + RaftProperties properties, Parameters parameters) { mCloser = Closer.create(); mPrintStream = mCloser.register(Objects.requireNonNull(printStream, "printStream == null")); + + this.cli = cli; + this.retryPolicy = retryPolicy != null? retryPolicy : DEFAULT_RETRY_POLICY; + this.properties = properties != null? properties : new RaftProperties(); + this.parameters = parameters; } /** @@ -47,6 +79,43 @@ public PrintStream getPrintStream() { return mPrintStream; } + /** Is this from CLI? */ + public boolean isCli() { + return cli; + } + + public RetryPolicy getRetryPolicy() { + return retryPolicy; + } + + public RaftProperties getProperties() { + return properties; + } + + public Parameters getParameters() { + return parameters; + } + + /** Create a new {@link RaftClient} from the given group. */ + public RaftClient newRaftClient(RaftGroup group) { + final RaftProperties p = getProperties(); + if (isCli()) { + RaftClientConfigKeys.Rpc.setRequestTimeout(p, DEFAULT_REQUEST_TIMEOUT); + + // Since ratis-shell support GENERIC_COMMAND_OPTIONS, here we should + // merge these options to raft p to make it work. + final Properties sys = System.getProperties(); + sys.stringPropertyNames().forEach(key -> p.set(key, sys.getProperty(key))); + } + + return RaftClient.newBuilder() + .setRaftGroup(group) + .setProperties(p) + .setParameters(getParameters()) + .setRetryPolicy(getRetryPolicy()) + .build(); + } + @Override public void close() throws IOException { mCloser.close(); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/PauseCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/PauseCommand.java index 4ea2969bac..f8a627a8fa 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/PauseCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/PauseCommand.java @@ -24,7 +24,6 @@ import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -61,7 +60,7 @@ public int run(CommandLine cl) throws IOException { printf("Peer not found: %s", strAddr); return -1; } - try(final RaftClient raftClient = RaftUtils.createClient(getRaftGroup())) { + try(final RaftClient raftClient = newRaftClient()) { RaftClientReply reply = raftClient.getLeaderElectionManagementApi(peerId).pause(); processReply(reply, () -> String.format("Failed to pause leader election on peer %s", strAddr)); printf(String.format("Successful pause leader election on peer %s", strAddr)); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/ResumeCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/ResumeCommand.java index 4b4dc225a0..1b5c80fac1 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/ResumeCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/ResumeCommand.java @@ -24,7 +24,6 @@ import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -61,7 +60,7 @@ public int run(CommandLine cl) throws IOException { printf("Can't find a sever with the address:%s", strAddr); return -1; } - try(final RaftClient raftClient = RaftUtils.createClient(getRaftGroup())) { + try(final RaftClient raftClient = newRaftClient()) { RaftClientReply reply = raftClient.getLeaderElectionManagementApi(peerId).resume(); processReply(reply, () -> String.format("Failed to resume leader election on peer %s", strAddr)); printf(String.format("Successful pause leader election on peer %s", strAddr)); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/StepDownCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/StepDownCommand.java index 911a2bb26a..f18921e5ec 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/StepDownCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/StepDownCommand.java @@ -21,7 +21,6 @@ import org.apache.ratis.client.RaftClient; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -48,7 +47,7 @@ public String getCommandName() { public int run(CommandLine cl) throws IOException { super.run(cl); - try (RaftClient client = RaftUtils.createClient(getRaftGroup())) { + try (RaftClient client = newRaftClient()) { RaftPeerId leaderId = RaftPeerId.valueOf(getLeader(getGroupInfoReply().getRoleInfoProto()).getId()); final RaftClientReply transferLeadershipReply = client.admin().transferLeadership(null, leaderId, 60_000); processReply(transferLeadershipReply, () -> "Failed to step down leader"); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/TransferCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/TransferCommand.java index c71d7f89f6..88cfec9148 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/TransferCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/election/TransferCommand.java @@ -25,7 +25,6 @@ import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.exceptions.TransferLeadershipException; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; import org.apache.ratis.util.TimeDuration; @@ -74,7 +73,7 @@ public int run(CommandLine cl) throws IOException { printf("Peer with address %s not found.", strAddr); return -2; } - try (RaftClient client = RaftUtils.createClient(getRaftGroup())) { + try (RaftClient client = newRaftClient()) { // transfer leadership if (!tryTransfer(client, newLeader, highestPriority, timeout.orElse(timeoutDefault))) { // legacy mode, transfer leadership by setting priority. diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupInfoCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupInfoCommand.java index d2c4e65991..0125440e90 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupInfoCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupInfoCommand.java @@ -54,6 +54,7 @@ public int run(CommandLine cl) throws IOException { printf("leader info: %s(%s)%n%n", leader.getId().toStringUtf8(), leader.getAddress()); } println(reply.getCommitInfos()); + println(reply.getLogInfoProto()); return 0; } diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupListCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupListCommand.java index 5bbd1939ad..214ed15078 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupListCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/group/GroupListCommand.java @@ -24,7 +24,7 @@ import org.apache.ratis.client.RaftClient; import org.apache.ratis.protocol.GroupListReply; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; +import org.apache.ratis.shell.cli.CliUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -61,15 +61,15 @@ public int run(CommandLine cl) throws IOException { address = getRaftGroup().getPeer(peerId).getAddress(); } else if (cl.hasOption(SERVER_ADDRESS_OPTION_NAME)) { address = cl.getOptionValue(SERVER_ADDRESS_OPTION_NAME); - final InetSocketAddress serverAddress = parseInetSocketAddress(address); - peerId = RaftUtils.getPeerId(serverAddress); + final InetSocketAddress serverAddress = CliUtils.parseInetSocketAddress(address); + peerId = CliUtils.getPeerId(serverAddress); } else { throw new IllegalArgumentException( "Both " + PEER_ID_OPTION_NAME + " and " + SERVER_ADDRESS_OPTION_NAME + " options are missing."); } - try(final RaftClient raftClient = RaftUtils.createClient(getRaftGroup())) { + try(final RaftClient raftClient = newRaftClient()) { GroupListReply reply = raftClient.getGroupManagementApi(peerId).list(); processReply(reply, () -> String.format("Failed to get group information of peerId %s (server %s)", peerId, address)); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java index 231c643ac3..a63b659375 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/local/RaftMetaConfCommand.java @@ -24,7 +24,8 @@ import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto; import org.apache.ratis.proto.RaftProtos.RaftPeerProto; import org.apache.ratis.proto.RaftProtos.RaftPeerRole; -import org.apache.ratis.shell.cli.RaftUtils; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.shell.cli.CliUtils; import org.apache.ratis.shell.cli.sh.command.AbstractCommand; import org.apache.ratis.shell.cli.sh.command.Context; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; @@ -32,11 +33,14 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * Command for generate a new raft-meta.conf file based on original raft-meta.conf and new peers, @@ -49,6 +53,7 @@ public class RaftMetaConfCommand extends AbstractCommand { private static final String RAFT_META_CONF = "raft-meta.conf"; private static final String NEW_RAFT_META_CONF = "new-raft-meta.conf"; + private static final String SEPARATOR = "\\|"; /** * @param context command context */ @@ -69,11 +74,49 @@ public int run(CommandLine cl) throws IOException { printf("peers or path can't be empty."); return -1; } + Set addresses = new HashSet<>(); + Set ids = new HashSet<>(); List raftPeerProtos = new ArrayList<>(); - for (String address : peersStr.split(",")) { - String peerId = RaftUtils.getPeerId(parseInetSocketAddress(address)).toString(); + for (String idWithAddress : peersStr.split(",")) { + String[] peerIdWithAddressArray = idWithAddress.split(SEPARATOR); + + if (peerIdWithAddressArray.length < 1 || peerIdWithAddressArray.length > 2) { + String message = + "Failed to parse peer's ID and address for: %s, " + + "from option: -peers %s. \n" + + "Please make sure to provide list of peers" + + " in format <[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>"; + printf(message, idWithAddress, peersStr); + return -1; + } + InetSocketAddress inetSocketAddress = CliUtils.parseInetSocketAddress( + peerIdWithAddressArray[peerIdWithAddressArray.length - 1]); + String addressString = inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort(); + if (addresses.contains(addressString)) { + printf("Found duplicated address: %s. Please make sure the address of peer have no duplicated value.", + addressString); + return -1; + } + addresses.add(addressString); + + String peerId; + if (peerIdWithAddressArray.length == 2) { + // Peer ID is provided + peerId = RaftPeerId.getRaftPeerId(peerIdWithAddressArray[0]).toString(); + + if (ids.contains(peerId)) { + printf("Found duplicated ID: %s. Please make sure the ID of peer have no duplicated value.", peerId); + return -1; + } + ids.add(peerId); + } else { + // If peer ID is not provided, use host address as peerId value + peerId = CliUtils.getPeerId(inetSocketAddress).toString(); + } + raftPeerProtos.add(RaftPeerProto.newBuilder() - .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))).setAddress(address) + .setId(ByteString.copyFrom(peerId.getBytes(StandardCharsets.UTF_8))) + .setAddress(addressString) .setStartupRole(RaftPeerRole.FOLLOWER).build()); } try (InputStream in = Files.newInputStream(Paths.get(path, RAFT_META_CONF)); @@ -93,7 +136,7 @@ public int run(CommandLine cl) throws IOException { @Override public String getUsage() { return String.format("%s" - + " -%s " + + " -%s <[P0_ID|]P0_HOST:P0_PORT,[P1_ID|]P1_HOST:P1_PORT,[P2_ID|]P2_HOST:P2_PORT>" + " -%s ", getCommandName(), PEER_OPTION_NAME, PATH_OPTION_NAME); } diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/AddCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/AddCommand.java index 3c65bb12de..be8f789954 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/AddCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/AddCommand.java @@ -25,9 +25,10 @@ import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; +import org.apache.ratis.shell.cli.CliUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; +import org.apache.ratis.util.Preconditions; import java.io.IOException; import java.net.InetSocketAddress; @@ -66,10 +67,10 @@ public int run(CommandLine cl) throws IOException { if (cl.hasOption(ADDRESS_OPTION_NAME) && cl.hasOption(PEER_ID_OPTION_NAME)) { ids = Arrays.stream(cl.getOptionValue(PEER_ID_OPTION_NAME).split(",")) .map(RaftPeerId::getRaftPeerId).collect(Collectors.toList()); - List addresses = - Arrays.stream(cl.getOptionValue(ADDRESS_OPTION_NAME).split(",")) - .map(s -> parseInetSocketAddress(s)).collect(Collectors.toList()); - assert ids.size() == addresses.size(); + final List addresses = Arrays.stream(cl.getOptionValue(ADDRESS_OPTION_NAME).split(",")) + .map(CliUtils::parseInetSocketAddress) + .collect(Collectors.toList()); + Preconditions.assertSame(ids.size(), addresses.size(), "size"); for (int i = 0; i < ids.size(); i++) { peersInfo.put(ids.get(i), addresses.get(i)); } @@ -80,7 +81,7 @@ public int run(CommandLine cl) throws IOException { "Both " + PEER_ID_OPTION_NAME + " and " + ADDRESS_OPTION_NAME + " options are missing."); } - try (RaftClient client = RaftUtils.createClient(getRaftGroup())) { + try (RaftClient client = newRaftClient()) { final Stream remaining = getPeerStream(RaftPeerRole.FOLLOWER); final Stream adding = ids.stream().map(raftPeerId -> RaftPeer.newBuilder() .setId(raftPeerId) diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/RemoveCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/RemoveCommand.java index 5918516070..904a897880 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/RemoveCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/RemoveCommand.java @@ -25,7 +25,6 @@ import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -66,7 +65,7 @@ public int run(CommandLine cl) throws IOException { throw new IllegalArgumentException( "Both " + PEER_ID_OPTION_NAME + " and " + ADDRESS_OPTION_NAME + " options are missing."); } - try (RaftClient client = RaftUtils.createClient(getRaftGroup())) { + try (RaftClient client = newRaftClient()) { final List peers = getPeerStream(RaftPeerRole.FOLLOWER) .filter(raftPeer -> !ids.contains(raftPeer.getId())).collect(Collectors.toList()); final List listeners = getPeerStream(RaftPeerRole.LISTENER) diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/SetPriorityCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/SetPriorityCommand.java index 01e81f3c34..2834ef5cc8 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/SetPriorityCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/peer/SetPriorityCommand.java @@ -24,7 +24,6 @@ import org.apache.ratis.proto.RaftProtos.RaftPeerRole; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeer; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -63,7 +62,7 @@ public int run(CommandLine cl) throws IOException { addressPriorityMap.put(str[0], Integer.parseInt(str[1])); } - try (RaftClient client = RaftUtils.createClient(getRaftGroup())) { + try (RaftClient client = newRaftClient()) { final List peers = getPeerStream(RaftPeerRole.FOLLOWER).map(peer -> { final Integer newPriority = addressPriorityMap.get(peer.getAddress()); final int priority = newPriority != null ? newPriority : peer.getPriority(); diff --git a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/snapshot/TakeSnapshotCommand.java b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/snapshot/TakeSnapshotCommand.java index 10bac34975..521b22e4e3 100644 --- a/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/snapshot/TakeSnapshotCommand.java +++ b/ratis-shell/src/main/java/org/apache/ratis/shell/cli/sh/snapshot/TakeSnapshotCommand.java @@ -23,7 +23,6 @@ import org.apache.ratis.client.RaftClient; import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.protocol.RaftPeerId; -import org.apache.ratis.shell.cli.RaftUtils; import org.apache.ratis.shell.cli.sh.command.AbstractRatisCommand; import org.apache.ratis.shell.cli.sh.command.Context; @@ -58,7 +57,7 @@ public int run(CommandLine cl) throws IOException { } else { timeout = 3000; } - try(final RaftClient raftClient = RaftUtils.createClient(getRaftGroup())) { + try(final RaftClient raftClient = newRaftClient()) { if (cl.hasOption(PEER_ID_OPTION_NAME)) { peerId = RaftPeerId.getRaftPeerId(cl.getOptionValue(PEER_ID_OPTION_NAME)); } else { diff --git a/ratis-test/pom.xml b/ratis-test/pom.xml index 40b6b2ec27..8a3d0f1cc5 100644 --- a/ratis-test/pom.xml +++ b/ratis-test/pom.xml @@ -17,14 +17,14 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-test Apache Ratis Test - 1.76 + 1.78.1 @@ -136,6 +136,21 @@ junit test + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-params + test + org.mockito mockito-core diff --git a/ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java b/ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java new file mode 100644 index 0000000000..9ea78e47cd --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/RaftLogTruncateTests.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis; + +import org.apache.ratis.RaftTestUtil.SimpleMessage; +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.client.RaftClientConfigKeys; +import org.apache.ratis.client.impl.OrderedAsync; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.server.RaftServer; +import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.impl.MiniRaftCluster; +import org.apache.ratis.server.impl.RaftServerTestUtil; +import org.apache.ratis.server.protocol.TermIndex; +import org.apache.ratis.server.raftlog.RaftLog; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLog; +import org.apache.ratis.statemachine.StateMachine; +import org.apache.ratis.statemachine.TransactionContext; +import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; +import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.MemoizedSupplier; +import org.apache.ratis.util.Slf4jUtils; +import org.apache.ratis.util.TimeDuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.event.Level; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.apache.ratis.RaftTestUtil.waitForLeader; + +@Timeout(value = 200) +public abstract class RaftLogTruncateTests extends BaseTest + implements MiniRaftCluster.Factory.Get { + public static final int NUM_SERVERS = 5; + final TimeDuration MIN_TIMEOUT = TimeDuration.valueOf(3, TimeUnit.SECONDS); + + static SimpleMessage[] arraycopy(SimpleMessage[] src1, SimpleMessage[] src2) { + final SimpleMessage[] dst = new SimpleMessage[src1.length + src2.length]; + System.arraycopy(src1, 0, dst, 0, src1.length); + System.arraycopy(src2, 0, dst, src1.length, src2.length); + return dst; + } + + { + Slf4jUtils.setLogLevel(RaftServerTestUtil.getTransactionContextLog(), Level.TRACE); + + Slf4jUtils.setLogLevel(OrderedAsync.LOG, Level.ERROR); + Slf4jUtils.setLogLevel(RaftServerConfigKeys.LOG, Level.ERROR); + Slf4jUtils.setLogLevel(RaftClientConfigKeys.LOG, Level.ERROR); + Slf4jUtils.setLogLevel(RaftClientConfigKeys.LOG, Level.ERROR); + + final RaftProperties p = getProperties(); + p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class); + + // set a long rpc timeout so, when the leader does not have the majority, it won't step down fast. + RaftServerConfigKeys.Rpc.setTimeoutMin(p, MIN_TIMEOUT); + RaftServerConfigKeys.Rpc.setTimeoutMax(p, MIN_TIMEOUT.multiply(2)); + RaftServerConfigKeys.Rpc.setFirstElectionTimeoutMin(p, TimeDuration.ONE_SECOND); + RaftServerConfigKeys.Rpc.setFirstElectionTimeoutMax(p, TimeDuration.ONE_SECOND.multiply(2)); + } + + @Test + public void testLogTruncate() throws Exception { + runWithNewCluster(NUM_SERVERS, this::runTestLogTruncate); + } + + void runTestLogTruncate(MiniRaftCluster cluster) throws Exception { + final RaftServer.Division oldLeader = waitForLeader(cluster); + final List oldFollowers = cluster.getFollowers(); + final List killedPeers = new ArrayList<>(); + final List remainingPeers = new ArrayList<>(); + + final int majorityIndex = NUM_SERVERS / 2 + 1; + Assertions.assertEquals(NUM_SERVERS - 1, oldFollowers.size()); + Assertions.assertTrue(majorityIndex < oldFollowers.size()); + + for (int i = 0; i < majorityIndex; i++) { + killedPeers.add(oldFollowers.get(i).getId()); + } + remainingPeers.add(oldLeader.getId()); + for (int i = majorityIndex; i < oldFollowers.size(); i++) { + remainingPeers.add(oldFollowers.get(i).getId()); + } + + try { + runTestLogTruncate(cluster, oldLeader, killedPeers, remainingPeers); + } catch (Throwable e) { + LOG.info("killedPeers : {}", killedPeers); + LOG.info("remainingPeers: {}", remainingPeers); + throw e; + } + } + + void runTestLogTruncate(MiniRaftCluster cluster, RaftServer.Division oldLeader, + List killedPeers, List remainingPeers) throws Exception { + final List exceptions = Collections.synchronizedList(new ArrayList<>()); + final long oldLeaderTerm = oldLeader.getInfo().getCurrentTerm(); + LOG.info("oldLeader: {}, term={}", oldLeader.getId(), oldLeaderTerm); + LOG.info("killedPeers : {}", killedPeers); + LOG.info("remainingPeers: {}", remainingPeers); + + final SimpleMessage[] firstBatch = SimpleMessage.create(5, "first"); + final SimpleMessage[] secondBatch = SimpleMessage.create(4, "second"); + + for (RaftPeer peer : cluster.getGroup().getPeers()) { + assertEmptyTransactionContextMap(cluster.getDivision(peer.getId())); + } + + try (final RaftClient client = cluster.createClient(oldLeader.getId())) { + // send some messages + for (SimpleMessage batch : firstBatch) { + final RaftClientReply reply = client.io().send(batch); + Assertions.assertTrue(reply.isSuccess()); + } + for (RaftPeer peer : cluster.getGroup().getPeers()) { + final RaftServer.Division division = cluster.getDivision(peer.getId()); + assertLogEntries(division, oldLeaderTerm, firstBatch); + assertEmptyTransactionContextMap(division); + } + + // kill a majority of followers + LOG.info("Before killServer {}: {}", killedPeers, cluster.printServers()); + for (RaftPeerId f : killedPeers) { + cluster.killServer(f); + } + LOG.info("After killServer {}: {}", killedPeers, cluster.printServers()); + + // send more messages, but they won't be committed due to not enough followers + final SimpleMessage[] messagesToBeTruncated = SimpleMessage.create(3, "messagesToBeTruncated"); + final AtomicBoolean done = new AtomicBoolean(); + for (SimpleMessage message : messagesToBeTruncated) { + client.async().send(message).whenComplete((r, e) -> { + if (!done.get()) { + exceptions.add(new IllegalStateException(message + " is completed: reply=" + r, e)); + } + }); + } + + // check log messages + final SimpleMessage[] expectedMessages = arraycopy(firstBatch, messagesToBeTruncated); + for (RaftPeerId f : remainingPeers) { + final RaftServer.Division division = cluster.getDivision(f); + assertLogEntries(division, oldLeaderTerm, expectedMessages); + if (!division.getId().equals(oldLeader.getId())) { + assertEntriesInTransactionContextMap(division, messagesToBeTruncated, firstBatch); + } + } + done.set(true); + LOG.info("done"); + } + + // kill remaining peers + LOG.info("Before killServer {}: {}", remainingPeers, cluster.printServers()); + for (RaftPeerId p : remainingPeers) { + cluster.killServer(p); + } + LOG.info("After killServer {}: {}", remainingPeers, cluster.printServers()); + + // restart the earlier followers + for (RaftPeerId f : killedPeers) { + cluster.restartServer(f, false); + } + + // The new leader should be one of the earlier followers + final RaftServer.Division newLeader = waitForLeader(cluster); + LOG.info("After restartServer {}: {}", killedPeers, cluster.printServers()); + final long newLeaderTerm = newLeader.getInfo().getCurrentTerm(); + + final SegmentedRaftLog newLeaderLog = (SegmentedRaftLog) newLeader.getRaftLog(); + LOG.info("newLeader: {}, term {}, last={}", newLeader.getId(), newLeaderTerm, + newLeaderLog.getLastEntryTermIndex()); + Assertions.assertTrue(killedPeers.contains(newLeader.getId())); + + // restart remaining peers + for (RaftPeerId p : remainingPeers) { + cluster.restartServer(p, false); + } + + // check RaftLog truncate + for (RaftPeerId f : remainingPeers) { + assertLogEntries(cluster.getDivision(f), oldLeaderTerm, firstBatch); + } + + try (final RaftClient client = cluster.createClient(newLeader.getId())) { + // send more messages + for (SimpleMessage batch : secondBatch) { + final RaftClientReply reply = client.io().send(batch); + Assertions.assertTrue(reply.isSuccess()); + } + } + + // check log messages -- it should be truncated and then append the new messages + final SimpleMessage[] expectedMessages = arraycopy(firstBatch, secondBatch); + for (RaftPeer peer : cluster.getGroup().getPeers()) { + final RaftServer.Division division = cluster.getDivision(peer.getId()); + assertLogEntries(division, oldLeaderTerm, expectedMessages); + final String name = "assertEmptyTransactionContextMap:" + division.getId(); + JavaUtils.attempt(() -> assertEmptyTransactionContextMap(division), + 10, HUNDRED_MILLIS, name, LOG); + + } + + if (!exceptions.isEmpty()) { + LOG.info("{} exceptions", exceptions.size()); + for(int i = 0 ; i < exceptions.size(); i++) { + LOG.info("exception {})", i, exceptions.get(i)); + } + Assertions.fail(); + } + } + + static void assertEmptyTransactionContextMap(RaftServer.Division d) { + final Map> map = RaftServerTestUtil.getTransactionContextMap(d); + Assertions.assertTrue(map.isEmpty(), () -> d.getId() + " TransactionContextMap is non-empty: " + map); + } + + static void assertEntriesInTransactionContextMap(RaftServer.Division division, + SimpleMessage[] existing, SimpleMessage[] nonExisting) { + final RaftLog log = division.getRaftLog(); + assertEntriesInTransactionContextMap(division, + RaftTestUtil.getStateMachineLogEntries(log, existing).values(), + RaftTestUtil.getStateMachineLogEntries(log, nonExisting).values()); + } + + static void assertEntriesInTransactionContextMap(RaftServer.Division division, + Collection existing, Collection nonExisting) { + final Map> map + = RaftServerTestUtil.getTransactionContextMap(division); + for(LogEntryProto e : existing) { + final TermIndex termIndex = TermIndex.valueOf(e); + Assertions.assertTrue(map.containsKey(termIndex), + () -> termIndex + " not found in " + division.getId()); + } + for(LogEntryProto e : nonExisting) { + final TermIndex termIndex = TermIndex.valueOf(e); + Assertions.assertFalse(map.containsKey(termIndex), + () -> termIndex + " found in " + division.getId()); + } + } + + private void assertLogEntries(RaftServer.Division server, long term, SimpleMessage[] expectedMessages) + throws Exception { + RaftTestUtil.assertLogEntries(server, term, expectedMessages, 30, LOG); + } +} diff --git a/ratis-test/src/test/java/org/apache/ratis/TestRaftServerNoLeaderTimeout.java b/ratis-test/src/test/java/org/apache/ratis/TestRaftServerNoLeaderTimeout.java index 7f4da0e0fb..61bacc3cf9 100644 --- a/ratis-test/src/test/java/org/apache/ratis/TestRaftServerNoLeaderTimeout.java +++ b/ratis-test/src/test/java/org/apache/ratis/TestRaftServerNoLeaderTimeout.java @@ -28,10 +28,10 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.io.IOException; @@ -48,25 +48,25 @@ public class TestRaftServerNoLeaderTimeout extends BaseTest { private static final int NUM_SERVERS = 3; - private static final RaftProperties properties = new RaftProperties(); + private static final RaftProperties PROPERTIES = new RaftProperties(); private final MiniRaftClusterWithSimulatedRpc cluster = MiniRaftClusterWithSimulatedRpc .FACTORY.newCluster(NUM_SERVERS, getProperties()); private static RaftProperties getProperties() { - RaftServerConfigKeys.Notification.setNoLeaderTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS)); - properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, + RaftServerConfigKeys.Notification.setNoLeaderTimeout(PROPERTIES, TimeDuration.valueOf(1, TimeUnit.SECONDS)); + PROPERTIES.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class); - return properties; + return PROPERTIES; } - @Before + @BeforeEach public void setup() throws IOException { - Assert.assertNull(cluster.getLeader()); + Assertions.assertNull(cluster.getLeader()); cluster.start(); } - @After + @AfterEach public void tearDown() { if (cluster != null) { cluster.shutdown(); @@ -91,10 +91,10 @@ public void testLeaderElectionDetection() throws Exception { RaftProtos.RoleInfoProto roleInfoProto = SimpleStateMachine4Testing.get(healthyFollower).getLeaderElectionTimeoutInfo(); - Assert.assertNotNull(roleInfoProto); + Assertions.assertNotNull(roleInfoProto); - Assert.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE); + Assertions.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE); final long noLeaderTimeoutMs = noLeaderTimeout.toLong(TimeUnit.MILLISECONDS); - Assert.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > noLeaderTimeoutMs); + Assertions.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > noLeaderTimeoutMs); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/TestRaftServerSlownessDetection.java b/ratis-test/src/test/java/org/apache/ratis/TestRaftServerSlownessDetection.java index 0ea0fe9cef..fd4aa1e068 100644 --- a/ratis-test/src/test/java/org/apache/ratis/TestRaftServerSlownessDetection.java +++ b/ratis-test/src/test/java/org/apache/ratis/TestRaftServerSlownessDetection.java @@ -30,12 +30,6 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - import java.io.IOException; import java.util.List; @@ -43,13 +37,18 @@ import java.util.concurrent.TimeUnit; import org.apache.ratis.thirdparty.com.codahale.metrics.Gauge; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; /** * Test Raft Server Slowness detection and notification to Leader's statemachine. */ //TODO: fix StateMachine.notifySlowness(..); see RATIS-370 -@Ignore +@Disabled public class TestRaftServerSlownessDetection extends BaseTest { static { Slf4jUtils.setLogLevel(RaftServer.Division.LOG, Level.DEBUG); @@ -57,26 +56,26 @@ public class TestRaftServerSlownessDetection extends BaseTest { public static final int NUM_SERVERS = 3; - protected static final RaftProperties properties = new RaftProperties(); + protected static final RaftProperties PROPERTIES = new RaftProperties(); private final MiniRaftClusterWithSimulatedRpc cluster = MiniRaftClusterWithSimulatedRpc .FACTORY.newCluster(NUM_SERVERS, getProperties()); public RaftProperties getProperties() { RaftServerConfigKeys.Rpc - .setSlownessTimeout(properties, TimeDuration.valueOf(1, TimeUnit.SECONDS)); - properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, + .setSlownessTimeout(PROPERTIES, TimeDuration.valueOf(1, TimeUnit.SECONDS)); + PROPERTIES.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class); - return properties; + return PROPERTIES; } - @Before + @BeforeEach public void setup() throws IOException { - Assert.assertNull(cluster.getLeader()); + Assertions.assertNull(cluster.getLeader()); cluster.start(); } - @After + @AfterEach public void tearDown() { if (cluster != null) { cluster.shutdown(); @@ -107,23 +106,23 @@ public void testSlownessDetection() throws Exception { Thread.sleep( slownessTimeout * 2); long followerHeartBeatElapsedMetricNew = (long) metric.getValue(); - Assert.assertTrue(followerHeartBeatElapsedMetricNew > followerHeartBeatElapsedMetric); + Assertions.assertTrue(followerHeartBeatElapsedMetricNew > followerHeartBeatElapsedMetric); // Followers should not get any failed not notification for (RaftServer.Division followerServer : cluster.getFollowers()) { - Assert.assertNull(SimpleStateMachine4Testing.get(followerServer).getSlownessInfo()); + Assertions.assertNull(SimpleStateMachine4Testing.get(followerServer).getSlownessInfo()); } // the leader should get notification that the follower has failed now RaftProtos.RoleInfoProto roleInfoProto = SimpleStateMachine4Testing.get(cluster.getLeader()).getSlownessInfo(); - Assert.assertNotNull(roleInfoProto); + Assertions.assertNotNull(roleInfoProto); List followers = roleInfoProto.getLeaderInfo().getFollowerInfoList(); //Assert that the node shutdown is lagging behind for (RaftProtos.ServerRpcProto serverProto : followers) { if (RaftPeerId.valueOf(serverProto.getId().getId()).equals(failedFollower.getId())) { - Assert.assertTrue(serverProto.getLastRpcElapsedTimeMs() > slownessTimeout); + Assertions.assertTrue(serverProto.getLastRpcElapsedTimeMs() > slownessTimeout); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/client/TestClientProtoUtils.java b/ratis-test/src/test/java/org/apache/ratis/client/TestClientProtoUtils.java index 9d85320539..56bf94a84e 100644 --- a/ratis-test/src/test/java/org/apache/ratis/client/TestClientProtoUtils.java +++ b/ratis-test/src/test/java/org/apache/ratis/client/TestClientProtoUtils.java @@ -29,8 +29,8 @@ import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.concurrent.TimeUnit; @@ -71,7 +71,7 @@ void runTestToRaftClientRequestProto(int n, SizeInBytes messageSize) final RaftClientRequest computed = ClientProtoUtils.toRaftClientRequest(proto); final TimeDuration r = startTime.elapsedTime().subtract(p); - Assert.assertEquals(request.getMessage().getContent(), computed.getMessage().getContent()); + Assertions.assertEquals(request.getMessage().getContent(), computed.getMessage().getContent()); toProto = toProto.add(p); toRequest = toRequest.add(r); diff --git a/ratis-test/src/test/java/org/apache/ratis/conf/TestConfUtils.java b/ratis-test/src/test/java/org/apache/ratis/conf/TestConfUtils.java index 1600da18dc..859c597b52 100644 --- a/ratis-test/src/test/java/org/apache/ratis/conf/TestConfUtils.java +++ b/ratis-test/src/test/java/org/apache/ratis/conf/TestConfUtils.java @@ -22,10 +22,42 @@ import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.grpc.GrpcConfigKeys; import org.apache.ratis.netty.NettyConfigKeys; +import org.apache.ratis.rpc.RpcType; import org.apache.ratis.server.RaftServerConfigKeys; -import org.junit.Test; +import org.apache.ratis.server.simulation.SimulatedRpc; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; public class TestConfUtils extends BaseTest { + @Test + public void testLogging() { + final AtomicInteger count = new AtomicInteger(); + final Consumer logger = s -> { + System.out.println("log: " + s); + count.incrementAndGet(); + }; + + final RaftProperties properties = new RaftProperties(); + final RpcType simulated = SimulatedRpc.get(); + + // get a value the first time + final RpcType defaultType = RaftConfigKeys.Rpc.type(properties, logger); + Assertions.assertEquals(1, count.get()); + Assertions.assertNotEquals(defaultType, simulated); + + // get the same value the second time + RaftConfigKeys.Rpc.type(properties, logger); + Assertions.assertEquals(1, count.get()); + + // get a different value + RaftConfigKeys.Rpc.setType(properties, SimulatedRpc.get()); + RaftConfigKeys.Rpc.type(properties, logger); + Assertions.assertEquals(2, count.get()); + } + @Test public void testRaftConfigKeys() { ConfUtils.printAll(RaftConfigKeys.class); diff --git a/ratis-test/src/test/java/org/apache/ratis/conf/TestRaftProperties.java b/ratis-test/src/test/java/org/apache/ratis/conf/TestRaftProperties.java index c65d00f43e..d17d6fb6f6 100644 --- a/ratis-test/src/test/java/org/apache/ratis/conf/TestRaftProperties.java +++ b/ratis-test/src/test/java/org/apache/ratis/conf/TestRaftProperties.java @@ -17,8 +17,10 @@ */ package org.apache.ratis.conf; -import org.junit.Assert; -import org.junit.Test; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; public class TestRaftProperties { enum Type {APPEND_ENTRIES} @@ -29,71 +31,72 @@ static class Request_Vote { static final String KEY = "key"; static void setUnderscoreValue(RaftProperties p, String valueWithUnderscore) { - Assert.assertTrue(valueWithUnderscore.contains("_")); + Assertions.assertTrue(valueWithUnderscore.contains("_")); p.set(KEY, valueWithUnderscore); } static void setNonUnderscoreValue(RaftProperties p, String valueWithoutUnderscore) { - Assert.assertFalse(valueWithoutUnderscore.contains("_")); + Assertions.assertFalse(valueWithoutUnderscore.contains("_")); p.set(KEY, valueWithoutUnderscore); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testUnderscore() { final RaftProperties p = new RaftProperties(); { // boolean - Assert.assertNull(p.getBoolean(KEY, null)); + Assertions.assertNull(p.getBoolean(KEY, null)); setNonUnderscoreValue(p, "true"); - Assert.assertTrue(p.getBoolean(KEY, null)); + Assertions.assertTrue(p.getBoolean(KEY, null)); setNonUnderscoreValue(p, "false"); - Assert.assertFalse(p.getBoolean(KEY, null)); + Assertions.assertFalse(p.getBoolean(KEY, null)); setUnderscoreValue(p, "fa_lse"); - Assert.assertNull(p.getBoolean(KEY, null)); + Assertions.assertNull(p.getBoolean(KEY, null)); p.unset(KEY); } { //int final Integer expected = 1000000; - Assert.assertNull(p.getInt(KEY, null)); + Assertions.assertNull(p.getInt(KEY, null)); setUnderscoreValue(p, "1_000_000"); - Assert.assertEquals(expected, p.getInt(KEY, null)); + Assertions.assertEquals(expected, p.getInt(KEY, null)); setNonUnderscoreValue(p, "1000000"); - Assert.assertEquals(expected, p.getInt(KEY, null)); + Assertions.assertEquals(expected, p.getInt(KEY, null)); p.unset(KEY); } { // long final Long expected = 1_000_000_000_000L; - Assert.assertNull(p.getLong(KEY, null)); + Assertions.assertNull(p.getLong(KEY, null)); setUnderscoreValue(p, "1_000_000_000_000"); - Assert.assertEquals(expected, p.getLong(KEY, null)); + Assertions.assertEquals(expected, p.getLong(KEY, null)); setNonUnderscoreValue(p, "1000000000000"); - Assert.assertEquals(expected, p.getLong(KEY, null)); + Assertions.assertEquals(expected, p.getLong(KEY, null)); p.unset(KEY); } { // File final String expected = "1_000_000"; - Assert.assertNull(p.getFile(KEY, null)); + Assertions.assertNull(p.getFile(KEY, null)); setUnderscoreValue(p, expected); - Assert.assertEquals(expected, p.getFile(KEY, null).getName()); + Assertions.assertEquals(expected, p.getFile(KEY, null).getName()); p.unset(KEY); } { // class final Type expected = Type.APPEND_ENTRIES; - Assert.assertNull(p.getEnum(KEY, Type.class, null)); + Assertions.assertNull(p.getEnum(KEY, Type.class, null)); setUnderscoreValue(p, expected.name()); - Assert.assertEquals(expected, p.getEnum(KEY, Type.class, null)); + Assertions.assertEquals(expected, p.getEnum(KEY, Type.class, null)); p.unset(KEY); } { // enum final Class expected = Request_Vote.class; - Assert.assertNull(p.getClass(KEY, null)); + Assertions.assertNull(p.getClass(KEY, null)); setUnderscoreValue(p, expected.getName()); - Assert.assertEquals(expected, p.getClass(KEY, null)); + Assertions.assertEquals(expected, p.getClass(KEY, null)); p.unset(KEY); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamAsyncClusterTests.java b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamAsyncClusterTests.java index 8c315070e5..eb25a369e6 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamAsyncClusterTests.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamAsyncClusterTests.java @@ -34,8 +34,9 @@ import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.function.CheckedBiFunction; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.event.Level; import java.io.IOException; @@ -47,15 +48,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +@Timeout(value = 300) public abstract class DataStreamAsyncClusterTests extends DataStreamClusterTests { final Executor executor = Executors.newFixedThreadPool(16); - @Override - public int getGlobalTimeoutSeconds() { - return 300; - } - @Test public void testSingleStreamsMultipleServers() throws Exception { Slf4jUtils.setLogLevel(NettyClientStreamRpc.LOG, Level.TRACE); @@ -103,15 +100,18 @@ void runTestDataStream(CLUSTER cluster) throws Exception { long runMultipleStreams(CLUSTER cluster, boolean stepDownLeader) { final List> futures = new ArrayList<>(); - futures.add(CompletableFuture.supplyAsync(() -> runTestDataStream(cluster, 5, 10, 100_000, 10, stepDownLeader), executor)); - futures.add(CompletableFuture.supplyAsync(() -> runTestDataStream(cluster, 2, 20, 1_000, 5_000, stepDownLeader), executor)); + futures.add(CompletableFuture.supplyAsync(() -> + runTestDataStream(cluster, 5, 10, 100_000, 10, stepDownLeader), executor)); + futures.add(CompletableFuture.supplyAsync(() -> + runTestDataStream(cluster, 2, 20, 1_000, 5_000, stepDownLeader), executor)); return futures.stream() .map(CompletableFuture::join) .max(Long::compareTo) .orElseThrow(IllegalStateException::new); } - void runTestDataStream(CLUSTER cluster, boolean stepDownLeader, CheckedBiFunction runMethod) throws Exception { + void runTestDataStream(CLUSTER cluster, boolean stepDownLeader, + CheckedBiFunction runMethod) throws Exception { RaftTestUtil.waitForLeader(cluster); final long maxIndex = runMethod.apply(cluster, stepDownLeader); @@ -130,14 +130,14 @@ void runTestDataStream(CLUSTER cluster, boolean stepDownLeader, CheckedBiFunctio // wait for all servers to catch up try (RaftClient client = cluster.createClient()) { RaftClientReply reply = client.async().watch(maxIndex, ReplicationLevel.ALL).join(); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } // assert all streams are linked for (RaftServer proxy : cluster.getServers()) { final RaftServer.Division impl = proxy.getDivision(cluster.getGroupId()); final MultiDataStreamStateMachine stateMachine = (MultiDataStreamStateMachine) impl.getStateMachine(); for (SingleDataStream s : stateMachine.getStreams()) { - Assert.assertFalse(s.getDataChannel().isOpen()); + Assertions.assertFalse(s.getDataChannel().isOpen()); DataStreamTestUtils.assertLogEntry(impl, s); } } @@ -150,7 +150,7 @@ Long runTestDataStream( futures.add(CompletableFuture.supplyAsync( () -> runTestDataStream(cluster, numStreams, bufferSize, bufferNum, stepDownLeader), executor)); } - Assert.assertEquals(numClients, futures.size()); + Assertions.assertEquals(numClients, futures.size()); return futures.stream() .map(CompletableFuture::join) .max(Long::compareTo) @@ -174,7 +174,7 @@ long runTestDataStream(CLUSTER cluster, int numStreams, int bufferSize, int buff futures.add(CompletableFuture.supplyAsync(() -> DataStreamTestUtils.writeAndCloseAndAssertReplies( servers, leader, out, bufferSize, bufferNum, client.getId(), stepDownLeader).join(), executor)); } - Assert.assertEquals(numStreams, futures.size()); + Assertions.assertEquals(numStreams, futures.size()); return futures.stream() .map(CompletableFuture::join) .map(RaftClientReply::getLogIndex) diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamBaseTest.java b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamBaseTest.java index 3f241a8b9c..ee56f334e1 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamBaseTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamBaseTest.java @@ -34,7 +34,7 @@ import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.impl.RaftServerTestUtil; import org.apache.ratis.util.CollectionUtils; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import java.io.IOException; import java.util.ArrayList; @@ -44,8 +44,8 @@ abstract class DataStreamBaseTest extends BaseTest { RaftConfiguration getRaftConf() { - final List peers = servers.stream().map(Server::getPeer).collect(Collectors.toList()); - return RaftServerTestUtil.newRaftConfiguration(peers); + final List peerList = servers.stream().map(Server::getPeer).collect(Collectors.toList()); + return RaftServerTestUtil.newRaftConfiguration(peerList); } static class Server { @@ -67,7 +67,7 @@ RaftServer getRaftServer() { return raftServer; } - void start() { + void start() throws IOException { dataStreamServer.getServerRpc().start(); } @@ -90,21 +90,21 @@ Server getPrimaryServer() { return servers.get(0); } - void setup(RaftGroupId groupId, List peers, List raftServers) { - raftGroup = RaftGroup.valueOf(groupId, peers); - this.peers = peers; - servers = new ArrayList<>(peers.size()); + void setup(RaftGroupId groupId, List peerList, List raftServers) throws Exception { + raftGroup = RaftGroup.valueOf(groupId, peerList); + this.peers = peerList; + servers = new ArrayList<>(peerList.size()); // start stream servers on raft peers. - for (int i = 0; i < peers.size(); i++) { - final Server server = new Server(peers.get(i), raftServers.get(i)); - server.addRaftPeers(removePeerFromList(peers.get(i), peers)); + for (int i = 0; i < peerList.size(); i++) { + final Server server = new Server(peerList.get(i), raftServers.get(i)); + server.addRaftPeers(removePeerFromList(peerList.get(i), peerList)); server.start(); servers.add(server); } } - private Collection removePeerFromList(RaftPeer peer, List peers) { - List otherPeers = new ArrayList<>(peers); + private Collection removePeerFromList(RaftPeer peer, List peerList) { + List otherPeers = new ArrayList<>(peerList); otherPeers.remove(peer); return otherPeers; } @@ -132,10 +132,10 @@ void runTestMockCluster(ClientId clientId, int bufferSize, int bufferNum, .stream(null, DataStreamTestUtils.getRoutingTableChainTopology(peers, getPrimaryServer().getPeer())); if (headerException != null) { final DataStreamReply headerReply = out.getHeaderFuture().join(); - Assert.assertFalse(headerReply.isSuccess()); + Assertions.assertFalse(headerReply.isSuccess()); final RaftClientReply clientReply = ClientProtoUtils.toRaftClientReply( ((DataStreamReplyByteBuffer)headerReply).slice()); - Assert.assertTrue(clientReply.getException().getMessage().contains(headerException.getMessage())); + Assertions.assertTrue(clientReply.getException().getMessage().contains(headerException.getMessage())); return; } @@ -143,11 +143,11 @@ void runTestMockCluster(ClientId clientId, int bufferSize, int bufferNum, CollectionUtils.as(servers, Server::getRaftServer), null, out, bufferSize, bufferNum, client.getId(), false).join(); if (expectedException != null) { - Assert.assertFalse(clientReply.isSuccess()); - Assert.assertTrue(clientReply.getException().getMessage().contains( + Assertions.assertFalse(clientReply.isSuccess()); + Assertions.assertTrue(clientReply.getException().getMessage().contains( expectedException.getMessage())); } else { - Assert.assertTrue(clientReply.isSuccess()); + Assertions.assertTrue(clientReply.isSuccess()); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamClusterTests.java b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamClusterTests.java index 352d98e650..dabc93dda2 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamClusterTests.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamClusterTests.java @@ -36,8 +36,8 @@ import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.Timestamp; import org.apache.ratis.util.function.CheckedConsumer; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.File; import java.nio.channels.FileChannel; @@ -52,6 +52,7 @@ public abstract class DataStreamClusterTests ex implements MiniRaftCluster.Factory.Get { { setStateMachine(MultiDataStreamStateMachine.class); + DataStreamTestUtils.enableResourceLeakDetector(); } public static final int NUM_SERVERS = 3; @@ -115,9 +116,9 @@ void runTestInvalidPrimaryInRoutingTable(CLUSTER cluster) throws Exception { } } - Assert.assertNotNull( - "Cannot find peer other than the primary", notPrimary); - Assert.assertNotEquals(primaryServer, notPrimary); + Assertions.assertNotNull(notPrimary, + "Cannot find peer other than the primary"); + Assertions.assertNotEquals(primaryServer, notPrimary); try (RaftClient client = cluster.createClient(primaryServer)) { RoutingTable routingTableWithWrongPrimary = @@ -156,7 +157,7 @@ static CheckedConsumer transferToWritableByteCh public void accept(DataStreamOutputImpl out) throws Exception { try (FileChannel in = FileUtils.newFileChannel(f, StandardOpenOption.READ)) { final long transferred = in.transferTo(0, size, out.getWritableByteChannel()); - Assert.assertEquals(size, transferred); + Assertions.assertEquals(size, transferred); } } @@ -196,7 +197,7 @@ void assertLogEntry(CLUSTER cluster, RaftClientRequest request) throws Exception final RaftServer.Division impl = proxy.getDivision(cluster.getGroupId()); final MultiDataStreamStateMachine stateMachine = (MultiDataStreamStateMachine) impl.getStateMachine(); final SingleDataStream s = stateMachine.getSingleDataStream(request); - Assert.assertFalse(s.getDataChannel().isOpen()); + Assertions.assertFalse(s.getDataChannel().isOpen()); DataStreamTestUtils.assertLogEntry(impl, s); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTestUtils.java b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTestUtils.java index 738cb0359c..4909f436d1 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTestUtils.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/DataStreamTestUtils.java @@ -46,10 +46,13 @@ import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.statemachine.impl.BaseStateMachine; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.thirdparty.io.netty.buffer.ByteBufUtil; +import org.apache.ratis.thirdparty.io.netty.util.ResourceLeakDetector; +import org.apache.ratis.thirdparty.io.netty.util.ResourceLeakDetector.Level; import org.apache.ratis.util.CollectionUtils; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.JavaUtils; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +72,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ThreadLocalRandom; +import static org.apache.ratis.server.storage.RaftStorageTestUtils.getLogUnsafe; + public interface DataStreamTestUtils { Logger LOG = LoggerFactory.getLogger(DataStreamTestUtils.class); @@ -87,7 +92,7 @@ static ByteBuffer initBuffer(int offset, int size) { buffer.put(pos2byte(offset + j)); } buffer.flip(); - Assert.assertEquals(length, buffer.remaining()); + Assertions.assertEquals(length, buffer.remaining()); return buffer; } @@ -117,7 +122,7 @@ public int read(ByteBuffer dst) { FileUtils.createDirectories(f.getParentFile()); try(FileChannel out = FileUtils.newFileChannel(f, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { final long transferred = out.transferFrom(source, 0, size); - Assert.assertEquals(size, transferred); + Assertions.assertEquals(size, transferred); } } @@ -149,6 +154,7 @@ class MultiDataStreamStateMachine extends BaseStateMachine { @Override public CompletableFuture stream(RaftClientRequest request) { final SingleDataStream s = new SingleDataStream(request); + LOG.info("XXX {} put {}, {}", this, ClientInvocationId.valueOf(request), s); streams.put(ClientInvocationId.valueOf(request), s); return CompletableFuture.completedFuture(s); } @@ -165,7 +171,7 @@ public CompletableFuture link(DataStream stream, LogEntryProto entry) { @Override public CompletableFuture applyTransaction(TransactionContext trx) { - final LogEntryProto entry = Objects.requireNonNull(trx.getLogEntry()); + final LogEntryProto entry = Objects.requireNonNull(trx.getLogEntryUnsafe()); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); final SingleDataStream s = getSingleDataStream(ClientInvocationId.valueOf(entry.getStateMachineLogEntry())); final ByteString bytesWritten = bytesWritten2ByteString(s.getDataChannel().getBytesWritten()); @@ -177,7 +183,9 @@ SingleDataStream getSingleDataStream(RaftClientRequest request) { } SingleDataStream getSingleDataStream(ClientInvocationId invocationId) { - return streams.get(invocationId); + final SingleDataStream s = streams.get(invocationId); + LOG.info("XXX {}: get {} return {}", this, invocationId, s); + return s; } Collection getStreams() { @@ -253,7 +261,7 @@ public int write(ByteBuffer src) { } final int remaining = src.remaining(); for (; src.remaining() > 0; ) { - Assert.assertEquals(pos2byte(bytesWritten), src.get()); + Assertions.assertEquals(pos2byte(bytesWritten), src.get()); bytesWritten += 1; } return remaining; @@ -302,9 +310,9 @@ static int writeAndAssertReplies(DataStreamOutputImpl out, int bufferSize, int b } static void assertSuccessReply(Type expectedType, long expectedBytesWritten, DataStreamReply reply) { - Assert.assertTrue(reply.isSuccess()); - Assert.assertEquals(expectedBytesWritten, reply.getBytesWritten()); - Assert.assertEquals(expectedType, reply.getType()); + Assertions.assertTrue(reply.isSuccess()); + Assertions.assertEquals(expectedBytesWritten, reply.getBytesWritten()); + Assertions.assertEquals(expectedType, reply.getType()); } static CompletableFuture writeAndCloseAndAssertReplies( @@ -327,27 +335,30 @@ static CompletableFuture writeAndCloseAndAssertReplies( static void assertHeader(RaftServer server, RaftClientRequest header, int dataSize, boolean stepDownLeader) throws Exception { + LOG.info("XXX {}: dataSize={}, stepDownLeader={}, header={}", + server.getId(), dataSize, stepDownLeader, header); // check header - Assert.assertEquals(RaftClientRequest.dataStreamRequestType(), header.getType()); + Assertions.assertEquals(RaftClientRequest.dataStreamRequestType(), header.getType()); // check stream - final MultiDataStreamStateMachine stateMachine = (MultiDataStreamStateMachine) server.getDivision(header.getRaftGroupId()).getStateMachine(); + final MultiDataStreamStateMachine stateMachine = + (MultiDataStreamStateMachine) server.getDivision(header.getRaftGroupId()).getStateMachine(); final SingleDataStream stream = stateMachine.getSingleDataStream(header); final MyDataChannel channel = stream.getDataChannel(); - Assert.assertEquals(dataSize, channel.getBytesWritten()); - Assert.assertEquals(dataSize, channel.getForcedPosition()); + Assertions.assertEquals(dataSize, channel.getBytesWritten()); + Assertions.assertEquals(dataSize, channel.getForcedPosition()); // check writeRequest final RaftClientRequest writeRequest = stream.getWriteRequest(); - Assert.assertEquals(RaftClientRequest.dataStreamRequestType(), writeRequest.getType()); + Assertions.assertEquals(RaftClientRequest.dataStreamRequestType(), writeRequest.getType()); assertRaftClientMessage(header, null, writeRequest, header.getClientId(), stepDownLeader); } static CompletableFuture assertCloseReply(DataStreamOutputImpl out, DataStreamReply dataStreamReply, long bytesWritten, RaftPeerId leader, ClientId clientId, boolean stepDownLeader) { // Test close idempotent - Assert.assertSame(dataStreamReply, out.closeAsync().join()); - Assert.assertEquals(dataStreamReply.getClientId(), clientId); + Assertions.assertSame(dataStreamReply, out.closeAsync().join()); + Assertions.assertEquals(dataStreamReply.getClientId(), clientId); BaseTest.testFailureCase("writeAsync should fail", () -> out.writeAsync(DataStreamRequestByteBuffer.EMPTY_BYTE_BUFFER).join(), CompletionException.class, (Logger) null, AlreadyClosedException.class); @@ -359,7 +370,7 @@ static CompletableFuture assertCloseReply(DataStreamOutputImpl if (reply.isSuccess()) { final ByteString bytes = reply.getMessage().getContent(); if (!bytes.equals(MOCK)) { - Assert.assertEquals(bytesWritten2ByteString(bytesWritten), bytes); + Assertions.assertEquals(bytesWritten2ByteString(bytesWritten), bytes); } } @@ -372,18 +383,18 @@ static CompletableFuture assertCloseReply(DataStreamOutputImpl static void assertRaftClientMessage( RaftClientMessage expected, RaftPeerId expectedServerId, RaftClientMessage computed, ClientId expectedClientId, boolean stepDownLeader) { - Assert.assertNotNull(computed); - Assert.assertEquals(expectedClientId, computed.getClientId()); + Assertions.assertNotNull(computed); + Assertions.assertEquals(expectedClientId, computed.getClientId()); if (!stepDownLeader) { - Assert.assertEquals( + Assertions.assertEquals( Optional.ofNullable(expectedServerId).orElseGet(expected::getServerId), computed.getServerId()); } - Assert.assertEquals(expected.getRaftGroupId(), computed.getRaftGroupId()); + Assertions.assertEquals(expected.getRaftGroupId(), computed.getRaftGroupId()); } static LogEntryProto searchLogEntry(ClientInvocationId invocationId, RaftLog log) throws Exception { for (LogEntryHeader termIndex : log.getEntries(0, Long.MAX_VALUE)) { - final LogEntryProto entry = log.get(termIndex.getIndex()); + final LogEntryProto entry = getLogUnsafe(log, termIndex.getIndex()); if (entry.hasStateMachineLogEntry()) { if (invocationId.match(entry.getStateMachineLogEntry())) { return entry; @@ -394,12 +405,12 @@ static LogEntryProto searchLogEntry(ClientInvocationId invocationId, RaftLog log } static void assertLogEntry(LogEntryProto logEntry, RaftClientRequest request) { - Assert.assertNotNull(logEntry); - Assert.assertTrue(logEntry.hasStateMachineLogEntry()); + Assertions.assertNotNull(logEntry); + Assertions.assertTrue(logEntry.hasStateMachineLogEntry()); final StateMachineLogEntryProto s = logEntry.getStateMachineLogEntry(); - Assert.assertEquals(StateMachineLogEntryProto.Type.DATASTREAM, s.getType()); - Assert.assertEquals(request.getCallId(), s.getCallId()); - Assert.assertEquals(request.getClientId().toByteString(), s.getClientId()); + Assertions.assertEquals(StateMachineLogEntryProto.Type.DATASTREAM, s.getType()); + Assertions.assertEquals(request.getCallId(), s.getCallId()); + Assertions.assertEquals(request.getClientId().toByteString(), s.getClientId()); } static void assertLogEntry(RaftServer.Division division, SingleDataStream stream) throws Exception { @@ -408,6 +419,15 @@ static void assertLogEntry(RaftServer.Division division, SingleDataStream stream assertLogEntry(entryFromStream, request); final LogEntryProto entryFromLog = searchLogEntry(ClientInvocationId.valueOf(request), division.getRaftLog()); - Assert.assertEquals(entryFromStream, entryFromLog); + Assertions.assertEquals(entryFromStream, entryFromLog); + } + + ResourceLeakDetector.LeakListener LEAK_LISTENER = (resourceType, records) -> { + throw new IllegalStateException("Leak detected for resource type: " + resourceType + records); + }; + + static void enableResourceLeakDetector() { + ResourceLeakDetector.setLevel(Level.PARANOID); + ByteBufUtil.setLeakListener(DataStreamTestUtils.LEAK_LISTENER); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.java b/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.java index 3396ada9b3..82fab6477a 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.java @@ -17,7 +17,6 @@ */ package org.apache.ratis.datastream; -import org.apache.ratis.security.TlsConf; import org.apache.ratis.server.impl.MiniRaftCluster; import org.apache.ratis.RaftConfigKeys; import org.apache.ratis.conf.Parameters; @@ -31,7 +30,7 @@ /** * A {@link MiniRaftCluster} with {{@link SupportedRpcType#GRPC}} and {@link SupportedDataStreamType#NETTY}. */ -public class MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty extends MiniRaftClusterWithGrpc { +public final class MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty extends MiniRaftClusterWithGrpc { static class Factory extends MiniRaftCluster.Factory { private final Parameters parameters; @@ -50,15 +49,16 @@ public MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty newCluster(String[] public static final Factory FACTORY = new Factory(null); - public interface FactoryGet extends MiniRaftCluster.Factory.Get { + public interface FactoryGet + extends MiniRaftCluster.Factory.Get { @Override default MiniRaftCluster.Factory getFactory() { return FACTORY; } } - private MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty(String[] ids, String[] listenerIds, RaftProperties properties, - Parameters parameters) { + private MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty(String[] ids, String[] listenerIds, + RaftProperties properties, Parameters parameters) { super(ids, listenerIds, properties, parameters); } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty.java b/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty.java index 1e5149b431..9179fa0393 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty.java @@ -30,7 +30,7 @@ /** * A {@link MiniRaftCluster} with {{@link SupportedRpcType#NETTY}} and {@link SupportedDataStreamType#NETTY}. */ -public class MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty extends MiniRaftClusterWithNetty { +public final class MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty extends MiniRaftClusterWithNetty { public static final Factory FACTORY = new Factory() { @Override @@ -49,7 +49,8 @@ default Factory getFactor } } - private MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty(String[] ids, String[] listenerIds, RaftProperties properties) { + private MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty(String[] ids, String[] listenerIds, + RaftProperties properties) { super(ids, listenerIds, properties); } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamDisabled.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamDisabled.java index 168a1b02dc..697e746877 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamDisabled.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamDisabled.java @@ -26,14 +26,14 @@ import org.apache.ratis.protocol.RaftGroup; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.protocol.RaftPeer; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class TestDataStreamDisabled extends BaseTest { @Test public void testDataStreamDisabled() throws Exception { final RaftProperties properties = new RaftProperties(); - Assert.assertEquals(SupportedDataStreamType.DISABLED, RaftConfigKeys.DataStream.type(properties, LOG::info)); + Assertions.assertEquals(SupportedDataStreamType.DISABLED, RaftConfigKeys.DataStream.type(properties, LOG::info)); final RaftPeer server = RaftPeer.newBuilder().setId("s0").build(); @@ -44,9 +44,9 @@ public void testDataStreamDisabled() throws Exception { .setProperties(properties) .build(); DataStreamOutput out = client.getDataStreamApi().stream()) { - Assert.fail("Unexpected object: " + out); + Assertions.fail("Unexpected object: " + out); } catch (UnsupportedOperationException e) { - Assert.assertTrue(e.getMessage().contains( + Assertions.assertTrue(e.getMessage().contains( DisabledDataStreamClientFactory.class.getName() + "$1 does not support streamAsync")); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamSslWithRpcTypeGrpcAndDataStreamTypeNetty.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamSslWithRpcTypeGrpcAndDataStreamTypeNetty.java index 8e423ab293..247a7bcac1 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamSslWithRpcTypeGrpcAndDataStreamTypeNetty.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestDataStreamSslWithRpcTypeGrpcAndDataStreamTypeNetty.java @@ -24,7 +24,7 @@ import org.apache.ratis.security.TlsConf; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Slf4jUtils; -import org.junit.Ignore; +import org.junit.jupiter.api.Disabled; import org.slf4j.event.Level; import java.util.function.Supplier; @@ -36,12 +36,12 @@ public class TestDataStreamSslWithRpcTypeGrpcAndDataStreamTypeNetty } Parameters newParameters() { - final Parameters parameters = new Parameters(); + final Parameters parameters1 = new Parameters(); final TlsConf serverTlsConfig = SecurityTestUtils.newServerTlsConfig(true); - NettyConfigKeys.DataStream.Server.setTlsConf(parameters, serverTlsConfig); + NettyConfigKeys.DataStream.Server.setTlsConf(parameters1, serverTlsConfig); final TlsConf clientTlsConfig = SecurityTestUtils.newClientTlsConfig(true); - NettyConfigKeys.DataStream.Client.setTlsConf(parameters, clientTlsConfig); - return parameters; + NettyConfigKeys.DataStream.Client.setTlsConf(parameters1, clientTlsConfig); + return parameters1; } private final Supplier parameters = JavaUtils.memoize(this::newParameters); @@ -55,22 +55,22 @@ public MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.Factory getFactory() return new MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.Factory(getParameters()); } - @Ignore + @Disabled @Override public void testStreamWrites() { } - @Ignore + @Disabled @Override public void testStreamWithInvalidRoutingTable() { } - @Ignore + @Disabled @Override public void testMultipleStreamsMultipleServers() { } - @Ignore + @Disabled @Override public void testMultipleStreamsMultipleServersStepDownLeader() { } diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamChainTopologyWithGrpcCluster.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamChainTopologyWithGrpcCluster.java index 31b28b4c2d..778ee8225c 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamChainTopologyWithGrpcCluster.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamChainTopologyWithGrpcCluster.java @@ -22,13 +22,13 @@ import org.apache.ratis.netty.NettyConfigKeys; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; public class TestNettyDataStreamChainTopologyWithGrpcCluster extends DataStreamAsyncClusterTests implements MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.FactoryGet { - @Before + @BeforeEach public void setup() { final RaftProperties p = getProperties(); RaftClientConfigKeys.DataStream.setRequestTimeout(p, TimeDuration.ONE_MINUTE); diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamStarTopologyWithGrpcCluster.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamStarTopologyWithGrpcCluster.java index 45247d489a..bd80d6b6b5 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamStarTopologyWithGrpcCluster.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamStarTopologyWithGrpcCluster.java @@ -25,7 +25,7 @@ import org.apache.ratis.protocol.RoutingTable; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; -import org.junit.Before; +import org.junit.jupiter.api.BeforeEach; import java.util.Collection; import java.util.List; @@ -35,7 +35,7 @@ public class TestNettyDataStreamStarTopologyWithGrpcCluster extends DataStreamAsyncClusterTests implements MiniRaftClusterWithRpcTypeGrpcAndDataStreamTypeNetty.FactoryGet { - @Before + @BeforeEach public void setup() { final RaftProperties p = getProperties(); RaftClientConfigKeys.DataStream.setRequestTimeout(p, TimeDuration.ONE_MINUTE); diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithMock.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithMock.java index 27a1ee102f..8193f1d984 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithMock.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithMock.java @@ -36,14 +36,14 @@ import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.impl.RaftServerTestUtil; import org.apache.ratis.util.NetUtils; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@Ignore +@Disabled public class TestNettyDataStreamWithMock extends DataStreamBaseTest { static RaftPeer newRaftPeer(RaftServer server) { return RaftPeer.newBuilder() @@ -53,18 +53,24 @@ static RaftPeer newRaftPeer(RaftServer server) { .build(); } - @Before + @BeforeEach public void setup() { properties = new RaftProperties(); RaftConfigKeys.DataStream.setType(properties, SupportedDataStreamType.NETTY); } - RaftServer.Division mockDivision(RaftServer server) { + + RaftServer.Division mockDivision(RaftServer server, RaftGroupId groupId) { final RaftServer.Division division = mock(RaftServer.Division.class); when(division.getRaftServer()).thenReturn(server); when(division.getRaftConf()).thenAnswer(i -> getRaftConf()); final MultiDataStreamStateMachine stateMachine = new MultiDataStreamStateMachine(); + try { + stateMachine.initialize(server, groupId, null); + } catch (IOException e) { + throw new IllegalStateException(e); + } when(division.getStateMachine()).thenReturn(stateMachine); final DataStreamMap streamMap = RaftServerTestUtil.newDataStreamMap(server.getId()); @@ -95,7 +101,7 @@ private void testMockCluster(int numServers, RaftException leaderException, when(raftServer.getId()).thenReturn(peerId); when(raftServer.getPeer()).thenReturn(RaftPeer.newBuilder().setId(peerId).build()); if (getStateMachineException == null) { - final RaftServer.Division myDivision = mockDivision(raftServer); + final RaftServer.Division myDivision = mockDivision(raftServer, groupId); when(raftServer.getDivision(Mockito.any(RaftGroupId.class))).thenReturn(myDivision); } else { when(raftServer.getDivision(Mockito.any(RaftGroupId.class))).thenThrow(getStateMachineException); @@ -108,8 +114,8 @@ private void testMockCluster(int numServers, RaftException leaderException, submitException != null ? submitException : leaderException, getStateMachineException); } - void runTestMockCluster(RaftGroupId groupId, List raftServers, ClientId clientId, int bufferSize, int bufferNum, - Exception expectedException, Exception headerException) throws Exception { + void runTestMockCluster(RaftGroupId groupId, List raftServers, ClientId clientId, int bufferSize, + int bufferNum, Exception expectedException, Exception headerException) throws Exception { try { final List peers = raftServers.stream() .map(TestNettyDataStreamWithMock::newRaftPeer) diff --git a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithNettyCluster.java b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithNettyCluster.java index 90af31425f..1a29d014e4 100644 --- a/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithNettyCluster.java +++ b/ratis-test/src/test/java/org/apache/ratis/datastream/TestNettyDataStreamWithNettyCluster.java @@ -17,9 +17,10 @@ */ package org.apache.ratis.datastream; -import org.junit.Ignore; -@Ignore("Ignored by runzhiwang, because NettyClientRpc does not support sendRequestAsync") +import org.junit.jupiter.api.Disabled; + +@Disabled("Ignored by runzhiwang, because NettyClientRpc does not support sendRequestAsync") public class TestNettyDataStreamWithNettyCluster extends DataStreamClusterTests implements MiniRaftClusterWithRpcTypeNettyAndDataStreamTypeNetty.FactoryGet { diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestCustomGrpcServices.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestCustomGrpcServices.java new file mode 100644 index 0000000000..13c4a59fba --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestCustomGrpcServices.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.grpc; + +import org.apache.ratis.BaseTest; +import org.apache.ratis.RaftTestUtil; +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.conf.Parameters; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.grpc.server.GrpcServices; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.server.RaftServerRpc; +import org.apache.ratis.test.proto.GreeterGrpc; +import org.apache.ratis.test.proto.HelloReply; +import org.apache.ratis.test.proto.HelloRequest; +import org.apache.ratis.thirdparty.io.grpc.ManagedChannel; +import org.apache.ratis.thirdparty.io.grpc.ManagedChannelBuilder; +import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder; +import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; +import org.apache.ratis.util.IOUtils; +import org.apache.ratis.util.NetUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.util.EnumSet; +import java.util.Objects; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import static org.apache.ratis.RaftTestUtil.waitForLeader; + +public class TestCustomGrpcServices extends BaseTest { + /** Add two different greeter services for client and admin. */ + class MyCustomizer implements GrpcServices.Customizer { + final GreeterImpl clientGreeter = new GreeterImpl("Hello"); + final GreeterImpl adminGreeter = new GreeterImpl("Hi"); + + @Override + public NettyServerBuilder customize(NettyServerBuilder builder, EnumSet types) { + if (types.contains(GrpcServices.Type.CLIENT)) { + return builder.addService(clientGreeter); + } + if (types.contains(GrpcServices.Type.ADMIN)) { + return builder.addService(adminGreeter); + } + return builder; + } + } + + class GreeterImpl extends GreeterGrpc.GreeterImplBase { + private final String prefix; + + GreeterImpl(String prefix) { + this.prefix = prefix; + } + + String toReply(String request) { + return prefix + " " + request; + } + + @Override + public StreamObserver hello(StreamObserver responseObserver) { + return new StreamObserver() { + @Override + public void onNext(HelloRequest helloRequest) { + final String reply = toReply(helloRequest.getName()); + responseObserver.onNext(HelloReply.newBuilder().setMessage(reply).build()); + } + + @Override + public void onError(Throwable throwable) { + LOG.error("onError", throwable); + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + } + + class GreeterClient implements Closeable { + private final ManagedChannel channel; + private final StreamObserver requestHandler; + private final Queue> replies = new ConcurrentLinkedQueue<>(); + + GreeterClient(int port) { + this.channel = ManagedChannelBuilder.forAddress(NetUtils.LOCALHOST, port) + .usePlaintext() + .build(); + + final StreamObserver responseHandler = new StreamObserver() { + @Override + public void onNext(HelloReply helloReply) { + Objects.requireNonNull(replies.poll(), "queue is empty") + .complete(helloReply.getMessage()); + } + + @Override + public void onError(Throwable throwable) { + LOG.info("onError", throwable); + completeExceptionally(throwable); + } + + @Override + public void onCompleted() { + LOG.info("onCompleted"); + completeExceptionally(new IllegalStateException("onCompleted")); + } + + void completeExceptionally(Throwable throwable) { + replies.forEach(f -> f.completeExceptionally(throwable)); + replies.clear(); + } + }; + this.requestHandler = GreeterGrpc.newStub(channel).hello(responseHandler); + } + + @Override + public void close() throws IOException { + try { + /* After the request handler is cancelled, no more life-cycle hooks are allowed, + * see {@link org.apache.ratis.thirdparty.io.grpc.ClientCall.Listener#cancel(String, Throwable)} */ + // requestHandler.onCompleted(); + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw IOUtils.toInterruptedIOException("Failed to close", e); + } + } + + CompletableFuture send(String name) { + LOG.info("send: {}", name); + final HelloRequest request = HelloRequest.newBuilder().setName(name).build(); + final CompletableFuture f = new CompletableFuture<>(); + try { + requestHandler.onNext(request); + replies.offer(f); + } catch (IllegalStateException e) { + // already closed + f.completeExceptionally(e); + } + return f.whenComplete((r, e) -> LOG.info("reply: {}", r)); + } + } + + @Test + public void testCustomServices() throws Exception { + final String[] ids = {"s0"}; + final RaftProperties properties = new RaftProperties(); + + final Parameters parameters = new Parameters(); + final MyCustomizer customizer = new MyCustomizer(); + GrpcConfigKeys.Server.setServicesCustomizer(parameters, customizer); + + try(MiniRaftClusterWithGrpc cluster = new MiniRaftClusterWithGrpc(ids, properties, parameters)) { + cluster.start(); + final RaftServerRpc server = waitForLeader(cluster).getRaftServer().getServerRpc(); + + // test Raft service + try (RaftClient client = cluster.createClient()) { + final RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("abc")); + Assertions.assertTrue(reply.isSuccess()); + } + + // test custom client service + final int clientPort = server.getClientServerAddress().getPort(); + try (GreeterClient client = new GreeterClient(clientPort)) { + sendAndAssertReply("world", client, customizer.clientGreeter); + } + + // test custom admin service + final int adminPort = server.getAdminServerAddress().getPort(); + try (GreeterClient admin = new GreeterClient(adminPort)) { + sendAndAssertReply("admin", admin, customizer.adminGreeter); + } + } + } + + static void sendAndAssertReply(String name, GreeterClient client, GreeterImpl greeter) { + final String computed = client.send(name).join(); + final String expected = greeter.toReply(name); + Assertions.assertEquals(expected, computed); + } +} diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcFactory.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcFactory.java index 76fbcee5eb..99a395d8a1 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcFactory.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcFactory.java @@ -18,16 +18,14 @@ package org.apache.ratis.grpc; import org.apache.ratis.BaseTest; -import org.apache.ratis.util.JavaUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class TestGrpcFactory extends BaseTest { @Test public void testUseCacheForAllThreads() { // trigger GrpcFactory static initializer final boolean value = GrpcFactory.checkPooledByteBufAllocatorUseCacheForAllThreads(LOG::info); - Assert.assertFalse(value); - LOG.info("value is {}", value); + Assertions.assertFalse(value); } } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderElectionWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderElectionWithGrpc.java index 7730cb1166..797124bc16 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderElectionWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderElectionWithGrpc.java @@ -19,7 +19,7 @@ import org.apache.ratis.server.impl.BlockRequestHandlingInjection; import org.apache.ratis.server.impl.LeaderElectionTests; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestLeaderElectionWithGrpc extends LeaderElectionTests @@ -30,7 +30,7 @@ public class TestLeaderElectionWithGrpc public void testEnforceLeader() throws Exception { super.testEnforceLeader(); - MiniRaftClusterWithGrpc.sendServerRequestInjection.clear(); + MiniRaftClusterWithGrpc.SEND_SERVER_REQUEST_INJECTION.clear(); BlockRequestHandlingInjection.getInstance().unblockAll(); } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderInstallSnapshot.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderInstallSnapshotWithGrpc.java similarity index 58% rename from ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderInstallSnapshot.java rename to ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderInstallSnapshotWithGrpc.java index 5f7a40f0f4..9b2c903849 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderInstallSnapshot.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLeaderInstallSnapshotWithGrpc.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,23 +18,38 @@ package org.apache.ratis.grpc; import org.apache.ratis.InstallSnapshotFromLeaderTests; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.Arrays; import java.util.Collection; -@RunWith(Parameterized.class) -public class TestLeaderInstallSnapshot +public class TestLeaderInstallSnapshotWithGrpc extends InstallSnapshotFromLeaderTests implements MiniRaftClusterWithGrpc.FactoryGet { - public TestLeaderInstallSnapshot(Boolean separateHeartbeat) { + public static Collection data() { + return Arrays.asList((new Boolean[][] {{Boolean.FALSE}, {Boolean.TRUE}})); + } + + @ParameterizedTest + @MethodSource("data") + public void testMultiFileInstallSnapshot(Boolean separateHeartbeat) throws Exception { GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); + super.testMultiFileInstallSnapshot(); } - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList((new Boolean[][] {{Boolean.FALSE}, {Boolean.TRUE}})); + @ParameterizedTest + @MethodSource("data") + public void testSeparateSnapshotInstallPath(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); + super.testSeparateSnapshotInstallPath(); + } + + @ParameterizedTest + @MethodSource("data") + public void testInstallSnapshotLeaderSwitch(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); + super.testInstallSnapshotLeaderSwitch(); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogAppenderWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogAppenderWithGrpc.java index c0d102f957..107cd7ba9a 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogAppenderWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogAppenderWithGrpc.java @@ -33,10 +33,9 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Slf4jUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.event.Level; import java.io.IOException; @@ -47,7 +46,6 @@ import static org.apache.ratis.RaftTestUtil.waitForLeader; -@RunWith(Parameterized.class) public class TestLogAppenderWithGrpc extends LogAppenderTests implements MiniRaftClusterWithGrpc.FactoryGet { @@ -55,17 +53,14 @@ public class TestLogAppenderWithGrpc Slf4jUtils.setLogLevel(FollowerInfo.LOG, Level.DEBUG); } - public TestLogAppenderWithGrpc(Boolean separateHeartbeat) { - GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); - } - - @Parameterized.Parameters public static Collection data() { return Arrays.asList((new Boolean[][] {{Boolean.FALSE}, {Boolean.TRUE}})); } - @Test - public void testPendingLimits() throws IOException, InterruptedException { + @ParameterizedTest + @MethodSource("data") + public void testPendingLimits(Boolean separateHeartbeat) throws IOException, InterruptedException { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); int maxAppends = 10; RaftProperties properties = new RaftProperties(); properties.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, @@ -94,7 +89,7 @@ public void testPendingLimits() throws IOException, InterruptedException { JavaUtils.attempt(() -> { for (long nextIndex : leader.getInfo().getFollowerNextIndices()) { // Verify nextIndex does not progress due to pendingRequests limit - Assert.assertEquals(initialNextIndex + maxAppends, nextIndex); + Assertions.assertEquals(initialNextIndex + maxAppends, nextIndex); } }, 10, ONE_SECOND, "matching nextIndex", LOG); for (RaftServer.Division server : cluster.getFollowers()) { @@ -107,8 +102,10 @@ public void testPendingLimits() throws IOException, InterruptedException { } } - @Test - public void testRestartLogAppender() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testRestartLogAppender(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(2, this::runTestRestartLogAppender); } @@ -120,7 +117,7 @@ private void runTestRestartLogAppender(MiniRaftClusterWithGrpc cluster) throws E try(RaftClient client = cluster.createClient(leader.getId())) { for(int i = 0; i < 10; i++) { final RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + ++messageCount)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } @@ -128,7 +125,7 @@ private void runTestRestartLogAppender(MiniRaftClusterWithGrpc cluster) throws E final GrpcServerMetrics leaderMetrics = new GrpcServerMetrics(leader.getMemberId().toString()); final String counter = String.format(GrpcServerMetrics.RATIS_GRPC_METRICS_LOG_APPENDER_INCONSISTENCY, cluster.getFollowers().iterator().next().getMemberId().getPeerId()); - Assert.assertEquals(0L, leaderMetrics.getRegistry().counter(counter).getCount()); + Assertions.assertEquals(0L, leaderMetrics.getRegistry().counter(counter).getCount()); // restart LogAppender RaftServerTestUtil.restartLogAppenders(leader); @@ -137,7 +134,7 @@ private void runTestRestartLogAppender(MiniRaftClusterWithGrpc cluster) throws E try(RaftClient client = cluster.createClient(leader.getId())) { for(int i = 0; i < 10; i++) { final RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + ++messageCount)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } @@ -148,7 +145,7 @@ private void runTestRestartLogAppender(MiniRaftClusterWithGrpc cluster) throws E // assert INCONSISTENCY counter >= 1 // If old LogAppender die before new LogAppender start, INCONSISTENCY equal to 1, // else INCONSISTENCY greater than 1 - Assert.assertTrue(newleaderMetrics.getRegistry().counter(counter).getCount() >= 1L); + Assertions.assertTrue(newleaderMetrics.getRegistry().counter(counter).getCount() >= 1L); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogMetadataTestsWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogMetadataTestsWithGrpc.java new file mode 100644 index 0000000000..1c69c42bbb --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestLogMetadataTestsWithGrpc.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ratis.grpc; + +import org.apache.ratis.server.impl.LogMetadataTests; + +public class TestLogMetadataTestsWithGrpc extends LogMetadataTests + implements MiniRaftClusterWithGrpc.FactoryGet{ + +} diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftLogTruncateWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftLogTruncateWithGrpc.java new file mode 100644 index 0000000000..dc28463747 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftLogTruncateWithGrpc.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.grpc; + +import org.apache.ratis.RaftLogTruncateTests; + +public class TestRaftLogTruncateWithGrpc extends RaftLogTruncateTests + implements MiniRaftClusterWithGrpc.FactoryGet { +} diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftOutputStreamWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftOutputStreamWithGrpc.java index fb35d958ab..2f1ef3f124 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftOutputStreamWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftOutputStreamWithGrpc.java @@ -18,12 +18,10 @@ package org.apache.ratis.grpc; import org.apache.ratis.OutputStreamBaseTest; +import org.junit.jupiter.api.Timeout; +@Timeout(value = 100) public class TestRaftOutputStreamWithGrpc extends OutputStreamBaseTest implements MiniRaftClusterWithGrpc.FactoryGet { - @Override - public int getGlobalTimeoutSeconds() { - return 100; - } } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java index 0af1d87cce..322eb52287 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftServerWithGrpc.java @@ -26,7 +26,6 @@ import org.apache.ratis.protocol.ClientInvocationId; import org.apache.ratis.server.RetryCache; import org.apache.ratis.util.JavaUtils; -import org.slf4j.event.Level; import org.apache.ratis.conf.Parameters; import org.apache.ratis.security.SecurityTestUtils; import org.apache.ratis.server.storage.RaftStorage; @@ -62,17 +61,20 @@ import org.apache.ratis.util.ProtoUtils; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.slf4j.event.Level; import javax.net.ssl.KeyManager; import javax.net.ssl.TrustManager; import java.io.IOException; import java.nio.channels.OverlappingFileLockException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; @@ -80,30 +82,26 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -@RunWith(Parameterized.class) public class TestRaftServerWithGrpc extends BaseTest implements MiniRaftClusterWithGrpc.FactoryGet { { Slf4jUtils.setLogLevel(GrpcClientProtocolClient.LOG, Level.TRACE); } - public TestRaftServerWithGrpc(Boolean separateHeartbeat) { - GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); - } - - @Parameterized.Parameters public static Collection data() { return Arrays.asList((new Boolean[][] {{Boolean.FALSE}, {Boolean.TRUE}})); } - @Before + @BeforeEach public void setup() { final RaftProperties p = getProperties(); p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class); RaftClientConfigKeys.Rpc.setRequestTimeout(p, TimeDuration.valueOf(1, TimeUnit.SECONDS)); } - @Test - public void testServerRestartOnException() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testServerRestartOnException(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(1, this::runTestServerRestartOnException); } @@ -149,8 +147,10 @@ void runTestServerRestartOnException(MiniRaftClusterWithGrpc cluster) throws Exc cluster.getServerFactory(leaderId).newRaftServerRpc(cluster.getServer(leaderId)); } - @Test - public void testUnsupportedMethods() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testUnsupportedMethods(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(1, this::runTestUnsupportedMethods); } @@ -167,8 +167,10 @@ void runTestUnsupportedMethods(MiniRaftClusterWithGrpc cluster) throws Exception UnsupportedOperationException.class); } - @Test - public void testLeaderRestart() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testLeaderRestart(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(3, this::runTestLeaderRestart); } @@ -178,7 +180,7 @@ void runTestLeaderRestart(MiniRaftClusterWithGrpc cluster) throws Exception { try (final RaftClient client = cluster.createClient()) { // send a request to make sure leader is ready final CompletableFuture f = client.async().send(new SimpleMessage("testing")); - Assert.assertTrue(f.get().isSuccess()); + Assertions.assertTrue(f.get().isSuccess()); } try (final RaftClient client = cluster.createClient()) { @@ -189,14 +191,14 @@ void runTestLeaderRestart(MiniRaftClusterWithGrpc cluster) throws Exception { { // send a request using rpc directly final RaftClientRequest request = newRaftClientRequest(client, seqNum.incrementAndGet()); - Assert.assertEquals(client.getId(), request.getClientId()); + Assertions.assertEquals(client.getId(), request.getClientId()); final CompletableFuture f = rpc.sendRequestAsync(request); final RaftClientReply reply = f.get(); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); RaftClientTestUtil.handleReply(request, reply, client); invocationId = ClientInvocationId.valueOf(request.getClientId(), request.getCallId()); final RetryCache.Entry entry = leader.getRetryCache().getIfPresent(invocationId); - Assert.assertNotNull(entry); + Assertions.assertNotNull(entry); LOG.info("cache entry {}", entry); } @@ -206,13 +208,13 @@ void runTestLeaderRestart(MiniRaftClusterWithGrpc cluster) throws Exception { final RaftClientRequest requestBlocked = newRaftClientRequest(client, seqNum.incrementAndGet()); final CompletableFuture futureBlocked = rpc.sendRequestAsync(requestBlocked); - JavaUtils.attempt(() -> Assert.assertNull(leader.getRetryCache().getIfPresent(invocationId)), + JavaUtils.attempt(() -> Assertions.assertNull(leader.getRetryCache().getIfPresent(invocationId)), 10, HUNDRED_MILLIS, "invalidate cache entry", LOG); LOG.info("cache entry not found for {}", invocationId); // change leader RaftTestUtil.changeLeader(cluster, leader.getId()); - Assert.assertNotEquals(RaftPeerRole.LEADER, leader.getInfo().getCurrentRole()); + Assertions.assertNotEquals(RaftPeerRole.LEADER, leader.getInfo().getCurrentRole()); // the blocked request should fail testFailureCase("request should fail", futureBlocked::get, @@ -229,13 +231,17 @@ void runTestLeaderRestart(MiniRaftClusterWithGrpc cluster) throws Exception { } - @Test - public void testRaftClientMetrics() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testRaftClientMetrics(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(3, this::testRaftClientRequestMetrics); } - @Test - public void testRaftServerMetrics() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testRaftServerMetrics(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); final RaftProperties p = getProperties(); RaftServerConfigKeys.Write.setElementLimit(p, 10); RaftServerConfigKeys.Write.setByteLimit(p, SizeInBytes.valueOf("1MB")); @@ -252,7 +258,7 @@ void testRequestMetrics(MiniRaftClusterWithGrpc cluster) throws Exception { try (RaftClient client = cluster.createClient()) { // send a request to make sure leader is ready final CompletableFuture< RaftClientReply > f = client.async().send(new SimpleMessage("testing")); - Assert.assertTrue(f.get().isSuccess()); + Assertions.assertTrue(f.get().isSuccess()); } SimpleStateMachine4Testing stateMachine = SimpleStateMachine4Testing.get(cluster.getLeader()); @@ -285,14 +291,14 @@ void testRequestMetrics(MiniRaftClusterWithGrpc cluster) throws Exception { client = cluster.createClient(cluster.getLeader().getId(), RetryPolicies.noRetry()); final SizeInBytes size = SizeInBytes.valueOf("1025kb"); final ByteString bytes = randomByteString(size.getSizeInt()); - Assert.assertEquals(size.getSizeInt(), bytes.size()); + Assertions.assertEquals(size.getSizeInt(), bytes.size()); client.async().send(new SimpleMessage(size + "-message", bytes)); clients.add(client); RaftTestUtil.waitFor(() -> getRaftServerMetrics(cluster.getLeader()) .getNumRequestsByteSizeLimitHits().getCount() == 1, 300, 5000); - Assert.assertEquals(2, getRaftServerMetrics(cluster.getLeader()) + Assertions.assertEquals(2, getRaftServerMetrics(cluster.getLeader()) .getNumResourceLimitHits().getCount()); } finally { for (RaftClient client : clients) { @@ -300,7 +306,7 @@ void testRequestMetrics(MiniRaftClusterWithGrpc cluster) throws Exception { } } } - + static ByteString randomByteString(int size) { final ByteString.Output out = ByteString.newOutput(size); final ThreadLocalRandom random = ThreadLocalRandom.current(); @@ -326,36 +332,36 @@ void testRaftClientRequestMetrics(MiniRaftClusterWithGrpc cluster) throws IOExce try (final RaftClient client = cluster.createClient()) { final CompletableFuture f1 = client.async().send(new SimpleMessage("testing")); - Assert.assertTrue(f1.get().isSuccess()); + Assertions.assertTrue(f1.get().isSuccess()); final DefaultTimekeeperImpl write = (DefaultTimekeeperImpl) registry.timer(RAFT_CLIENT_WRITE_REQUEST); - JavaUtils.attempt(() -> Assert.assertTrue(write.getTimer().getCount() > 0), + JavaUtils.attempt(() -> Assertions.assertTrue(write.getTimer().getCount() > 0), 3, TimeDuration.ONE_SECOND, "writeTimer metrics", LOG); final CompletableFuture f2 = client.async().sendReadOnly(new SimpleMessage("testing")); - Assert.assertTrue(f2.get().isSuccess()); + Assertions.assertTrue(f2.get().isSuccess()); final DefaultTimekeeperImpl read = (DefaultTimekeeperImpl) registry.timer(RAFT_CLIENT_READ_REQUEST); - JavaUtils.attempt(() -> Assert.assertTrue(read.getTimer().getCount() > 0), + JavaUtils.attempt(() -> Assertions.assertTrue(read.getTimer().getCount() > 0), 3, TimeDuration.ONE_SECOND, "readTimer metrics", LOG); final CompletableFuture f3 = client.async().sendStaleRead(new SimpleMessage("testing"), 0, leader.getId()); - Assert.assertTrue(f3.get().isSuccess()); + Assertions.assertTrue(f3.get().isSuccess()); final DefaultTimekeeperImpl staleRead = (DefaultTimekeeperImpl) registry.timer(RAFT_CLIENT_STALE_READ_REQUEST); - JavaUtils.attempt(() -> Assert.assertTrue(staleRead.getTimer().getCount() > 0), + JavaUtils.attempt(() -> Assertions.assertTrue(staleRead.getTimer().getCount() > 0), 3, TimeDuration.ONE_SECOND, "staleReadTimer metrics", LOG); final CompletableFuture f4 = client.async().watch(0, RaftProtos.ReplicationLevel.ALL); - Assert.assertTrue(f4.get().isSuccess()); + Assertions.assertTrue(f4.get().isSuccess()); final DefaultTimekeeperImpl watchAll = (DefaultTimekeeperImpl) registry.timer( String.format(RAFT_CLIENT_WATCH_REQUEST, "-ALL")); - JavaUtils.attempt(() -> Assert.assertTrue(watchAll.getTimer().getCount() > 0), + JavaUtils.attempt(() -> Assertions.assertTrue(watchAll.getTimer().getCount() > 0), 3, TimeDuration.ONE_SECOND, "watchAllTimer metrics", LOG); final CompletableFuture f5 = client.async().watch(0, RaftProtos.ReplicationLevel.MAJORITY); - Assert.assertTrue(f5.get().isSuccess()); + Assertions.assertTrue(f5.get().isSuccess()); final DefaultTimekeeperImpl watch = (DefaultTimekeeperImpl) registry.timer( String.format(RAFT_CLIENT_WATCH_REQUEST, "")); - JavaUtils.attempt(() -> Assert.assertTrue(watch.getTimer().getCount() > 0), + JavaUtils.attempt(() -> Assertions.assertTrue(watch.getTimer().getCount() > 0), 3, TimeDuration.ONE_SECOND, "watchTimer metrics", LOG); } } @@ -366,8 +372,10 @@ static RaftClientRequest newRaftClientRequest(RaftClient client, long seqNum) { RaftClientRequest.writeRequestType(), ProtoUtils.toSlidingWindowEntry(seqNum, seqNum == 1L)); } - @Test - public void testTlsWithKeyAndTrustManager() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testTlsWithKeyAndTrustManager(Boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); final RaftProperties p = getProperties(); RaftServerConfigKeys.Write.setElementLimit(p, 10); RaftServerConfigKeys.Write.setByteLimit(p, SizeInBytes.valueOf("1MB")); diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftSnapshotWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftSnapshotWithGrpc.java index 2d8524f26c..e6c2f6613b 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftSnapshotWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftSnapshotWithGrpc.java @@ -26,8 +26,10 @@ import org.apache.ratis.metrics.RatisMetricRegistry; import org.apache.ratis.server.RaftServer; import org.apache.ratis.statemachine.RaftSnapshotBaseTest; -import org.junit.Assert; +import org.apache.ratis.test.tag.Flaky; +import org.junit.jupiter.api.Assertions; +@Flaky("RATIS-2261") public class TestRaftSnapshotWithGrpc extends RaftSnapshotBaseTest { @Override public MiniRaftCluster.Factory getFactory() { @@ -39,10 +41,10 @@ protected void verifyInstallSnapshotMetric(RaftServer.Division leader) { MetricRegistryInfo info = new MetricRegistryInfo(leader.getMemberId().toString(), "ratis_grpc", "log_appender", "Metrics for Ratis Grpc Log Appender"); Optional metricRegistry = MetricRegistries.global().get(info); - Assert.assertTrue(metricRegistry.isPresent()); + Assertions.assertTrue(metricRegistry.isPresent()); final LongCounter installSnapshotCounter = metricRegistry.get().counter("num_install_snapshot"); - Assert.assertNotNull(installSnapshotCounter); - Assert.assertTrue(installSnapshotCounter.getCount() >= 1); + Assertions.assertNotNull(installSnapshotCounter); + Assertions.assertTrue(installSnapshotCounter.getCount() >= 1); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java index bc0061f5f8..42211cefc1 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRaftWithGrpc.java @@ -28,21 +28,20 @@ import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; +import org.apache.ratis.test.tag.Flaky; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; -import java.util.Arrays; -import java.util.Collection; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import static org.apache.ratis.RaftTestUtil.waitForLeader; -@RunWith(Parameterized.class) public class TestRaftWithGrpc extends RaftBasicTests implements MiniRaftClusterWithGrpc.FactoryGet { @@ -52,35 +51,33 @@ public class TestRaftWithGrpc SimpleStateMachine4Testing.class, StateMachine.class); } - public TestRaftWithGrpc(Boolean separateHeartbeat) { - GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); - } - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList((new Boolean[][] {{Boolean.FALSE}, {Boolean.TRUE}})); + @Disabled + @Override + public void testWithLoad() { + // skip testWithLoad() from parent, called from parameterized testWithLoad(boolean) } - @Override - @Test - public void testWithLoad() throws Exception { + @Flaky("RATIS-2253") + @Timeout(300) + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testWithLoad(boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); super.testWithLoad(); BlockRequestHandlingInjection.getInstance().unblockAll(); } - @Test - public void testRequestTimeout() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testRequestTimeout(boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(NUM_SERVERS, cluster -> testRequestTimeout(false, cluster, LOG)); } - @Test - public void testStateMachineMetrics() throws Exception { - runWithNewCluster(NUM_SERVERS, cluster -> - testStateMachineMetrics(false, cluster, LOG)); - } - - @Test - public void testUpdateViaHeartbeat() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testUpdateViaHeartbeat(boolean separateHeartbeat) throws Exception { + GrpcConfigKeys.Server.setHeartbeatChannel(getProperties(), separateHeartbeat); runWithNewCluster(NUM_SERVERS, this::runTestUpdateViaHeartbeat); } @@ -97,7 +94,7 @@ void runTestUpdateViaHeartbeat(MiniRaftClusterWithGrpc cluster) throws Exception replyFuture = client.async().send(new RaftTestUtil.SimpleMessage("abc")); TimeDuration.valueOf(5 , TimeUnit.SECONDS).sleep(); // replyFuture should not be completed until append request is unblocked. - Assert.assertFalse(replyFuture.isDone()); + Assertions.assertFalse(replyFuture.isDone()); // unblock append request. cluster.getServerAliveStream() .filter(impl -> !impl.getInfo().isLeader()) @@ -113,9 +110,9 @@ void runTestUpdateViaHeartbeat(MiniRaftClusterWithGrpc cluster) throws Exception final LogEntryHeader[] leaderEntries = leaderLog.getEntries(0, Long.MAX_VALUE); final RaftLog followerLog = raftServer.getRaftLog(); - Assert.assertEquals(leaderNextIndex, followerLog.getNextIndex()); + Assertions.assertEquals(leaderNextIndex, followerLog.getNextIndex()); final LogEntryHeader[] serverEntries = followerLog.getEntries(0, Long.MAX_VALUE); - Assert.assertArrayEquals(serverEntries, leaderEntries); + Assertions.assertArrayEquals(serverEntries, leaderEntries); }, 10, HUNDRED_MILLIS, "assertRaftLog-" + raftServer.getId(), LOG))); // Wait for heartbeats from leader to be received by followers @@ -125,8 +122,8 @@ void runTestUpdateViaHeartbeat(MiniRaftClusterWithGrpc cluster) throws Exception final long leaderNextIndex = leaderLog.getNextIndex(); // FollowerInfo in the leader state should have updated next and match index. final long followerMatchIndex = logAppender.getFollower().getMatchIndex(); - Assert.assertTrue(followerMatchIndex >= leaderNextIndex - 1); - Assert.assertEquals(followerMatchIndex + 1, logAppender.getFollower().getNextIndex()); + Assertions.assertTrue(followerMatchIndex >= leaderNextIndex - 1); + Assertions.assertEquals(followerMatchIndex + 1, logAppender.getFollower().getNextIndex()); }, 10, HUNDRED_MILLIS, "assertRaftLog-" + logAppender.getFollower(), LOG))); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRetryCacheWithGrpc.java b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRetryCacheWithGrpc.java index a39a4d1ef4..5a704689da 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestRetryCacheWithGrpc.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/TestRetryCacheWithGrpc.java @@ -37,8 +37,9 @@ import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.event.Level; import java.io.IOException; @@ -66,7 +67,7 @@ public void testInvalidateRepliedCalls() throws Exception { } static long assertReply(RaftClientReply reply) { - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); return reply.getCallId(); } @@ -88,9 +89,10 @@ void assertRetryCacheEntry(RaftClient client, long callId, boolean exist) throws assertRetryCacheEntry(client, callId, exist, false); } - void assertRetryCacheEntry(RaftClient client, long callId, boolean exist, boolean eventually) throws InterruptedException { + void assertRetryCacheEntry(RaftClient client, long callId, boolean exist, boolean eventually) + throws InterruptedException { Supplier lookup = () -> RetryCacheTestUtil.get(leader, client.getId(), callId); - Consumer assertion = exist ? Assert::assertNotNull : Assert::assertNull; + Consumer assertion = exist ? Assertions::assertNotNull : Assertions::assertNull; if (eventually) { JavaUtils.attempt(() -> assertion.accept(lookup.get()), 100, TimeDuration.ONE_MILLISECOND, "retry cache entry", null); @@ -144,7 +146,7 @@ void run() throws Exception { ONE_SECOND.sleep(); // No calls can be completed. for (CompletableFuture f : asyncCalls) { - Assert.assertFalse(f.isDone()); + Assertions.assertFalse(f.isDone()); } stateMachine.unblockApplyTransaction(); // No calls can be invalidated. @@ -170,7 +172,8 @@ void run() throws Exception { } } - @Test(timeout = 10000) + @Test + @Timeout(value = 10) public void testRetryOnResourceUnavailableException() throws InterruptedException, IOException { RaftProperties properties = new RaftProperties(); diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcMessageMetrics.java b/ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcMessageMetrics.java similarity index 81% rename from ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcMessageMetrics.java rename to ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcMessageMetrics.java index 2abba79300..8094069cf5 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/TestGrpcMessageMetrics.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcMessageMetrics.java @@ -15,13 +15,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ratis.grpc; +package org.apache.ratis.grpc.server; import org.apache.ratis.BaseTest; +import org.apache.ratis.grpc.MiniRaftClusterWithGrpc; import org.apache.ratis.server.impl.MiniRaftCluster; import org.apache.ratis.RaftTestUtil; import org.apache.ratis.client.RaftClient; -import org.apache.ratis.grpc.server.GrpcService; import org.apache.ratis.metrics.impl.JvmMetrics; import org.apache.ratis.metrics.RatisMetricRegistry; import org.apache.ratis.protocol.RaftClientReply; @@ -29,8 +29,8 @@ import org.apache.ratis.server.impl.RaftServerTestUtil; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -60,14 +60,16 @@ static void sendMessages(MiniRaftCluster cluster) throws Exception { client.async().send(new RaftTestUtil.SimpleMessage("abc")); } // Wait for commits to happen on leader - JavaUtils.attempt(() -> assertMessageCount(cluster.getLeader()), 100, HUNDRED_MILLIS, cluster.getLeader().getId() + "-assertMessageCount", null); + JavaUtils.attempt(() -> assertMessageCount(cluster.getLeader()), 100, HUNDRED_MILLIS, + cluster.getLeader().getId() + "-assertMessageCount", null); } static void assertMessageCount(RaftServer.Division server) { String serverId = server.getId().toString(); - GrpcService service = (GrpcService) RaftServerTestUtil.getServerRpc(server); - RatisMetricRegistry registry = service.getServerInterceptor().getMetrics().getRegistry(); + final GrpcServicesImpl services = (GrpcServicesImpl) RaftServerTestUtil.getServerRpc(server); + final RatisMetricRegistry registry = services.getMessageMetrics().getRegistry(); String counter_prefix = serverId + "_" + "ratis.grpc.RaftServerProtocolService"; - Assert.assertTrue(registry.counter(counter_prefix + "_" + "requestVote" + "_OK_completed_total").getCount() > 0); + Assertions.assertTrue( + registry.counter(counter_prefix + "_" + "requestVote" + "_OK_completed_total").getCount() > 0); } } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcServerMetrics.java b/ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcServerMetrics.java index 04f8ded95d..1b57730594 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcServerMetrics.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/server/TestGrpcServerMetrics.java @@ -38,9 +38,9 @@ import org.apache.ratis.protocol.RaftGroupMemberId; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.thirdparty.com.codahale.metrics.Gauge; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; public class TestGrpcServerMetrics { @@ -50,7 +50,7 @@ public class TestGrpcServerMetrics { private static RaftPeerId raftPeerId; private static RaftPeerId followerId; - @BeforeClass + @BeforeAll public static void setUp() throws Exception { raftGroupId = RaftGroupId.randomId(); raftPeerId = RaftPeerId.valueOf("TestId"); @@ -74,11 +74,11 @@ public void testGrpcLogAppenderLatencyTimer() throws Exception { final String format = RATIS_GRPC_METRICS_LOG_APPENDER_LATENCY + GrpcServerMetrics.getHeartbeatSuffix(heartbeat); final String name = String.format(format, followerId); final DefaultTimekeeperImpl t = (DefaultTimekeeperImpl) ratisMetricRegistry.timer(name); - Assert.assertEquals(0L, t.getTimer().getSnapshot().getMax()); + Assertions.assertEquals(0L, t.getTimer().getSnapshot().getMax()); req.startRequestTimer(); Thread.sleep(1000L); req.stopRequestTimer(); - Assert.assertTrue(t.getTimer().getSnapshot().getMax() > 1000L); + Assertions.assertTrue(t.getTimer().getSnapshot().getMax() > 1000L); } } @@ -89,7 +89,7 @@ public void testGrpcLogRequestTotal() { RATIS_GRPC_METRICS_REQUESTS_COUNT + GrpcServerMetrics .getHeartbeatSuffix(heartbeat)).getCount(); grpcServerMetrics.onRequestCreate(heartbeat); - Assert.assertEquals(reqTotal + 1, ratisMetricRegistry.counter( + Assertions.assertEquals(reqTotal + 1, ratisMetricRegistry.counter( RATIS_GRPC_METRICS_REQUESTS_COUNT + GrpcServerMetrics .getHeartbeatSuffix(heartbeat)).getCount()); } @@ -97,9 +97,9 @@ public void testGrpcLogRequestTotal() { @Test public void testGrpcLogRequestRetry() { - Assert.assertEquals(0L, ratisMetricRegistry.counter(RATIS_GRPC_METRICS_REQUEST_RETRY_COUNT).getCount()); + Assertions.assertEquals(0L, ratisMetricRegistry.counter(RATIS_GRPC_METRICS_REQUEST_RETRY_COUNT).getCount()); grpcServerMetrics.onRequestRetry(); - Assert.assertEquals(1L, ratisMetricRegistry.counter(RATIS_GRPC_METRICS_REQUEST_RETRY_COUNT).getCount()); + Assertions.assertEquals(1L, ratisMetricRegistry.counter(RATIS_GRPC_METRICS_REQUEST_RETRY_COUNT).getCount()); } @Test @@ -110,9 +110,9 @@ public void testGrpcLogPendingRequestCount() { pendingRequest::logRequestsSize); final String name = String.format(RATIS_GRPC_METRICS_LOG_APPENDER_PENDING_COUNT, raftPeerId); final Gauge gauge = ServerMetricsTestUtils.getGaugeWithName(name, grpcServerMetrics::getRegistry); - Assert.assertEquals(0, gauge.getValue()); + Assertions.assertEquals(0, gauge.getValue()); when(pendingRequest.logRequestsSize()).thenReturn(10); - Assert.assertEquals(10, gauge.getValue()); + Assertions.assertEquals(10, gauge.getValue()); } @Test @@ -133,8 +133,8 @@ public void testGrpcLogAppenderRequestCounters() { private void assertCounterIncremented(String counterVar, Consumer incFunction) { String counter = String.format(counterVar, raftPeerId.toString()); - Assert.assertEquals(0L, ratisMetricRegistry.counter(counter).getCount()); + Assertions.assertEquals(0L, ratisMetricRegistry.counter(counter).getCount()); incFunction.accept(raftPeerId.toString()); - Assert.assertEquals(1L, ratisMetricRegistry.counter(counter).getCount()); + Assertions.assertEquals(1L, ratisMetricRegistry.counter(counter).getCount()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestClient.java b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestClient.java index 130c05eb94..ca8957e2d4 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestClient.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestClient.java @@ -37,6 +37,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; /** gRPC client for testing */ @@ -55,8 +56,11 @@ static StreamObserverFactory withDeadline(TimeDuration timeout) { static StreamObserverFactory withTimeout(TimeDuration timeout) { final String className = JavaUtils.getClassSimpleName(HelloRequest.class) + ":"; + final AtomicBoolean initialized = new AtomicBoolean(); return (stub, responseHandler) -> StreamObserverWithTimeout.newInstance("test", - r -> className + r.getName(), timeout, 2, + r -> className + r.getName(), + () -> initialized.getAndSet(true) ? timeout : TimeDuration.ONE_MINUTE.add(timeout), + 2, i -> stub.withInterceptors(i).hello(responseHandler)); } diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestServer.java b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestServer.java index d497ac1cd0..345c565f30 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestServer.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcTestServer.java @@ -23,6 +23,8 @@ import org.apache.ratis.thirdparty.io.grpc.Server; import org.apache.ratis.thirdparty.io.grpc.ServerBuilder; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; +import org.apache.ratis.thirdparty.io.netty.util.concurrent.ThreadPerTaskExecutor; +import org.apache.ratis.util.Daemon; import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; @@ -31,16 +33,22 @@ import java.io.Closeable; import java.io.IOException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** gRPC server for testing */ class GrpcTestServer implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(GrpcTestServer.class); + private static final AtomicLong COUNTER = new AtomicLong(); private final Server server; - GrpcTestServer(int port, int slow, TimeDuration timeout) { + GrpcTestServer(int port, int warmup, int slow, TimeDuration timeout) { this.server = ServerBuilder.forPort(port) - .addService(new GreeterImpl(slow, timeout)) + .executor(new ThreadPerTaskExecutor(r -> Daemon.newBuilder() + .setName("test-server-" + COUNTER.getAndIncrement()) + .setRunnable(r) + .build())) + .addService(new GreeterImpl(warmup, slow, timeout)) .build(); } @@ -64,14 +72,16 @@ static String toReplySuffix(String request) { return ") Hello " + request; } + private final int warmup; private final int slow; private final TimeDuration shortSleepTime; private final TimeDuration longSleepTime; private int count = 0; - GreeterImpl(int slow, TimeDuration timeout) { + GreeterImpl(int warmup, int slow, TimeDuration timeout) { + this.warmup = warmup; this.slow = slow; - this.shortSleepTime = timeout.multiply(0.1); + this.shortSleepTime = timeout.multiply(0.25); this.longSleepTime = timeout.multiply(2); } @@ -81,7 +91,8 @@ public StreamObserver hello(StreamObserver responseObs @Override public void onNext(HelloRequest helloRequest) { final String reply = count + toReplySuffix(helloRequest.getName()); - final TimeDuration sleepTime = count < slow ? shortSleepTime : longSleepTime; + final TimeDuration sleepTime = count < warmup ? TimeDuration.ZERO : + count < (warmup + slow) ? shortSleepTime : longSleepTime; LOG.info("count = {}, slow = {}, sleep {}", reply, slow, sleepTime); try { sleepTime.sleep(); @@ -105,4 +116,4 @@ public void onCompleted() { }; } } -} +} \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestClient.java b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestClient.java index 791d5a6d24..ef66b72be2 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestClient.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestClient.java @@ -29,27 +29,24 @@ import org.apache.ratis.thirdparty.io.grpc.ManagedChannelBuilder; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.apache.ratis.util.IOUtils; +import org.apache.ratis.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Objects; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; -import java.util.function.BiFunction; +import java.util.function.Consumer; /** gRPC client for testing */ class GrpcZeroCopyTestClient implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(GrpcZeroCopyTestClient.class); - @FunctionalInterface - interface StreamObserverFactory - extends BiFunction, StreamObserver> { - } - private final ManagedChannel channel; private final StreamObserver helloRequestHandler; @@ -67,7 +64,9 @@ interface StreamObserverFactory final StreamObserver helloResponseHandler = new StreamObserver() { @Override public void onNext(HelloReply helloReply) { - helloReplies.poll().complete(helloReply.getMessage()); + final CompletableFuture polled = helloReplies.poll(); + Objects.requireNonNull(polled, "polled"); + polled.complete(helloReply.getMessage()); } @Override @@ -93,7 +92,9 @@ void completeExceptionally(Throwable throwable) { final StreamObserver binaryResponseHandler = new StreamObserver() { @Override public void onNext(BinaryReply binaryReply) { - binaryReplies.poll().complete(binaryReply.getData()); + final CompletableFuture polled = binaryReplies.poll(); + Objects.requireNonNull(polled, "polled"); + polled.complete(binaryReply.getData()); } @Override @@ -132,29 +133,27 @@ public void close() throws IOException { CompletableFuture send(String name) { LOG.info("send message {}", name); final HelloRequest request = HelloRequest.newBuilder().setName(name).build(); - final CompletableFuture f = new CompletableFuture<>(); - try { - helloRequestHandler.onNext(request); - helloReplies.offer(f); - } catch (IllegalStateException e) { - // already closed - f.completeExceptionally(e); - } - return f; + return send(request, helloReplies, helloRequestHandler::onNext); } CompletableFuture send(ByteBuffer data) { LOG.info("send data: size={}, direct? {}", data.remaining(), data.isDirect()); final BinaryRequest request = BinaryRequest.newBuilder().setData(UnsafeByteOperations.unsafeWrap(data)).build(); - final CompletableFuture f = new CompletableFuture<>(); + return send(request, binaryReplies, binaryRequestHandler::onNext); + } + + static CompletableFuture send(REQUEST request, + Queue> queue, Consumer onNext) { + final CompletableFuture f = new CompletableFuture<>(); + queue.offer(f); try { - binaryRequestHandler.onNext(request); - binaryReplies.offer(f); - } catch (IllegalStateException e) { + onNext.accept(request); + } catch (Exception e) { // already closed f.completeExceptionally(e); + final CompletableFuture polled = queue.poll(); + Preconditions.assertSame(f, polled, "future"); } return f; } - } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestServer.java b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestServer.java index e1bfe4e222..77e234b3ed 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestServer.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/util/GrpcZeroCopyTestServer.java @@ -32,7 +32,7 @@ import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.TraditionalBinaryPrefix; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,12 +75,14 @@ public synchronized String toString() { private final Count zeroCopyCount = new Count(); private final Count nonZeroCopyCount = new Count(); + private final Count releasedCount = new Count(); private final Server server; private final ZeroCopyMessageMarshaller marshaller = new ZeroCopyMessageMarshaller<>( BinaryRequest.getDefaultInstance(), zeroCopyCount::inc, - nonZeroCopyCount::inc); + nonZeroCopyCount::inc, + releasedCount::inc); GrpcZeroCopyTestServer(int port) { final GreeterImpl greeter = new GreeterImpl(); @@ -115,10 +117,10 @@ Count getNonZeroCopyCount() { void assertCounts(int expectNumElements, long expectNumBytes) { LOG.info("ZeroCopyCount = {}", zeroCopyCount); LOG.info("nonZeroCopyCount = {}", nonZeroCopyCount); - Assert.assertEquals("zeroCopyCount.getNumElements()", expectNumElements, zeroCopyCount.getNumElements()); - Assert.assertEquals("zeroCopyCount.getNumBytes()", expectNumBytes, zeroCopyCount.getNumBytes()); - Assert.assertEquals("nonZeroCopyCount.getNumElements()", 0, nonZeroCopyCount.getNumElements()); - Assert.assertEquals("nonZeroCopyCount.getNumBytes()", 0, nonZeroCopyCount.getNumBytes()); + Assertions.assertEquals(expectNumElements, zeroCopyCount.getNumElements(), "zeroCopyCount.getNumElements()"); + Assertions.assertEquals(expectNumBytes, zeroCopyCount.getNumBytes()," zeroCopyCount.getNumBytes()"); + Assertions.assertEquals(0, nonZeroCopyCount.getNumElements(), "nonZeroCopyCount.getNumElements()"); + Assertions.assertEquals(0, nonZeroCopyCount.getNumBytes(), "nonZeroCopyCount.getNumBytes()"); } int start() throws IOException { diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestGrpcZeroCopy.java b/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestGrpcZeroCopy.java index 032a9c1db5..9ffe7ecd08 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestGrpcZeroCopy.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestGrpcZeroCopy.java @@ -26,8 +26,8 @@ import org.apache.ratis.thirdparty.io.netty.buffer.PooledByteBufAllocator; import org.apache.ratis.util.NetUtils; import org.apache.ratis.util.TraditionalBinaryPrefix; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; @@ -39,29 +39,29 @@ */ public final class TestGrpcZeroCopy extends BaseTest { static class RandomData { - private static final Random random = new Random(); - private static final byte[] array = new byte[4096]; + private static final Random RANDOM = new Random(); + private static final byte[] ARRAY = new byte[4096]; static void fill(long seed, int size, ByteBuf buf) { - random.setSeed(seed); + RANDOM.setSeed(seed); for(int offset = 0; offset < size; ) { - final int remaining = Math.min(size - offset, array.length); - random.nextBytes(array); - buf.writeBytes(array, 0, remaining); + final int remaining = Math.min(size - offset, ARRAY.length); + RANDOM.nextBytes(ARRAY); + buf.writeBytes(ARRAY, 0, remaining); offset += remaining; } } static void verify(long seed, ByteString b) { - random.setSeed(seed); + RANDOM.setSeed(seed); final int size = b.size(); for(int offset = 0; offset < size; ) { - final int remaining = Math.min(size - offset, array.length); - random.nextBytes(array); - final ByteString expected = UnsafeByteOperations.unsafeWrap(array, 0, remaining); + final int remaining = Math.min(size - offset, ARRAY.length); + RANDOM.nextBytes(ARRAY); + final ByteString expected = UnsafeByteOperations.unsafeWrap(ARRAY, 0, remaining); final ByteString computed = b.substring(offset, offset + remaining); - Assert.assertEquals(expected.size(), computed.size()); - Assert.assertEquals(expected, computed); + Assertions.assertEquals(expected.size(), computed.size()); + Assertions.assertEquals(expected, computed); offset += remaining; } } @@ -99,7 +99,7 @@ public static boolean isReady() { /** Test a zero-copy marshaller is available from the versions of gRPC and Protobuf. */ @Test public void testReadiness() { - Assert.assertTrue(isReady()); + Assertions.assertTrue(isReady()); } @@ -134,7 +134,7 @@ void sendMessages(int n, GrpcZeroCopyTestClient client, GrpcZeroCopyTestServer s for (int i = 0; i < futures.size(); i++) { final String expected = GrpcZeroCopyTestServer.toReply(i, messages.get(i)); final String reply = futures.get(i).get(); - Assert.assertEquals("expected = " + expected + " != reply = " + reply, expected, reply); + Assertions.assertEquals(expected, reply, "expected = " + expected + " != reply = " + reply); server.assertCounts(numElements, numBytes); } } @@ -159,8 +159,8 @@ void sendBinaries(int n, GrpcZeroCopyTestClient client, GrpcZeroCopyTestServer s } final ByteString reply = future.get(); - Assert.assertEquals(4, reply.size()); - Assert.assertEquals(size, reply.asReadOnlyByteBuffer().getInt()); + Assertions.assertEquals(4, reply.size()); + Assertions.assertEquals(size, reply.asReadOnlyByteBuffer().getInt()); numElements++; numBytes += size; diff --git a/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestStreamObserverWithTimeout.java b/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestStreamObserverWithTimeout.java index 7a32fb96af..b279736f39 100644 --- a/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestStreamObserverWithTimeout.java +++ b/ratis-test/src/test/java/org/apache/ratis/grpc/util/TestStreamObserverWithTimeout.java @@ -24,8 +24,9 @@ import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.StringUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.apache.ratis.util.TimeoutTimer; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.util.ArrayList; @@ -37,6 +38,8 @@ public class TestStreamObserverWithTimeout extends BaseTest { { Slf4jUtils.setLogLevel(ResponseNotifyClientInterceptor.LOG, Level.TRACE); + Slf4jUtils.setLogLevel(StreamObserverWithTimeout.LOG, Level.DEBUG); + Slf4jUtils.setLogLevel(TimeoutTimer.LOG, Level.DEBUG); } enum Type { @@ -57,14 +60,14 @@ StreamObserverFactory createFunction(TimeDuration timeout) { @Test public void testWithDeadline() throws Exception { //the total sleep time is within the deadline - runTestTimeout(8, Type.WithDeadline); + runTestTimeout(2, Type.WithDeadline); } @Test public void testWithDeadlineFailure() { //Expected to have DEADLINE_EXCEEDED testFailureCase("total sleep time is longer than the deadline", - () -> runTestTimeout(12, Type.WithDeadline), + () -> runTestTimeout(5, Type.WithDeadline), ExecutionException.class, StatusRuntimeException.class); } @@ -72,7 +75,7 @@ public void testWithDeadlineFailure() { public void testWithTimeout() throws Exception { //Each sleep time is within the timeout, //Note that the total sleep time is longer than the timeout, but it does not matter. - runTestTimeout(12, Type.WithTimeout); + runTestTimeout(5, Type.WithTimeout); } void runTestTimeout(int slow, Type type) throws Exception { @@ -80,14 +83,20 @@ void runTestTimeout(int slow, Type type) throws Exception { final TimeDuration timeout = ONE_SECOND.multiply(0.5); final StreamObserverFactory function = type.createFunction(timeout); + // first request may take longer due to initialization + final int warmup = type == Type.WithTimeout ? 1 : 0; final List messages = new ArrayList<>(); for (int i = 0; i < 2 * slow; i++) { - messages.add("m" + i); + messages.add("m" + (i + warmup)); } - try (GrpcTestServer server = new GrpcTestServer(NetUtils.getFreePort(), slow, timeout)) { + try (GrpcTestServer server = new GrpcTestServer(NetUtils.getFreePort(), warmup, slow, timeout)) { final int port = server.start(); try (GrpcTestClient client = new GrpcTestClient(NetUtils.LOCALHOST, port, function)) { + if (warmup == 1) { + client.send("warmup").join(); + } + final List> futures = new ArrayList<>(); for (String m : messages) { futures.add(client.send(m)); @@ -95,20 +104,20 @@ void runTestTimeout(int slow, Type type) throws Exception { int i = 0; for (; i < slow; i++) { - final String expected = i + GrpcTestServer.GreeterImpl.toReplySuffix(messages.get(i)); + final String expected = (i + warmup) + GrpcTestServer.GreeterImpl.toReplySuffix(messages.get(i)); final String reply = futures.get(i).get(); - Assert.assertEquals("expected = " + expected + " != reply = " + reply, expected, reply); - LOG.info("{}) passed", i); + Assertions.assertEquals(expected, reply); + LOG.info("{}) passed", (i + warmup)); } for (; i < messages.size(); i++) { final CompletableFuture f = futures.get(i); try { final String reply = f.get(); - Assert.fail(i + ") reply = " + reply + ", " + Assertions.fail((i + warmup) + ") reply = " + reply + ", " + StringUtils.completableFuture2String(f, false)); } catch (ExecutionException e) { - LOG.info("GOOD! {}) {}, {}", i, StringUtils.completableFuture2String(f, true), e); + LOG.info("GOOD! {}) {}", (i + warmup), StringUtils.completableFuture2String(f, true), e); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/netty/TestLeaderElectionWithNetty.java b/ratis-test/src/test/java/org/apache/ratis/netty/TestLeaderElectionWithNetty.java index f84bbb7360..0b77b5beda 100644 --- a/ratis-test/src/test/java/org/apache/ratis/netty/TestLeaderElectionWithNetty.java +++ b/ratis-test/src/test/java/org/apache/ratis/netty/TestLeaderElectionWithNetty.java @@ -19,7 +19,7 @@ import org.apache.ratis.server.impl.BlockRequestHandlingInjection; import org.apache.ratis.server.impl.LeaderElectionTests; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestLeaderElectionWithNetty extends LeaderElectionTests @@ -29,7 +29,7 @@ public class TestLeaderElectionWithNetty public void testEnforceLeader() throws Exception { super.testEnforceLeader(); - MiniRaftClusterWithNetty.sendServerRequest.clear(); + MiniRaftClusterWithNetty.SEND_SERVER_REQUEST.clear(); BlockRequestHandlingInjection.getInstance().unblockAll(); } diff --git a/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftAsyncWithNetty.java b/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftAsyncWithNetty.java new file mode 100644 index 0000000000..ebaa33d505 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftAsyncWithNetty.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.netty; + +import org.apache.ratis.RaftAsyncTests; + +public class TestRaftAsyncWithNetty + extends RaftAsyncTests + implements MiniRaftClusterWithNetty.FactoryGet { +} \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftWithNetty.java b/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftWithNetty.java index 28815d76ce..55b9c37286 100644 --- a/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftWithNetty.java +++ b/ratis-test/src/test/java/org/apache/ratis/netty/TestRaftWithNetty.java @@ -19,7 +19,7 @@ import org.apache.ratis.RaftBasicTests; import org.apache.ratis.server.impl.BlockRequestHandlingInjection; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestRaftWithNetty extends RaftBasicTests diff --git a/ratis-test/src/test/java/org/apache/ratis/netty/TestTlsConfWithNetty.java b/ratis-test/src/test/java/org/apache/ratis/netty/TestTlsConfWithNetty.java index db967e391f..dbdcf1ebd8 100644 --- a/ratis-test/src/test/java/org/apache/ratis/netty/TestTlsConfWithNetty.java +++ b/ratis-test/src/test/java/org/apache/ratis/netty/TestTlsConfWithNetty.java @@ -38,8 +38,8 @@ import org.apache.ratis.thirdparty.io.netty.handler.logging.LoggingHandler; import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContext; import org.apache.ratis.util.JavaUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +58,7 @@ * Testing {@link TlsConf} and the security related utility methods in {@link NettyUtils}. */ public class TestTlsConfWithNetty { - private final static Logger LOG = LoggerFactory.getLogger(TestTlsConfWithNetty.class); + private static final Logger LOG = LoggerFactory.getLogger(TestTlsConfWithNetty.class); static String buffer2String(ByteBuf buf){ try { @@ -112,7 +112,7 @@ static void runTest(int port, TlsConf serverSslConf, TlsConf clientSslConf) thro final CompletableFuture future = replyFutures.get(i); final String reply = future.get(3, TimeUnit.SECONDS); LOG.info(reply); - Assert.assertEquals(NettyTestServer.toReply(words[i]), reply); + Assertions.assertEquals(NettyTestServer.toReply(words[i]), reply); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftGroup.java b/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftGroup.java index 5267b22385..f4a88e40fe 100644 --- a/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftGroup.java +++ b/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftGroup.java @@ -18,27 +18,25 @@ package org.apache.ratis.protocol; import org.apache.ratis.BaseTest; -import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.LinkedList; import java.util.List; import java.util.UUID; +@Timeout(value = 1) public class TestRaftGroup extends BaseTest { - @Override - public int getGlobalTimeoutSeconds() { - return 1; - } - @Test(expected = IllegalStateException.class) + @Test public void testDuplicatePeerId() throws Exception { UUID groupId = UUID.fromString("02511d47-d67c-49a3-9011-abb3109a44c1"); List peers = new LinkedList<>(); peers.add(RaftPeer.newBuilder().setId("n0").build()); peers.add(RaftPeer.newBuilder().setId("n0").build()); - RaftGroup.valueOf(RaftGroupId.valueOf(groupId), peers); + Assertions.assertThrows(IllegalStateException.class, + () -> RaftGroup.valueOf(RaftGroupId.valueOf(groupId), peers)); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftId.java b/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftId.java index 6610b3d049..907235e117 100644 --- a/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftId.java +++ b/ratis-test/src/test/java/org/apache/ratis/protocol/TestRaftId.java @@ -19,15 +19,21 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; -import org.junit.Assert; -import org.junit.Test; +import org.apache.ratis.util.WeakValueCache; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.UUID; +@Timeout(value = 1) public class TestRaftId extends BaseTest { - @Override - public int getGlobalTimeoutSeconds() { - return 1; + public static WeakValueCache getClientIdCache() { + return ClientId.getCache(); + } + + public static WeakValueCache getRaftGroupIdCache() { + return RaftGroupId.getCache(); } @Test @@ -39,35 +45,35 @@ public void testRaftId() { static void assertRaftId(UUID original, ByteString expected) { final ByteString bytes = RaftId.toByteString(original); if (expected != null) { - Assert.assertEquals(expected, bytes); + Assertions.assertEquals(expected, bytes); } final UUID computed = RaftId.toUuid(bytes); - Assert.assertEquals(original, computed); - Assert.assertEquals(bytes, RaftId.toByteString(computed)); + Assertions.assertEquals(original, computed); + Assertions.assertEquals(bytes, RaftId.toByteString(computed)); } @Test public void testClientId() { final ClientId id = ClientId.randomId(); final ByteString bytes = id.toByteString(); - Assert.assertEquals(bytes, id.toByteString()); - Assert.assertEquals(id, ClientId.valueOf(bytes)); + Assertions.assertEquals(bytes, id.toByteString()); + Assertions.assertEquals(id, ClientId.valueOf(bytes)); } @Test public void testRaftGroupId() { final RaftGroupId id = RaftGroupId.randomId(); final ByteString bytes = id.toByteString(); - Assert.assertEquals(bytes, id.toByteString()); - Assert.assertEquals(id, RaftGroupId.valueOf(bytes)); + Assertions.assertEquals(bytes, id.toByteString()); + Assertions.assertEquals(id, RaftGroupId.valueOf(bytes)); } @Test public void testRaftPeerId() { final RaftPeerId id = RaftPeerId.valueOf("abc"); final ByteString bytes = id.toByteString(); - Assert.assertEquals(bytes, id.toByteString()); - Assert.assertEquals(id, RaftPeerId.valueOf(bytes)); + Assertions.assertEquals(bytes, id.toByteString()); + Assertions.assertEquals(id, RaftPeerId.valueOf(bytes)); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/protocol/TestRoutingTable.java b/ratis-test/src/test/java/org/apache/ratis/protocol/TestRoutingTable.java index 58bdf07fac..f3e08ece7a 100644 --- a/ratis-test/src/test/java/org/apache/ratis/protocol/TestRoutingTable.java +++ b/ratis-test/src/test/java/org/apache/ratis/protocol/TestRoutingTable.java @@ -18,14 +18,12 @@ package org.apache.ratis.protocol; import org.apache.ratis.BaseTest; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +@Timeout(value = 1) public class TestRoutingTable extends BaseTest { - @Override - public int getGlobalTimeoutSeconds() { - return 1; - } private final RaftPeerId[] peers = new RaftPeerId[10]; @@ -69,7 +67,7 @@ RoutingTable newRoutingTable(int... peerIndices) { } void testFailureCase(String name, int... peerIndices) { - Assert.assertEquals(0, peerIndices.length % 2); + Assertions.assertEquals(0, peerIndices.length % 2); testFailureCase(name + ": " + toString(peerIndices), () -> newRoutingTable(peerIndices), @@ -77,7 +75,7 @@ void testFailureCase(String name, int... peerIndices) { } String toString(int... peerIndices) { - Assert.assertEquals(0, peerIndices.length % 2); + Assertions.assertEquals(0, peerIndices.length % 2); if (peerIndices.length == 0) { return ""; } diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java index 7eba6a82b9..560c0f9fdd 100644 --- a/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java +++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestExceptionDependentRetry.java @@ -21,7 +21,6 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.server.impl.MiniRaftCluster; import org.apache.ratis.RaftTestUtil; -import org.apache.ratis.client.retry.ClientRetryEvent; import org.apache.ratis.client.RaftClient; import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.conf.RaftProperties; @@ -33,16 +32,14 @@ import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import static org.junit.Assert.fail; - /** * Class to test {@link ExceptionDependentRetry}. */ @@ -99,9 +96,9 @@ public void testExceptionDependentRetryFailureWithExceptionDuplicate() { builder.setExceptionToPolicy(IOException.class, RetryPolicies.retryUpToMaximumCountWithFixedSleep(1, TimeDuration.valueOf(1, TimeUnit.SECONDS))); - fail("testExceptionDependentRetryFailure failed"); + Assertions.fail("testExceptionDependentRetryFailure failed"); } catch (Exception ex) { - Assert.assertEquals(IllegalStateException.class, ex.getClass()); + Assertions.assertEquals(IllegalStateException.class, ex.getClass()); } } @@ -115,9 +112,9 @@ public void testExceptionDependentRetryFailureWithExceptionMappedToNull() { RetryPolicies.retryUpToMaximumCountWithFixedSleep(1, TimeDuration.valueOf(1, TimeUnit.SECONDS))); builder.setExceptionToPolicy(IOException.class, null); - fail("testExceptionDependentRetryFailure failed"); + Assertions.fail("testExceptionDependentRetryFailure failed"); } catch (Exception ex) { - Assert.assertEquals(IllegalStateException.class, ex.getClass()); + Assertions.assertEquals(IllegalStateException.class, ex.getClass()); } } @@ -131,9 +128,9 @@ public void testExceptionDependentRetryFailureWithNoDefault() { RetryPolicies.retryUpToMaximumCountWithFixedSleep(1, TimeDuration.valueOf(1, TimeUnit.SECONDS))); builder.build(); - fail("testExceptionDependentRetryFailureWithNoDefault failed"); + Assertions.fail("testExceptionDependentRetryFailureWithNoDefault failed"); } catch (Exception ex) { - Assert.assertEquals(IllegalStateException.class, ex.getClass()); + Assertions.assertEquals(IllegalStateException.class, ex.getClass()); } try { @@ -143,9 +140,9 @@ public void testExceptionDependentRetryFailureWithNoDefault() { RetryPolicies.retryUpToMaximumCountWithFixedSleep(1, TimeDuration.valueOf(1, TimeUnit.SECONDS))); builder.setDefaultPolicy(null); - fail("testExceptionDependentRetryFailureWithNoDefault failed"); + Assertions.fail("testExceptionDependentRetryFailureWithNoDefault failed"); } catch (Exception ex) { - Assert.assertEquals(IllegalStateException.class, ex.getClass()); + Assertions.assertEquals(IllegalStateException.class, ex.getClass()); } } @@ -154,14 +151,14 @@ private void testException(int retries, int maxAttempts, long sleepTime) { for (int i = 0; i < retries + 1; i++) { RetryPolicy.Action action = exceptionDependentRetry - .handleAttemptFailure(new ClientRetryEvent(i, null, exception)); + .handleAttemptFailure(TestRetryPolicy.newClientRetryEvent(i, null, exception)); final boolean expected = i < retries && i < maxAttempts; - Assert.assertEquals(expected, action.shouldRetry()); + Assertions.assertEquals(expected, action.shouldRetry()); if (expected) { - Assert.assertEquals(sleepTime, action.getSleepTime().getDuration()); + Assertions.assertEquals(sleepTime, action.getSleepTime().getDuration()); } else { - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } } } @@ -178,12 +175,14 @@ public void testExceptionRetryAttempts() throws Exception { } void runTestExceptionRetryAttempts(MiniRaftClusterWithGrpc cluster) throws Exception { - final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster); + final int retryCount = 5; + final RetryPolicy timeoutPolicy = MultipleLinearRandomRetry.parseCommaSeparated("1ms, " + retryCount); final ExceptionDependentRetry policy = ExceptionDependentRetry.newBuilder() - .setExceptionToPolicy(TimeoutIOException.class, MultipleLinearRandomRetry.parseCommaSeparated("1ms, 5")) + .setExceptionToPolicy(TimeoutIOException.class, timeoutPolicy) .setDefaultPolicy(RetryPolicies.retryForeverNoSleep()) .build(); + final RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster); // create a client with the exception dependent policy try (final RaftClient client = cluster.createClient(policy)) { client.async().send(new RaftTestUtil.SimpleMessage("1")).get(); @@ -193,10 +192,11 @@ void runTestExceptionRetryAttempts(MiniRaftClusterWithGrpc cluster) throws Excep SimpleStateMachine4Testing.get(leader).blockWriteStateMachineData(); client.async().send(new RaftTestUtil.SimpleMessage("2")).get(); - Assert.fail("Test should have failed."); + Assertions.fail("Test should have failed."); } catch (ExecutionException e) { RaftRetryFailureException rrfe = (RaftRetryFailureException) e.getCause(); - Assert.assertEquals(16, rrfe.getAttemptCount()); + final int expectedCount = 1 + retryCount; // new request attempt + retry attempts + Assertions.assertEquals(expectedCount, rrfe.getAttemptCount()); } finally { SimpleStateMachine4Testing.get(leader).unblockWriteStateMachineData(); cluster.shutdown(); diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestExponentialBackoffRetry.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestExponentialBackoffRetry.java index 74f1807640..c4d1d3a9c0 100644 --- a/ratis-test/src/test/java/org/apache/ratis/retry/TestExponentialBackoffRetry.java +++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestExponentialBackoffRetry.java @@ -19,8 +19,8 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.concurrent.TimeUnit; @@ -36,12 +36,12 @@ public void testExponentialBackoffRetry() { // Test maxAttempts ExponentialBackoffRetry retryPolicy = createPolicy(baseSleep, null, 1); - Assert.assertFalse(retryPolicy.handleAttemptFailure(() -> 1).shouldRetry()); + Assertions.assertFalse(retryPolicy.handleAttemptFailure(() -> 1).shouldRetry()); try { // baseSleep should not be null createPolicy(null, null, 1); - Assert.fail("Policy creation should have failed"); + Assertions.fail("Policy creation should have failed"); } catch (Exception e) { } @@ -67,9 +67,9 @@ private void assertSleep(ExponentialBackoffRetry retryPolicy, // sleep time with randomness added long randomizedDuration = action.getSleepTime().toLong(TimeUnit.MILLISECONDS); - Assert.assertTrue(action.shouldRetry()); - Assert.assertTrue(randomizedDuration >= d * 0.5); - Assert.assertTrue(randomizedDuration <= d * 1.5); + Assertions.assertTrue(action.shouldRetry()); + Assertions.assertTrue(randomizedDuration >= d * 0.5); + Assertions.assertTrue(randomizedDuration <= d * 1.5); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestMultipleLinearRandomRetry.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestMultipleLinearRandomRetry.java index 09ae55f33d..77bcb70f74 100644 --- a/ratis-test/src/test/java/org/apache/ratis/retry/TestMultipleLinearRandomRetry.java +++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestMultipleLinearRandomRetry.java @@ -19,15 +19,12 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +@Timeout(value = 1) public class TestMultipleLinearRandomRetry extends BaseTest { - @Override - public int getGlobalTimeoutSeconds() { - return 1; - } - @Test public void testParseCommaSeparated() { assertIllegalInput(""); @@ -49,12 +46,12 @@ public void testParseCommaSeparated() { private static void assertIllegalInput(String input) { final MultipleLinearRandomRetry computed = MultipleLinearRandomRetry.parseCommaSeparated(input); - Assert.assertNull(computed); + Assertions.assertNull(computed); } private static MultipleLinearRandomRetry assertLegalInput(String expected, String input) { final MultipleLinearRandomRetry computed = MultipleLinearRandomRetry.parseCommaSeparated(input); - Assert.assertNotNull(computed); - Assert.assertTrue(computed.toString().endsWith(expected)); + Assertions.assertNotNull(computed); + Assertions.assertTrue(computed.toString().endsWith(expected)); return computed; } @@ -69,18 +66,18 @@ public void testMultipleLinearRandomRetry() { for (int j = 1; j <= counts[i]; j++) { final int attempt = ++k; final RetryPolicy.Action action = r.handleAttemptFailure(() -> attempt); - Assert.assertTrue(action.shouldRetry()); + Assertions.assertTrue(action.shouldRetry()); final TimeDuration randomized = action.getSleepTime(); final TimeDuration expected = times[i].to(randomized.getUnit()); final long d = expected.getDuration(); LOG.info("times[{},{}] = {}, randomized={}", i, j, times[i], randomized); - Assert.assertTrue(randomized.getDuration() >= d*0.5); - Assert.assertTrue(randomized.getDuration() < (d*1.5 + precision)); + Assertions.assertTrue(randomized.getDuration() >= d*0.5); + Assertions.assertTrue(randomized.getDuration() < (d*1.5 + precision)); } } final int attempt = ++k; final RetryPolicy.Action action = r.handleAttemptFailure(() -> attempt); - Assert.assertFalse(action.shouldRetry()); + Assertions.assertFalse(action.shouldRetry()); } } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java b/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java index d69cd1a2e6..43b2fedd1a 100644 --- a/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java +++ b/ratis-test/src/test/java/org/apache/ratis/retry/TestRetryPolicy.java @@ -33,8 +33,10 @@ import org.apache.ratis.protocol.exceptions.TimeoutIOException; import org.apache.ratis.protocol.exceptions.ResourceUnavailableException; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.apache.ratis.util.Timestamp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.IOException; import java.util.ArrayList; @@ -44,12 +46,8 @@ import java.util.concurrent.TimeUnit; /** Test {@link RetryPolicy}. */ +@Timeout(value = 1) public class TestRetryPolicy extends BaseTest { - @Override - public int getGlobalTimeoutSeconds() { - return 1; - } - @Test public void testRetryMultipleTimesWithFixedSleep() { final int n = 4; @@ -61,15 +59,19 @@ public void testRetryMultipleTimesWithFixedSleep() { final RetryPolicy.Action action = policy.handleAttemptFailure(event); final boolean expected = i < n; - Assert.assertEquals(expected, action.shouldRetry()); + Assertions.assertEquals(expected, action.shouldRetry()); if (expected) { - Assert.assertEquals(sleepTime, action.getSleepTime()); + Assertions.assertEquals(sleepTime, action.getSleepTime()); } else { - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } } } + static ClientRetryEvent newClientRetryEvent(int attemptCount, RaftClientRequest request, Throwable cause) { + return new ClientRetryEvent(attemptCount, request, attemptCount, cause, Timestamp.currentTime()); + } + @Test public void testRequestTypeDependentRetry() { final RequestTypeDependentRetryPolicy.Builder b = RequestTypeDependentRetryPolicy.newBuilder(); @@ -88,37 +90,37 @@ public void testRequestTypeDependentRetry() { RaftClientRequest.watchRequestType(1, ReplicationLevel.MAJORITY)); for(int i = 1; i < 2*n; i++) { { //write - final ClientRetryEvent event = new ClientRetryEvent(i, writeRequest, null); + final ClientRetryEvent event = newClientRetryEvent(i, writeRequest, null); final RetryPolicy.Action action = policy.handleAttemptFailure(event); final boolean expected = i < n; - Assert.assertEquals(expected, action.shouldRetry()); + Assertions.assertEquals(expected, action.shouldRetry()); if (expected) { - Assert.assertEquals(writeSleep, action.getSleepTime()); + Assertions.assertEquals(writeSleep, action.getSleepTime()); } else { - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } } { //read and stale read are using default - final ClientRetryEvent event = new ClientRetryEvent(i, readRequest, null); + final ClientRetryEvent event = newClientRetryEvent(i, readRequest, null); final RetryPolicy.Action action = policy.handleAttemptFailure(event); - Assert.assertTrue(action.shouldRetry()); - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertTrue(action.shouldRetry()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } { - final ClientRetryEvent event = new ClientRetryEvent(i, staleReadRequest, null); + final ClientRetryEvent event = newClientRetryEvent(i, staleReadRequest, null); final RetryPolicy.Action action = policy.handleAttemptFailure(event); - Assert.assertTrue(action.shouldRetry()); - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertTrue(action.shouldRetry()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } { //watch has no retry - final ClientRetryEvent event = new ClientRetryEvent(i, watchRequest, null); + final ClientRetryEvent event = newClientRetryEvent(i, watchRequest, null); final RetryPolicy.Action action = policy.handleAttemptFailure(event); - Assert.assertFalse(action.shouldRetry()); - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertFalse(action.shouldRetry()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } } @@ -148,17 +150,17 @@ public RaftClientRequest newRequestImpl() { }; for (RaftClientRequest request : requests) { - final ClientRetryEvent event = new ClientRetryEvent(request, new Exception(), pending); + final ClientRetryEvent event = pending.newClientRetryEvent(request, new Exception()); final RetryPolicy.Action action = policy.handleAttemptFailure(event); - Assert.assertTrue(action.shouldRetry()); - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertTrue(action.shouldRetry()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } timeout.sleep(); for (RaftClientRequest request : requests) { - final ClientRetryEvent event = new ClientRetryEvent(request, new Exception(), pending); + final ClientRetryEvent event = pending.newClientRetryEvent(request, new Exception()); final RetryPolicy.Action action = policy.handleAttemptFailure(event); - Assert.assertFalse(action.shouldRetry()); + Assertions.assertFalse(action.shouldRetry()); } } @@ -218,16 +220,15 @@ public void testRequestTypeDependentRetryWithExceptionDependentPolicy() throws E */ private void checkEvent(int exceptionAttemptCount, RetryPolicy retryPolicy, RaftClientRequest raftClientRequest, Throwable exception, Pair exceptionPolicyPair) { - final ClientRetryEvent event = - new ClientRetryEvent(exceptionAttemptCount, raftClientRequest, exception); + final ClientRetryEvent event = newClientRetryEvent(exceptionAttemptCount, raftClientRequest, exception); final RetryPolicy.Action action = retryPolicy.handleAttemptFailure(event); final boolean expected = exceptionAttemptCount < exceptionPolicyPair.retries; - Assert.assertEquals(expected, action.shouldRetry()); + Assertions.assertEquals(expected, action.shouldRetry()); if (expected) { - Assert.assertEquals(exceptionPolicyPair.sleepTime, action.getSleepTime().getDuration()); + Assertions.assertEquals(exceptionPolicyPair.sleepTime, action.getSleepTime().getDuration()); } else { - Assert.assertEquals(0L, action.getSleepTime().getDuration()); + Assertions.assertEquals(0L, action.getSleepTime().getDuration()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/security/SecurityTestUtils.java b/ratis-test/src/test/java/org/apache/ratis/security/SecurityTestUtils.java index d6222b2271..cbb8e1b799 100644 --- a/ratis-test/src/test/java/org/apache/ratis/security/SecurityTestUtils.java +++ b/ratis-test/src/test/java/org/apache/ratis/security/SecurityTestUtils.java @@ -23,7 +23,7 @@ import org.apache.ratis.util.FileUtils; import org.bouncycastle.util.io.pem.PemObject; import org.bouncycastle.util.io.pem.PemReader; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +54,23 @@ public interface SecurityTestUtils { ClassLoader CLASS_LOADER = SecurityTestUtils.class.getClassLoader(); + TrustManager EMPTY_TRUST_MANAGER = new X509TrustManager() { + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + + @Override + public void checkClientTrusted(X509Certificate[] certs, String authType) { } + + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) { } + }; + + static TrustManager emptyTrustManager() { + return EMPTY_TRUST_MANAGER; + } + static File getResource(String name) { final File file = Optional.ofNullable(CLASS_LOADER.getResource(name)) .map(URL::getFile) @@ -100,7 +117,7 @@ static PrivateKey getPrivateKey(String keyPath) { KeyFactory keyFactory = KeyFactory.getInstance("RSA"); return keyFactory.generatePrivate(privKeySpec); } catch (Exception e) { - Assert.fail("Failed to get private key from " + keyPath + ". Error: " + + Assertions.fail("Failed to get private key from " + keyPath + ". Error: " + e.getMessage()); } return null; @@ -116,7 +133,7 @@ static X509Certificate[] getCertificate(String certPath) { } return certificate; } catch (Exception e) { - Assert.fail("Failed to get certificate from " + certPath + ". Error: " + + Assertions.fail("Failed to get certificate from " + certPath + ". Error: " + e.getMessage()); } return null; @@ -133,7 +150,7 @@ static KeyStore getServerKeyStore() { keyStore.setKeyEntry("ratis-server-key", privateKey, new char[0], certificate); return keyStore; } catch (Exception e) { - Assert.fail("Failed to get sever key store " + e.getMessage()); + Assertions.fail("Failed to get sever key store " + e.getMessage()); } return null; } @@ -149,7 +166,7 @@ static KeyStore getClientKeyStore() { keyStore.setKeyEntry("ratis-client-key", privateKey, new char[0], certificate); return keyStore; } catch (Exception e) { - Assert.fail("Failed to get client key store " + e.getMessage()); + Assertions.fail("Failed to get client key store " + e.getMessage()); } return null; } @@ -167,7 +184,7 @@ static KeyStore getTrustStore() { } return trustStore; } catch (Exception e) { - Assert.fail("Failed to get sever key store " + e.getMessage()); + Assertions.fail("Failed to get sever key store " + e.getMessage()); } return null; } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/ServerBuilderTest.java b/ratis-test/src/test/java/org/apache/ratis/server/ServerBuilderTest.java index dd76a2ec1e..58d553367c 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/ServerBuilderTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/ServerBuilderTest.java @@ -26,8 +26,8 @@ import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.statemachine.impl.BaseStateMachine; import org.apache.ratis.util.Preconditions; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; /** * Test {@link RaftServer.Builder}. @@ -58,9 +58,9 @@ public void testPeerIdNotInRaftGroup() { .setStateMachine(new BaseStateMachine()) .setProperties(new RaftProperties()) .build(); - Assert.fail("did not get expected exception"); + Assertions.fail("did not get expected exception"); } catch (IOException e) { - Preconditions.assertInstanceOf(e.getCause(), IllegalStateException.class); + Preconditions.assertInstanceOf(e.getCause(), NullPointerException.class); } } @@ -74,9 +74,9 @@ public void testNullPeerIdWithRaftGroup() { .setStateMachine(new BaseStateMachine()) .setProperties(new RaftProperties()) .build(); - Assert.fail("did not get expected exception"); + Assertions.fail("did not get expected exception"); } catch (IOException e) { - Preconditions.assertInstanceOf(e.getCause(), IllegalStateException.class); + Preconditions.assertInstanceOf(e.getCause(), NullPointerException.class); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java b/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java index 73ff1eb53e..71be6884ac 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/ServerRestartTests.java @@ -35,8 +35,8 @@ import org.apache.ratis.server.raftlog.RaftLogIOException; import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogFormat; import org.apache.ratis.server.RaftServerConfigKeys.Log; +import org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtils; import org.apache.ratis.server.raftlog.segmented.TestSegmentedRaftLog; -import org.apache.ratis.server.raftlog.segmented.LogSegmentPath; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.FileUtils; @@ -46,8 +46,8 @@ import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.StringUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.event.Level; @@ -63,7 +63,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import java.util.stream.Collectors; + +import static org.apache.ratis.server.storage.RaftStorageTestUtils.getLogUnsafe; /** * Test restarting raft peers. @@ -111,7 +112,7 @@ void runTestRestartFollower(MiniRaftCluster cluster) throws Exception { // make sure the restarted follower can catchup final RaftServer.Division followerState = cluster.getDivision(followerId); JavaUtils.attemptRepeatedly(() -> { - Assert.assertTrue(followerState.getInfo().getLastAppliedIndex() >= leaderLastIndex); + Assertions.assertTrue(followerState.getInfo().getLastAppliedIndex() >= leaderLastIndex); return null; }, 10, ONE_SECOND, "follower catchup", LOG); @@ -119,9 +120,9 @@ void runTestRestartFollower(MiniRaftCluster cluster) throws Exception { final RaftServer.Division follower = cluster.restartServer(followerId, false); final RaftLog followerLog = follower.getRaftLog(); final long followerLastIndex = followerLog.getLastEntryTermIndex().getIndex(); - Assert.assertTrue(followerLastIndex >= leaderLastIndex); + Assertions.assertTrue(followerLastIndex >= leaderLastIndex); final long leaderFinalIndex = cluster.getLeader().getRaftLog().getLastEntryTermIndex().getIndex(); - Assert.assertEquals(leaderFinalIndex, followerLastIndex); + Assertions.assertEquals(leaderFinalIndex, followerLastIndex); final File followerOpenLogFile = getOpenLogFile(follower); final File leaderOpenLogFile = getOpenLogFile(cluster.getDivision(leaderId)); @@ -148,7 +149,7 @@ void runTestRestartFollower(MiniRaftCluster cluster) throws Exception { if (i != truncatedMessageIndex) { final Message m = new SimpleMessage("m" + i); final RaftClientReply reply = client.io().sendReadOnly(m); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); LOG.info("query {}: {} {}", m, reply, LogEntryProto.parseFrom(reply.getMessage().getContent())); } } @@ -159,32 +160,26 @@ static void writeSomething(Supplier newMessage, MiniRaftCluster cluster try(final RaftClient client = cluster.createClient()) { // write some messages for(int i = 0; i < 10; i++) { - Assert.assertTrue(client.io().send(newMessage.get()).isSuccess()); + Assertions.assertTrue(client.io().send(newMessage.get()).isSuccess()); } } } - static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception { + static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) + throws Exception { // truncate log if (openLogFile.length() > 0) { FileUtils.truncateFile(openLogFile, openLogFile.length() - 1); } final RaftServer.Division server = cluster.restartServer(id, false); // the last index should be one less than before - Assert.assertEquals(lastIndex - 1, server.getRaftLog().getLastEntryTermIndex().getIndex()); + Assertions.assertEquals(lastIndex - 1, server.getRaftLog().getLastEntryTermIndex().getIndex()); server.getRaftServer().close(); } - static List getOpenLogFiles(RaftServer.Division server) throws Exception { - return LogSegmentPath.getLogSegmentPaths(server.getRaftStorage()).stream() - .filter(p -> p.getStartEnd().isOpen()) - .map(LogSegmentPath::getPath) - .collect(Collectors.toList()); - } - static File getOpenLogFile(RaftServer.Division server) throws Exception { - final List openLogs = getOpenLogFiles(server); - Assert.assertEquals(1, openLogs.size()); + final List openLogs = SegmentedRaftLogTestUtils.getOpenLogFiles(server); + Assertions.assertEquals(1, openLogs.size()); return openLogs.get(0).toFile(); } @@ -210,20 +205,20 @@ void runTestRestartWithCorruptedLogHeader(MiniRaftCluster cluster) throws Except 10, HUNDRED_MILLIS, impl.getId() + "-getOpenLogFile", LOG); for(int i = 0; i < SegmentedRaftLogFormat.getHeaderLength(); i++) { assertCorruptedLogHeader(impl.getId(), openLogFile, i, cluster, LOG); - Assert.assertTrue(getOpenLogFiles(impl).isEmpty()); + Assertions.assertTrue(SegmentedRaftLogTestUtils.getOpenLogFiles(impl).isEmpty()); } } } static void assertCorruptedLogHeader(RaftPeerId id, File openLogFile, int partialLength, - MiniRaftCluster cluster, Logger LOG) throws Exception { + MiniRaftCluster cluster, Logger log) throws Exception { Preconditions.assertTrue(partialLength < SegmentedRaftLogFormat.getHeaderLength()); try(final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) { final ByteBuffer header = SegmentedRaftLogFormat.getHeaderBytebuffer(); - LOG.info("header = {}", StringUtils.bytes2HexString(header)); + log.info("header = {}", StringUtils.bytes2HexString(header)); final byte[] corrupted = new byte[header.remaining()]; header.get(corrupted, 0, partialLength); - LOG.info("corrupted = {}", StringUtils.bytes2HexString(corrupted)); + log.info("corrupted = {}", StringUtils.bytes2HexString(corrupted)); raf.write(corrupted); } final RaftServer.Division server = cluster.restartServer(id, false); @@ -245,7 +240,7 @@ void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception { final SimpleMessage m = messages[i]; new Thread(() -> { try (final RaftClient client = cluster.createClient()) { - Assert.assertTrue(client.io().send(m).isSuccess()); + Assertions.assertTrue(client.io().send(m).isSuccess()); } catch (IOException e) { throw new IllegalStateException("Failed to send " + m, e); } @@ -261,17 +256,17 @@ void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception { final RaftPeerId leaderId = leader.getId(); ids.add(leaderId); - RaftTestUtil.getStateMachineLogEntries(leaderLog); + RaftTestUtil.getStateMachineLogEntries(leaderLog, LOG::info); // check that the last metadata entry is written to the log JavaUtils.attempt(() -> assertLastLogEntry(leader), 20, HUNDRED_MILLIS, "leader last metadata entry", LOG); final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex(); LOG.info("{}: leader lastIndex={}", leaderId, lastIndex); - final LogEntryProto lastEntry = leaderLog.get(lastIndex); + final LogEntryProto lastEntry = getLogUnsafe(leaderLog, lastIndex); LOG.info("{}: leader lastEntry entry[{}] = {}", leaderId, lastIndex, LogProtoUtils.toLogEntryString(lastEntry)); final long loggedCommitIndex = lastEntry.getMetadataEntry().getCommitIndex(); - final LogEntryProto lastCommittedEntry = leaderLog.get(loggedCommitIndex); + final LogEntryProto lastCommittedEntry = getLogUnsafe(leaderLog, loggedCommitIndex); LOG.info("{}: leader lastCommittedEntry = entry[{}] = {}", leaderId, loggedCommitIndex, LogProtoUtils.toLogEntryString(lastCommittedEntry)); @@ -301,11 +296,11 @@ void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception { final RaftServer.Division server = cluster.getDivision(id); final RaftLog raftLog = server.getRaftLog(); JavaUtils.attemptRepeatedly(() -> { - Assert.assertTrue(raftLog.getLastCommittedIndex() >= loggedCommitIndex); + Assertions.assertTrue(raftLog.getLastCommittedIndex() >= loggedCommitIndex); return null; }, 10, HUNDRED_MILLIS, id + "(commitIndex >= loggedCommitIndex)", LOG); JavaUtils.attemptRepeatedly(() -> { - Assert.assertTrue(server.getInfo().getLastAppliedIndex() >= loggedCommitIndex); + Assertions.assertTrue(server.getInfo().getLastAppliedIndex() >= loggedCommitIndex); return null; }, 10, HUNDRED_MILLIS, id + "(lastAppliedIndex >= loggedCommitIndex)", LOG); LOG.info("{}: commitIndex={}, lastAppliedIndex={}", @@ -317,17 +312,17 @@ void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception { static void assertLastLogEntry(RaftServer.Division server) throws RaftLogIOException { final RaftLog raftLog = server.getRaftLog(); final long lastIndex = raftLog.getLastEntryTermIndex().getIndex(); - final LogEntryProto lastEntry = raftLog.get(lastIndex); - Assert.assertTrue(lastEntry.hasMetadataEntry()); + final LogEntryProto lastEntry = getLogUnsafe(raftLog, lastIndex); + Assertions.assertTrue(lastEntry.hasMetadataEntry()); final long loggedCommitIndex = lastEntry.getMetadataEntry().getCommitIndex(); - final LogEntryProto lastCommittedEntry = raftLog.get(loggedCommitIndex); - Assert.assertTrue(lastCommittedEntry.hasStateMachineLogEntry()); + final LogEntryProto lastCommittedEntry = getLogUnsafe(raftLog, loggedCommitIndex); + Assertions.assertTrue(lastCommittedEntry.hasStateMachineLogEntry()); final SimpleStateMachine4Testing leaderStateMachine = SimpleStateMachine4Testing.get(server); final TermIndex lastAppliedTermIndex = leaderStateMachine.getLastAppliedTermIndex(); - Assert.assertEquals(lastCommittedEntry.getTerm(), lastAppliedTermIndex.getTerm()); - Assert.assertTrue(lastCommittedEntry.getIndex() <= lastAppliedTermIndex.getIndex()); + Assertions.assertEquals(lastCommittedEntry.getTerm(), lastAppliedTermIndex.getTerm()); + Assertions.assertTrue(lastCommittedEntry.getIndex() <= lastAppliedTermIndex.getIndex()); } @Test @@ -364,11 +359,11 @@ private void runTestRestartWithCorruptedLogEntry(CLUSTER cluster) throws Excepti final SimpleMessage lastMessage = messages[messages.length - 1]; try (final RaftClient client = cluster.createClient()) { for (SimpleMessage m : messages) { - Assert.assertTrue(client.io().send(m).isSuccess()); + Assertions.assertTrue(client.io().send(m).isSuccess()); } // assert that the last message exists - Assert.assertTrue(client.io().sendReadOnly(lastMessage).isSuccess()); + Assertions.assertTrue(client.io().sendReadOnly(lastMessage).isSuccess()); } final RaftLog log = leader.getRaftLog(); diff --git a/ratis-test/src/test/java/org/apache/ratis/server/TestRaftServerConfigKeys.java b/ratis-test/src/test/java/org/apache/ratis/server/TestRaftServerConfigKeys.java index bb386e8309..a9e509e7a8 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/TestRaftServerConfigKeys.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/TestRaftServerConfigKeys.java @@ -23,9 +23,9 @@ import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.SizeInBytes; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -46,14 +46,14 @@ */ public class TestRaftServerConfigKeys { - private static final Supplier rootTestDir = JavaUtils.memoize( + private static final Supplier ROOT_TEST_DIR = JavaUtils.memoize( () -> new File(BaseTest.getRootTestDir(), JavaUtils.getClassSimpleName(TestRaftServerConfigKeys.class) + Integer.toHexString(ThreadLocalRandom.current().nextInt()))); - @AfterClass + @AfterAll public static void tearDown() throws IOException { - FileUtils.deleteFully(rootTestDir.get()); + FileUtils.deleteFully(ROOT_TEST_DIR.get()); } /** @@ -63,7 +63,7 @@ public static void tearDown() throws IOException { @Test public void testStorageDirProperty() { final File testDir = new File( - rootTestDir.get(), UUID.randomUUID().toString()); + ROOT_TEST_DIR.get(), UUID.randomUUID().toString()); final List directories = new ArrayList<>(); final RaftProperties properties = new RaftProperties(); @@ -74,7 +74,7 @@ public void testStorageDirProperty() { final String expected = directories.stream().map(File::getAbsolutePath) .collect(Collectors.joining(",")); final String actual = properties.get(RaftServerConfigKeys.STORAGE_DIR_KEY); - Assert.assertEquals(expected, actual); + Assertions.assertEquals(expected, actual); } /** @@ -84,7 +84,7 @@ public void testStorageDirProperty() { @Test public void testStorageDir() { final File testDir = new File( - rootTestDir.get(), UUID.randomUUID().toString()); + ROOT_TEST_DIR.get(), UUID.randomUUID().toString()); final List directories = new ArrayList<>(); IntStream.range(0, 10).mapToObj((i) -> new File(testDir, Integer.toString(i))).forEach(directories::add); @@ -97,8 +97,8 @@ public void testStorageDir() { final List actualDirs = storageDirs.stream() .map(File::getAbsolutePath).collect(Collectors.toList()); actualDirs.removeAll(expectedDirs); - Assert.assertEquals(directories.size(), storageDirs.size()); - Assert.assertEquals(0, actualDirs.size()); + Assertions.assertEquals(directories.size(), storageDirs.size()); + Assertions.assertEquals(0, actualDirs.size()); } /** @@ -113,6 +113,6 @@ public void testStorageDir() { int pendingRequestMegabyteLimit = Math.toIntExact( RaftServerConfigKeys.Write.byteLimit(properties).getSize() / SizeInBytes.ONE_MB.getSize()); - Assert.assertEquals(4096, pendingRequestMegabyteLimit); + Assertions.assertEquals(4096, pendingRequestMegabyteLimit); } } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestLeaderElectionServerInterface.java b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestLeaderElectionServerInterface.java new file mode 100644 index 0000000000..876633db1d --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestLeaderElectionServerInterface.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.server.impl; + +import org.apache.ratis.BaseTest; +import org.apache.ratis.metrics.Timekeeper; +import org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto; +import org.apache.ratis.proto.RaftProtos.RequestVoteRequestProto; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftGroupMemberId; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.server.protocol.TermIndex; +import org.apache.ratis.util.TimeDuration; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class TestLeaderElectionServerInterface extends BaseTest { + private final List peers = IntStream.range(0, 3).boxed() + .map(i -> RaftPeer.newBuilder().setId("s" + i).build()) + .collect(Collectors.toList()); + private final RaftGroup group = RaftGroup.valueOf(RaftGroupId.randomId(), peers); + private final RaftConfigurationImpl conf = RaftConfigurationImpl.newBuilder().setLogEntryIndex(0).setConf(peers).build(); + private final ThreadGroup threadGroup = new ThreadGroup("ServerInterface"); + + private final RaftGroupMemberId candidate = RaftGroupMemberId.valueOf(peers.get(0).getId(), group.getGroupId()); + + LeaderElection.ServerInterface newServerInterface(boolean expectToPass, + Map lastEntries) { + return new LeaderElection.ServerInterface() { + private volatile boolean isAlive = true; + + @Override + public RaftGroupMemberId getMemberId() { + return candidate; + } + + @Override + public boolean isAlive() { + return isAlive; + } + + @Override + public boolean isCandidate() { + return true; + } + + @Override + public long getCurrentTerm() { + final TermIndex lastEntry = getLastEntry(); + return lastEntry != null? lastEntry.getTerm() : TermIndex.INITIAL_VALUE.getTerm(); + } + + @Override + public long getLastCommittedIndex() { + final TermIndex lastEntry = getLastEntry(); + return lastEntry != null? lastEntry.getIndex() : TermIndex.INITIAL_VALUE.getIndex(); + } + + @Override + public TermIndex getLastEntry() { + return lastEntries.get(getId()); + } + + @Override + public boolean isPreVoteEnabled() { + return false; + } + + @Override + public LeaderElection.ConfAndTerm initElection(LeaderElection.Phase phase) { + return new LeaderElection.ConfAndTerm(conf, getCurrentTerm()); + } + + @Override + public RequestVoteReplyProto requestVote(RequestVoteRequestProto r) { + final RaftPeerId voterPeerId = RaftPeerId.valueOf(r.getServerRequest().getReplyId()); + final RaftGroupMemberId voter = RaftGroupMemberId.valueOf(voterPeerId, group.getGroupId()); + final TermIndex lastEntry = lastEntries.get(voterPeerId); + final long term = (lastEntry != null? lastEntry : TermIndex.INITIAL_VALUE).getTerm(); + + // voter replies to candidate + return ServerProtoUtils.toRequestVoteReplyProto(getId(), voter, true, term, false, lastEntry); + } + + @Override + public void changeToLeader() { + assertTrue(expectToPass); + isAlive = false; + } + + @Override + public void rejected(long term, LeaderElection.ResultAndTerm result) { + assertFalse(expectToPass); + isAlive = false; + } + + @Override + public void shutdown() { + fail(); + } + + @Override + public Timekeeper getLeaderElectionTimer() { + final long start = System.nanoTime(); + final Timekeeper.Context context = () -> System.nanoTime() - start; + return () -> context; + } + + @Override + public void onNewLeaderElectionCompletion() { + // no op + } + + @Override + public TimeDuration getRandomElectionTimeout() { + final int millis = 100 + ThreadLocalRandom.current().nextInt(100); + return TimeDuration.valueOf(millis, TimeUnit.MILLISECONDS); + } + + @Override + public ThreadGroup getThreadGroup() { + return threadGroup; + } + }; + } + + @Test + public void testVoterWithEmptyLog() { + // all the candidate and the voters have an empty log + // expect to pass: empty-log-candidate will accept votes from empty-log-voters + runTestVoterWithEmptyLog(true); + + // candidate: non-empty commit + // voter 1 : empty log + // voter 2 : empty log + // expect to fail: non-empty-commit-candidate will NOT accept votes from empty-log-voters + final TermIndex candidateLastEntry = TermIndex.valueOf(2, 9); + runTestVoterWithEmptyLog(false, candidateLastEntry); + + // candidate: non-empty commit + // voter 1 : non-empty log + // voter 2 : empty log + // expect to pass: non-empty-commit-candidate will accept votes from non-empty-log-voters + final TermIndex voterLastEntry = TermIndex.valueOf(2, 7); + runTestVoterWithEmptyLog(true, candidateLastEntry, voterLastEntry); + + // candidate: non-empty log + // voter 1 : older version + // voter 2 : empty log + // expect to pass: non-empty-commit-candidate will accept votes from older-version-voters + runTestVoterWithEmptyLog(true, candidateLastEntry, TermIndex.PROTO_DEFAULT); + } + + void runTestVoterWithEmptyLog(boolean expectToPass, TermIndex... lastEntries) { + LOG.info("expectToPass? {}, lastEntries={}", + expectToPass, lastEntries); + final Map map = new HashMap<>(); + for(int i = 0; i < lastEntries.length; i++) { + map.put(peers.get(i).getId(), lastEntries[i]); + } + final LeaderElection election = new LeaderElection(newServerInterface(expectToPass, map), false); + election.startInForeground(); + } + +} \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestPeerConfiguration.java b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestPeerConfiguration.java index e1adcfa442..df4d53cf0b 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestPeerConfiguration.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestPeerConfiguration.java @@ -20,14 +20,14 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collection; import java.util.stream.Collectors; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestPeerConfiguration extends BaseTest { @Test diff --git a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftConfiguration.java b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftConfiguration.java index 14e0030e6d..fa2c524e71 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftConfiguration.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftConfiguration.java @@ -22,14 +22,14 @@ import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.RaftConfiguration; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.Collection; import java.util.stream.Collectors; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestRaftConfiguration extends BaseTest { @Test @@ -39,8 +39,8 @@ public void testIsHighestPriority() { Integer node3 = 2; PeerConfiguration peerConfig = new PeerConfiguration(raftPeersWithPriority(node1, node2, node3)); RaftConfiguration config = RaftConfigurationImpl.newBuilder().setConf(peerConfig).build(); - RaftPeer[] allRaftPeers = peerConfig.getPeers(RaftProtos.RaftPeerRole.FOLLOWER).toArray(new RaftPeer[peerConfig.getPeers( - RaftProtos.RaftPeerRole.FOLLOWER).size()]); + RaftPeer[] allRaftPeers = peerConfig.getPeers(RaftProtos.RaftPeerRole.FOLLOWER).toArray( + new RaftPeer[peerConfig.getPeers(RaftProtos.RaftPeerRole.FOLLOWER).size()]); // First member should not have highest priority assertFalse(RaftServerTestUtil.isHighestPriority(config, @@ -65,29 +65,29 @@ public void testSingleMode() { RaftConfigurationImpl config = RaftConfigurationImpl.newBuilder() .setConf(new PeerConfiguration(raftPeersWithPriority(1))) .build(); - assertTrue("Peer is in single mode.", config.isSingleMode(RaftPeerId.valueOf("1"))); + assertTrue(config.isSingleMode(RaftPeerId.valueOf("1")), "Peer is in single mode."); config = RaftConfigurationImpl.newBuilder() .setConf(new PeerConfiguration(raftPeersWithPriority(0, 1))) .setOldConf(new PeerConfiguration(raftPeersWithPriority(0))) .build(); - assertTrue("Peer is in single mode.", config.isSingleMode(RaftPeerId.valueOf("0"))); - assertFalse("Peer is a new peer.", config.isSingleMode(RaftPeerId.valueOf("1"))); + assertTrue(config.isSingleMode(RaftPeerId.valueOf("0")), "Peer is in single mode."); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("1")), "Peer is a new peer"); config = RaftConfigurationImpl.newBuilder() .setConf(new PeerConfiguration(raftPeersWithPriority(0, 1))) .build(); - assertFalse("Peer is in ha mode.", config.isSingleMode(RaftPeerId.valueOf("0"))); - assertFalse("Peer is in ha mode.", config.isSingleMode(RaftPeerId.valueOf("1"))); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("0")), "Peer is in ha mode."); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("1")), "Peer is in ha mode."); config = RaftConfigurationImpl.newBuilder() .setConf(new PeerConfiguration(raftPeersWithPriority(0, 1))) .setOldConf(new PeerConfiguration(raftPeersWithPriority(2, 3))) .build(); - assertFalse("Peer is in ha mode.", config.isSingleMode(RaftPeerId.valueOf("0"))); - assertFalse("Peer is in ha mode.", config.isSingleMode(RaftPeerId.valueOf("1"))); - assertFalse("Peer is in ha mode.", config.isSingleMode(RaftPeerId.valueOf("3"))); - assertFalse("Peer is in ha mode.", config.isSingleMode(RaftPeerId.valueOf("4"))); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("0")), "Peer is in ha mode."); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("1")), "Peer is in ha mode."); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("3")), "Peer is in ha mode."); + assertFalse(config.isSingleMode(RaftPeerId.valueOf("4")), "Peer is in ha mode."); } @Test @@ -96,8 +96,8 @@ public void testChangeMajority() { RaftConfigurationImpl config = RaftConfigurationImpl.newBuilder() .setConf(new PeerConfiguration(raftPeersWithPriority(1))) .build(); - assertFalse("Change from single mode to ha mode is not considered as changing majority.", - config.changeMajority(raftPeersWithPriority(1, 2))); + assertFalse(config.changeMajority(raftPeersWithPriority(1, 2)), + "Change from single mode to ha mode is not considered as changing majority."); // Case 2: {1} --> {2}. assertTrue(config.changeMajority(raftPeersWithPriority(2))); diff --git a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftServerJmx.java b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftServerJmx.java index 0bd77e16a7..b24ec72c06 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftServerJmx.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/impl/TestRaftServerJmx.java @@ -24,8 +24,9 @@ import org.apache.ratis.server.RaftServerMXBean; import org.apache.ratis.server.simulation.MiniRaftClusterWithSimulatedRpc; import org.apache.ratis.util.JmxRegister; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import javax.management.JMException; import javax.management.MBeanServer; @@ -38,9 +39,10 @@ import static org.apache.ratis.RaftTestUtil.waitForLeader; public class TestRaftServerJmx extends BaseTest { - @Test(timeout = 30000) + @Test + @Timeout(value = 30) public void testJmxBeans() throws Exception { - final int NUM_SERVERS = 3; + final int numServers = 3; final MiniRaftClusterWithSimulatedRpc cluster = MiniRaftClusterWithSimulatedRpc.FACTORY.newCluster(3, new RaftProperties()); cluster.start(); @@ -48,16 +50,17 @@ public void testJmxBeans() throws Exception { MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer(); Set objectInstances = platformMBeanServer.queryMBeans(new ObjectName("Ratis:*"), null); - Assert.assertEquals(NUM_SERVERS, objectInstances.size()); + Assertions.assertEquals(numServers, objectInstances.size()); for (ObjectInstance instance : objectInstances) { Object groupId = platformMBeanServer.getAttribute(instance.getObjectName(), "GroupId"); - Assert.assertEquals(cluster.getGroupId().toString(), groupId); + Assertions.assertEquals(cluster.getGroupId().toString(), groupId); } cluster.shutdown(); } - @Test(timeout = 30000) + @Test + @Timeout(value = 30) public void testRegister() throws JMException { { final JmxRegister jmx = new JmxRegister(); @@ -84,29 +87,42 @@ public void testRegister() throws JMException { static void runRegister(boolean expectToSucceed, String name, JmxRegister jmx) { final RaftServerMXBean mBean = new RaftServerMXBean() { @Override - public String getId() { return null; } + public String getId() { + return null; + } @Override - public String getLeaderId() { return null; } + public String getLeaderId() { + return null; + } @Override - public long getCurrentTerm() { return 0; } + public long getCurrentTerm() { + return 0; + } @Override - public String getGroupId() { return null; } + public String getGroupId() { + return null; + } @Override - public String getRole() { return null; } + public String getRole() { + return null; + } @Override - public List getFollowers() { return null; } + public List getFollowers() { + return null; + } @Override - public List getGroups() { return null; } - + public List getGroups() { + return null; + } }; - final RaftPeerId id = RaftPeerId.valueOf(name); - final RaftGroupId groupId = RaftGroupId.randomId(); - final boolean succeeded = RaftServerImpl.registerMBean(id, groupId, mBean, jmx); - Assert.assertEquals(expectToSucceed, succeeded); + final String id = RaftPeerId.valueOf(name).toString(); + final String groupId = RaftGroupId.randomId().toString(); + final boolean succeeded = RaftServerJmxAdapter.registerMBean(id, groupId, mBean, jmx); + Assertions.assertEquals(expectToSucceed, succeeded); } static void runUnregister(boolean expectToSucceed, JmxRegister jmx) throws JMException { final boolean succeeded = jmx.unregister(); - Assert.assertEquals(expectToSucceed, succeeded); + Assertions.assertEquals(expectToSucceed, succeeded); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/protocol/ProtocolTestUtils.java b/ratis-test/src/test/java/org/apache/ratis/server/protocol/ProtocolTestUtils.java new file mode 100644 index 0000000000..dee3f224c4 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/server/protocol/ProtocolTestUtils.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.server.protocol; + +import org.apache.ratis.util.BiWeakValueCache; + +public interface ProtocolTestUtils { + static BiWeakValueCache getTermIndexCache() { + return TermIndex.Impl.getCache(); + } +} \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogIndex.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogIndex.java index ac655222ba..d3b4b5ccb0 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogIndex.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogIndex.java @@ -18,8 +18,8 @@ package org.apache.ratis.server.raftlog; import org.apache.ratis.BaseTest; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -34,10 +34,10 @@ static void assertUpdate(RaftLogIndex index, BiFunction update, long oldValue, LongUnaryOperator op, boolean expectUpdate) { - Assert.assertEquals(oldValue, index.get()); + Assertions.assertEquals(oldValue, index.get()); final boolean updated = update.apply(index, op); - Assert.assertEquals(expectUpdate, updated); - Assert.assertEquals(expectUpdate? op.applyAsLong(oldValue): oldValue, index.get()); + Assertions.assertEquals(expectUpdate, updated); + Assertions.assertEquals(expectUpdate? op.applyAsLong(oldValue): oldValue, index.get()); } @@ -45,7 +45,7 @@ static void assertUpdate(RaftLogIndex index, BiFunction log = System.out::println; { // test updateIncreasingly diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java index 70b185e554..6f57d42a5d 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/TestRaftLogMetrics.java @@ -36,8 +36,8 @@ import org.apache.ratis.statemachine.impl.BaseStateMachine; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import javax.management.ObjectName; import java.lang.management.ManagementFactory; @@ -110,13 +110,14 @@ static void runTestRaftLogMetrics(MiniRaftCluster cluster) throws Exception { } // Wait for commits to happen on leader - JavaUtils.attempt(() -> assertCommitCount(cluster.getLeader(), numMsg), 10, HUNDRED_MILLIS, cluster.getLeader().getId() + "-assertCommitCount", null); + JavaUtils.attempt(() -> assertCommitCount(cluster.getLeader(), numMsg), 10, HUNDRED_MILLIS, + cluster.getLeader().getId() + "-assertCommitCount", null); } static void assertCommitCount(RaftServer.Division server, int expectedMsgs) { final RatisMetricRegistry rlm = ((RatisMetrics)server.getRaftLog().getRaftLogMetrics()).getRegistry(); long stmCount = rlm.counter(STATE_MACHINE_LOG_ENTRY_COUNT).getCount(); - Assert.assertEquals(expectedMsgs, stmCount); + Assertions.assertEquals(expectedMsgs, stmCount); } static RatisMetricRegistryImpl getRegistry(RaftGroupMemberId memberId) { @@ -127,67 +128,71 @@ static void assertFlushCount(RaftServer.Division server) throws Exception { final String flushTimeMetric = RaftStorageTestUtils.getLogFlushTimeMetric(server.getMemberId().toString()); final RatisMetricRegistryImpl ratisMetricRegistry = getRegistry(server.getMemberId()); Timer tm = (Timer) ratisMetricRegistry.get(RAFT_LOG_FLUSH_TIME); - Assert.assertNotNull(tm); + Assertions.assertNotNull(tm); final MetricsStateMachine stateMachine = MetricsStateMachine.get(server); final int expectedFlush = stateMachine.getFlushCount(); JavaUtils.attemptRepeatedly(() -> { - Assert.assertEquals(expectedFlush, tm.getCount()); + Assertions.assertEquals(expectedFlush, tm.getCount()); return null; }, 50, HUNDRED_MILLIS, "expectedFlush == tm.getCount()", null); - Assert.assertTrue(tm.getMeanRate() > 0); + Assertions.assertTrue(tm.getMeanRate() > 0); // Test jmx ObjectName oname = new ObjectName(RATIS_APPLICATION_NAME_METRICS, "name", flushTimeMetric); - Assert.assertEquals(expectedFlush, + Assertions.assertEquals(expectedFlush, ((Long) ManagementFactory.getPlatformMBeanServer().getAttribute(oname, "Count")) .intValue()); } static void assertRaftLogWritePathMetrics(RaftServer.Division server) throws Exception { - final String syncTimeMetric = RaftStorageTestUtils.getRaftLogFullMetric(server.getMemberId().toString(), RAFT_LOG_SYNC_TIME); + final String syncTimeMetric = RaftStorageTestUtils.getRaftLogFullMetric(server.getMemberId().toString(), + RAFT_LOG_SYNC_TIME); final RatisMetricRegistryImpl ratisMetricRegistry = getRegistry(server.getMemberId()); //Test sync count Timer tm = (Timer) ratisMetricRegistry.get(RAFT_LOG_SYNC_TIME); - Assert.assertNotNull(tm); + Assertions.assertNotNull(tm); final MetricsStateMachine stateMachine = MetricsStateMachine.get(server); final int expectedFlush = stateMachine.getFlushCount(); - Assert.assertEquals(expectedFlush, tm.getCount()); // Ideally, flushCount should be same as syncCount. - Assert.assertTrue(tm.getMeanRate() > 0); + Assertions.assertEquals(expectedFlush, tm.getCount()); // Ideally, flushCount should be same as syncCount. + Assertions.assertTrue(tm.getMeanRate() > 0); // Test jmx. Just testing one metric's JMX is good enough. ObjectName oname = new ObjectName(RATIS_APPLICATION_NAME_METRICS, "name", syncTimeMetric); - Assert.assertEquals(expectedFlush, + Assertions.assertEquals(expectedFlush, ((Long) ManagementFactory.getPlatformMBeanServer().getAttribute(oname, "Count")) .intValue()); long cacheMissCount = ratisMetricRegistry.counter(RAFT_LOG_CACHE_MISS_COUNT).getCount(); - Assert.assertEquals(0, cacheMissCount); + Assertions.assertEquals(0, cacheMissCount); long cacheHitsCount = ratisMetricRegistry.counter(RAFT_LOG_CACHE_HIT_COUNT).getCount(); - Assert.assertTrue(cacheHitsCount > 0); + Assertions.assertTrue(cacheHitsCount > 0); - Assert.assertTrue(ratisMetricRegistry.counter(RAFT_LOG_FLUSH_COUNT).getCount() > 0); - Assert.assertTrue(ratisMetricRegistry.counter(RAFT_LOG_APPEND_ENTRY_COUNT).getCount() > 0); + Assertions.assertTrue(ratisMetricRegistry.counter(RAFT_LOG_FLUSH_COUNT).getCount() > 0); + Assertions.assertTrue(ratisMetricRegistry.counter(RAFT_LOG_APPEND_ENTRY_COUNT).getCount() > 0); - final DefaultTimekeeperImpl appendEntry = (DefaultTimekeeperImpl) ratisMetricRegistry.timer(RAFT_LOG_APPEND_ENTRY_LATENCY); - Assert.assertTrue(appendEntry.getTimer().getMeanRate() > 0); + final DefaultTimekeeperImpl appendEntry = + (DefaultTimekeeperImpl) ratisMetricRegistry.timer(RAFT_LOG_APPEND_ENTRY_LATENCY); + Assertions.assertTrue(appendEntry.getTimer().getMeanRate() > 0); - final DefaultTimekeeperImpl taskQueue = (DefaultTimekeeperImpl) ratisMetricRegistry.timer(RAFT_LOG_TASK_QUEUE_TIME); - Assert.assertTrue(taskQueue.getTimer().getMeanRate() > 0); + final DefaultTimekeeperImpl taskQueue = + (DefaultTimekeeperImpl) ratisMetricRegistry.timer(RAFT_LOG_TASK_QUEUE_TIME); + Assertions.assertTrue(taskQueue.getTimer().getMeanRate() > 0); - final DefaultTimekeeperImpl enqueueDelay = (DefaultTimekeeperImpl) ratisMetricRegistry.timer(RAFT_LOG_TASK_ENQUEUE_DELAY); - Assert.assertTrue(enqueueDelay.getTimer().getMeanRate() > 0); + final DefaultTimekeeperImpl enqueueDelay = + (DefaultTimekeeperImpl) ratisMetricRegistry.timer(RAFT_LOG_TASK_ENQUEUE_DELAY); + Assertions.assertTrue(enqueueDelay.getTimer().getMeanRate() > 0); final DefaultTimekeeperImpl write = (DefaultTimekeeperImpl) ratisMetricRegistry.timer( String.format(RAFT_LOG_TASK_EXECUTION_TIME, "writelog")); - Assert.assertTrue(write.getTimer().getMeanRate() > 0); + Assertions.assertTrue(write.getTimer().getMeanRate() > 0); - Assert.assertNotNull(ratisMetricRegistry.get(RAFT_LOG_DATA_QUEUE_SIZE)); - Assert.assertNotNull(ratisMetricRegistry.get(RAFT_LOG_WORKER_QUEUE_SIZE)); - Assert.assertNotNull(ratisMetricRegistry.get(RAFT_LOG_SYNC_BATCH_SIZE)); + Assertions.assertNotNull(ratisMetricRegistry.get(RAFT_LOG_DATA_QUEUE_SIZE)); + Assertions.assertNotNull(ratisMetricRegistry.get(RAFT_LOG_WORKER_QUEUE_SIZE)); + Assertions.assertNotNull(ratisMetricRegistry.get(RAFT_LOG_SYNC_BATCH_SIZE)); } } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLogTest.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLogTest.java index 5d8d090a38..e88699b052 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLogTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/memory/MemoryRaftLogTest.java @@ -17,7 +17,7 @@ */ package org.apache.ratis.server.raftlog.memory; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.util.ArrayList; import java.util.List; @@ -33,8 +33,9 @@ import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.Slf4jUtils; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; public class MemoryRaftLogTest extends BaseTest { @@ -56,11 +57,11 @@ public void testEntryDoNotPerformTruncation() throws Exception { List entries1 = new ArrayList<>(); entries1.add(LogEntryProto.newBuilder().setIndex(0).setTerm(0).build()); entries1.add(LogEntryProto.newBuilder().setIndex(1).setTerm(0).build()); - raftLog.append(entries1).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(entries1)).forEach(CompletableFuture::join); List entries2 = new ArrayList<>(); entries2.add(LogEntryProto.newBuilder().setIndex(0).setTerm(0).build()); - raftLog.append(entries2).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(entries2)).forEach(CompletableFuture::join); final LogEntryHeader[] termIndices = raftLog.getEntries(0, 10); assertEquals(2, termIndices.length); @@ -84,11 +85,11 @@ public void testEntryPerformTruncation() throws Exception { List entries1 = new ArrayList<>(); entries1.add(LogEntryProto.newBuilder().setIndex(0).setTerm(0).build()); entries1.add(LogEntryProto.newBuilder().setIndex(1).setTerm(0).build()); - raftLog.append(entries1).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(entries1)).forEach(CompletableFuture::join); List entries2 = new ArrayList<>(); entries2.add(LogEntryProto.newBuilder().setIndex(0).setTerm(2).build()); - raftLog.append(entries2).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(entries2)).forEach(CompletableFuture::join); final LogEntryHeader[] termIndices = raftLog.getEntries(0, 10); assertEquals(1, termIndices.length); diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestBufferedWriteChannel.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestBufferedWriteChannel.java index c9d7928555..4df5f4992c 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestBufferedWriteChannel.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestBufferedWriteChannel.java @@ -18,9 +18,8 @@ package org.apache.ratis.server.raftlog.segmented; import org.apache.ratis.BaseTest; -import org.apache.ratis.util.StringUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.ByteBuffer; @@ -40,8 +39,8 @@ class FakeFileChannel extends FileChannel { private long forcedPosition = 0; void assertValues(long expectedPosition, long expectedForcedPosition) { - Assert.assertEquals(expectedPosition, position); - Assert.assertEquals(expectedForcedPosition, forcedPosition); + Assertions.assertEquals(expectedPosition, position); + Assertions.assertEquals(expectedForcedPosition, forcedPosition); } @Override @@ -55,7 +54,7 @@ public long read(ByteBuffer[] dsts, int offset, int length) { } @Override - public int read(ByteBuffer dst, long position) { + public int read(ByteBuffer dst, long pos) { throw new UnsupportedOperationException(); } @@ -65,7 +64,7 @@ public long write(ByteBuffer[] srcs, int offset, int length) { } @Override - public int write(ByteBuffer src, long position) { + public int write(ByteBuffer src, long pos) { throw new UnsupportedOperationException(); } @@ -104,27 +103,27 @@ public void force(boolean metaData) { } @Override - public long transferTo(long position, long count, WritableByteChannel target) { + public long transferTo(long pos, long count, WritableByteChannel target) { throw new UnsupportedOperationException(); } @Override - public long transferFrom(ReadableByteChannel src, long position, long count) { + public long transferFrom(ReadableByteChannel src, long pos, long count) { throw new UnsupportedOperationException(); } @Override - public MappedByteBuffer map(MapMode mode, long position, long size) { + public MappedByteBuffer map(MapMode mode, long pos, long size) { throw new UnsupportedOperationException(); } @Override - public FileLock lock(long position, long size, boolean shared) { + public FileLock lock(long pos, long size, boolean shared) { throw new UnsupportedOperationException(); } @Override - public FileLock tryLock(long position, long size, boolean shared) { + public FileLock tryLock(long pos, long size, boolean shared) { throw new UnsupportedOperationException(); } @@ -200,7 +199,7 @@ static void flush(BufferedWriteChannel out, FakeFileChannel fake, AtomicInteger pos, AtomicInteger force) throws IOException { final int existing = out.writeBufferPosition(); out.flush(); - Assert.assertEquals(0, out.writeBufferPosition()); + Assertions.assertEquals(0, out.writeBufferPosition()); pos.addAndGet(existing); force.set(pos.get()); fake.assertValues(pos.get(), force.get()); @@ -213,9 +212,9 @@ static void writeToBuffer(BufferedWriteChannel out, FakeFileChannel fake, Atomic out.writeToBuffer(n, b -> b.put(buffer)); if (existing + n > bufferCapacity) { pos.addAndGet(existing); - Assert.assertEquals(n, out.writeBufferPosition()); + Assertions.assertEquals(n, out.writeBufferPosition()); } else { - Assert.assertEquals(existing + n, out.writeBufferPosition()); + Assertions.assertEquals(existing + n, out.writeBufferPosition()); } fake.assertValues(pos.get(), force.get()); } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestCacheEviction.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestCacheEviction.java index 996f7ef527..ae8bf34323 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestCacheEviction.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestCacheEviction.java @@ -38,9 +38,10 @@ import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.JavaUtils; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.SizeInBytes; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import java.io.File; @@ -52,10 +53,10 @@ import static org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtils.MAX_OP_SIZE; public class TestCacheEviction extends BaseTest { - private static final CacheInvalidationPolicy policy = new CacheInvalidationPolicyDefault(); + private static final CacheInvalidationPolicy POLICY = new CacheInvalidationPolicyDefault(); static LogSegmentList prepareSegments(int numSegments, boolean[] cached, long start, long size) { - Assert.assertEquals(numSegments, cached.length); + Assertions.assertEquals(numSegments, cached.length); final LogSegmentList segments = new LogSegmentList(JavaUtils.getClassSimpleName(TestCacheEviction.class)); for (int i = 0; i < numSegments; i++) { LogSegment s = LogSegment.newCloseSegment(null, start, start + size - 1, MAX_OP_SIZE, null); @@ -76,36 +77,36 @@ public void testBasicEviction() throws Exception { new boolean[]{true, true, true, true, true}, 0, 10); // case 1, make sure we do not evict cache for segments behind local flushed index - List evicted = policy.evict(null, 5, 15, segments, maxCached); - Assert.assertEquals(0, evicted.size()); + List evicted = POLICY.evict(null, 5, 15, segments, maxCached); + Assertions.assertEquals(0, evicted.size()); // case 2, suppose the local flushed index is in the 3rd segment, then we // can evict the first two segment - evicted = policy.evict(null, 25, 30, segments, maxCached); - Assert.assertEquals(2, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(0)); - Assert.assertSame(evicted.get(1), segments.get(1)); + evicted = POLICY.evict(null, 25, 30, segments, maxCached); + Assertions.assertEquals(2, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(0)); + Assertions.assertSame(evicted.get(1), segments.get(1)); // case 3, similar with case 2, but the local applied index is less than // the local flushed index. - evicted = policy.evict(null, 25, 15, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(0)); + evicted = POLICY.evict(null, 25, 15, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(0)); // case 4, the local applied index is very small, then evict cache behind it // first and let the state machine load the segments later - evicted = policy.evict(null, 35, 5, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(2)); + evicted = POLICY.evict(null, 35, 5, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(2)); Mockito.when(segments.get(2).hasCache()).thenReturn(false); - evicted = policy.evict(null, 35, 5, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(1)); + evicted = POLICY.evict(null, 35, 5, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(1)); Mockito.when(segments.get(1).hasCache()).thenReturn(false); - evicted = policy.evict(null, 35, 5, segments, maxCached); - Assert.assertEquals(0, evicted.size()); + evicted = POLICY.evict(null, 35, 5, segments, maxCached); + Assertions.assertEquals(0, evicted.size()); } @Test @@ -116,39 +117,39 @@ public void testEvictionWithFollowerIndices() throws Exception { // case 1, no matter where the followers are, we do not evict segments behind local // flushed index - List evicted = policy.evict(new long[]{20, 40, 40}, 5, 15, segments, + List evicted = POLICY.evict(new long[]{20, 40, 40}, 5, 15, segments, maxCached); - Assert.assertEquals(0, evicted.size()); + Assertions.assertEquals(0, evicted.size()); // case 2, the follower indices are behind the local flushed index - evicted = policy.evict(new long[]{30, 40, 45}, 25, 30, segments, maxCached); - Assert.assertEquals(2, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(0)); - Assert.assertSame(evicted.get(1), segments.get(1)); + evicted = POLICY.evict(new long[]{30, 40, 45}, 25, 30, segments, maxCached); + Assertions.assertEquals(2, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(0)); + Assertions.assertSame(evicted.get(1), segments.get(1)); // case 3, similar with case 3 in basic eviction test - evicted = policy.evict(new long[]{30, 40, 45}, 25, 15, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(0)); + evicted = POLICY.evict(new long[]{30, 40, 45}, 25, 15, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(0)); // case 4, the followers are slower than local flush - evicted = policy.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(0)); + evicted = POLICY.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(0)); Mockito.when(segments.get(0).hasCache()).thenReturn(false); - evicted = policy.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(2)); + evicted = POLICY.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(2)); Mockito.when(segments.get(2).hasCache()).thenReturn(false); - evicted = policy.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); - Assert.assertEquals(1, evicted.size()); - Assert.assertSame(evicted.get(0), segments.get(3)); + evicted = POLICY.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); + Assertions.assertEquals(1, evicted.size()); + Assertions.assertSame(evicted.get(0), segments.get(3)); Mockito.when(segments.get(3).hasCache()).thenReturn(false); - evicted = policy.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); - Assert.assertEquals(0, evicted.size()); + evicted = POLICY.evict(new long[]{15, 45, 45}, 55, 50, segments, maxCached); + Assertions.assertEquals(0, evicted.size()); } @Test @@ -174,21 +175,21 @@ public void testEvictionInSegmentedLog() throws Exception { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); List slist = TestSegmentedRaftLog.prepareRanges(0, maxCachedNum, 7, 0); List entries = generateEntries(slist); - raftLog.append(entries).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(entries)).forEach(CompletableFuture::join); // check the current cached segment number: the last segment is still open - Assert.assertEquals(maxCachedNum - 1, + Assertions.assertEquals(maxCachedNum - 1, raftLog.getRaftLogCache().getCachedSegmentNum()); Mockito.when(info.getLastAppliedIndex()).thenReturn(35L); Mockito.when(info.getFollowerNextIndices()).thenReturn(new long[]{21, 40, 40}); slist = TestSegmentedRaftLog.prepareRanges(maxCachedNum, maxCachedNum + 2, 7, 7 * maxCachedNum); entries = generateEntries(slist); - raftLog.append(entries).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(entries)).forEach(CompletableFuture::join); // check the cached segment number again. since the slowest follower is on // index 21, the eviction should happen and evict 3 segments - Assert.assertEquals(maxCachedNum + 1 - 3, + Assertions.assertEquals(maxCachedNum + 1 - 3, raftLog.getRaftLogCache().getCachedSegmentNum()); } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java index ece17a0524..5e822de445 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestLogSegment.java @@ -37,10 +37,10 @@ import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TraditionalBinaryPrefix; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -60,12 +60,14 @@ * Test basic functionality of {@link LogSegment} */ public class TestLogSegment extends BaseTest { + public static final LogSegmentStartEnd ZERO_START_NULL_END = LogSegmentStartEnd.valueOf(0); + private File storageDir; private long segmentMaxSize; private long preallocatedSize; private int bufferSize; - @Before + @BeforeEach public void setup() { RaftProperties properties = new RaftProperties(); storageDir = getTestDir(); @@ -78,7 +80,7 @@ public void setup() { RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); } - @After + @AfterEach public void tearDown() throws Exception { if (storageDir != null) { FileUtils.deleteFully(storageDir.getParentFile()); @@ -88,7 +90,8 @@ public void tearDown() throws Exception { File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, boolean isLastEntryPartiallyWritten) throws IOException { if (!isOpen) { - Preconditions.assertTrue(!isLastEntryPartiallyWritten, "For closed log, the last entry cannot be partially written."); + Preconditions.assertTrue(!isLastEntryPartiallyWritten, + "For closed log, the last entry cannot be partially written."); } RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); final File file = LogSegmentStartEnd.valueOf(startIndex, startIndex + numEntries - 1, isOpen).getFile(storage); @@ -110,11 +113,11 @@ File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, bool // 0 < truncatedEntrySize < entrySize final long fileLength = file.length(); final long truncatedFileLength = fileLength - (entrySize - truncatedEntrySize); - Assert.assertTrue(truncatedFileLength < fileLength); + Assertions.assertTrue(truncatedFileLength < fileLength); LOG.info("truncate last entry: entry(size={}, truncated={}), file(length={}, truncated={})", entrySize, truncatedEntrySize, fileLength, truncatedFileLength); FileUtils.truncateFile(file, truncatedFileLength); - Assert.assertEquals(truncatedFileLength, file.length()); + Assertions.assertEquals(truncatedFileLength, file.length()); } storage.close(); @@ -128,24 +131,25 @@ static int size(LogEntryProto entry) { static void checkLogSegment(LogSegment segment, long start, long end, boolean isOpen, long totalSize, long term) throws Exception { - Assert.assertEquals(start, segment.getStartIndex()); - Assert.assertEquals(end, segment.getEndIndex()); - Assert.assertEquals(isOpen, segment.isOpen()); - Assert.assertEquals(totalSize, segment.getTotalFileSize()); + Assertions.assertEquals(start, segment.getStartIndex()); + Assertions.assertEquals(end, segment.getEndIndex()); + Assertions.assertEquals(isOpen, segment.isOpen()); + Assertions.assertEquals(totalSize, segment.getTotalFileSize()); long offset = SegmentedRaftLogFormat.getHeaderLength(); for (long i = start; i <= end; i++) { LogSegment.LogRecord record = segment.getLogRecord(i); + Assertions.assertNotNull(record); final TermIndex ti = record.getTermIndex(); - Assert.assertEquals(i, ti.getIndex()); - Assert.assertEquals(term, ti.getTerm()); - Assert.assertEquals(offset, record.getOffset()); + Assertions.assertEquals(i, ti.getIndex()); + Assertions.assertEquals(term, ti.getTerm()); + Assertions.assertEquals(offset, record.getOffset()); - LogEntryProto entry = segment.getEntryFromCache(ti); + ReferenceCountedObject entry = segment.getEntryFromCache(ti); if (entry == null) { - entry = segment.loadCache(record); + entry = segment.loadCache(ti); } - offset += getEntrySize(entry, Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE); + offset += getEntrySize(entry.get(), Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE); } } @@ -174,12 +178,12 @@ private void testLoadSegment(boolean loadInitial, boolean isLastEntryPartiallyWr final File openSegmentFile = prepareLog(true, 0, 100, 0, isLastEntryPartiallyWritten); RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); final LogSegment openSegment = LogSegment.loadSegment(storage, openSegmentFile, - LogSegmentStartEnd.valueOf(0), MAX_OP_SIZE, loadInitial, null, null); + ZERO_START_NULL_END, MAX_OP_SIZE, loadInitial, null, null); final int delta = isLastEntryPartiallyWritten? 1: 0; checkLogSegment(openSegment, 0, 99 - delta, true, openSegmentFile.length(), 0); storage.close(); // for open segment we currently always keep log entries in the memory - Assert.assertEquals(0, openSegment.getLoadingTimes()); + Assertions.assertEquals(0, openSegment.getLoadingTimes()); // load a closed segment (1000-1099) final File closedSegmentFile = prepareLog(false, 1000, 100, 1, false); @@ -187,7 +191,7 @@ private void testLoadSegment(boolean loadInitial, boolean isLastEntryPartiallyWr LogSegmentStartEnd.valueOf(1000, 1099L), MAX_OP_SIZE, loadInitial, null, null); checkLogSegment(closedSegment, 1000, 1099, false, closedSegment.getTotalFileSize(), 1); - Assert.assertEquals(loadInitial ? 0 : 1, closedSegment.getLoadingTimes()); + Assertions.assertEquals(loadInitial ? 0 : 1, closedSegment.getLoadingTimes()); } @Test @@ -208,7 +212,7 @@ public void testAppendEntries() throws Exception { segment.appendToOpenSegment(Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE, ReferenceCountedObject.wrap(entry)); } - Assert.assertTrue(segment.getTotalFileSize() >= max); + Assertions.assertTrue(segment.getTotalFileSize() >= max); checkLogSegment(segment, start, i - 1 + start, true, size, term); } @@ -219,14 +223,14 @@ public void testAppendEntryMetric() throws Exception { final File openSegmentFile = prepareLog(true, 0, 100, 0, true); RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); final LogSegment openSegment = LogSegment.loadSegment(storage, openSegmentFile, - LogSegmentStartEnd.valueOf(0), MAX_OP_SIZE, true, null, raftLogMetrics); + ZERO_START_NULL_END, MAX_OP_SIZE, true, null, raftLogMetrics); checkLogSegment(openSegment, 0, 98, true, openSegmentFile.length(), 0); storage.close(); final DefaultTimekeeperImpl readEntryTimer = (DefaultTimekeeperImpl) raftLogMetrics.getReadEntryTimer(); - Assert.assertNotNull(readEntryTimer); - Assert.assertEquals(100, readEntryTimer.getTimer().getCount()); - Assert.assertTrue(readEntryTimer.getTimer().getMeanRate() > 0); + Assertions.assertNotNull(readEntryTimer); + Assertions.assertEquals(100, readEntryTimer.getTimer().getCount()); + Assertions.assertTrue(readEntryTimer.getTimer().getMeanRate() > 0); } @@ -238,7 +242,7 @@ public void testAppendWithGap() throws Exception { try { LogEntryProto entry = LogProtoUtils.toLogEntryProto(m, 0, 1001); segment.appendToOpenSegment(Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE, ReferenceCountedObject.wrap(entry)); - Assert.fail("should fail since the entry's index needs to be 1000"); + Assertions.fail("should fail since the entry's index needs to be 1000"); } catch (IllegalStateException e) { // the exception is expected. } @@ -249,7 +253,7 @@ public void testAppendWithGap() throws Exception { try { entry = LogProtoUtils.toLogEntryProto(m, 0, 1002); segment.appendToOpenSegment(Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE, ReferenceCountedObject.wrap(entry)); - Assert.fail("should fail since the entry's index needs to be 1001"); + Assertions.fail("should fail since the entry's index needs to be 1001"); } catch (IllegalStateException e) { // the exception is expected. } @@ -269,18 +273,18 @@ public void testTruncate() throws Exception { // truncate an open segment (remove 1080~1099) long newSize = segment.getLogRecord(start + 80).getOffset(); segment.truncate(start + 80); - Assert.assertEquals(80, segment.numOfEntries()); + Assertions.assertEquals(80, segment.numOfEntries()); checkLogSegment(segment, start, start + 79, false, newSize, term); // truncate a closed segment (remove 1050~1079) newSize = segment.getLogRecord(start + 50).getOffset(); segment.truncate(start + 50); - Assert.assertEquals(50, segment.numOfEntries()); + Assertions.assertEquals(50, segment.numOfEntries()); checkLogSegment(segment, start, start + 49, false, newSize, term); // truncate all the remaining entries segment.truncate(start); - Assert.assertEquals(0, segment.numOfEntries()); + Assertions.assertEquals(0, segment.numOfEntries()); checkLogSegment(segment, start, start - 1, false, SegmentedRaftLogFormat.getHeaderLength(), term); } @@ -288,7 +292,7 @@ public void testTruncate() throws Exception { @Test public void testPreallocateSegment() throws Exception { RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File file = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File file = ZERO_START_NULL_END.getFile(storage); final int[] maxSizes = new int[]{1024, 1025, 1024 * 1024 - 1, 1024 * 1024, 1024 * 1024 + 1, 2 * 1024 * 1024 - 1, 2 * 1024 * 1024, 2 * 1024 * 1024 + 1, 8 * 1024 * 1024}; @@ -300,12 +304,12 @@ public void testPreallocateSegment() throws Exception { for (int a : preallocated) { try(SegmentedRaftLogOutputStream ignored = new SegmentedRaftLogOutputStream(file, false, max, a, ByteBuffer.allocateDirect(bufferSize))) { - Assert.assertEquals("max=" + max + ", a=" + a, file.length(), Math.min(max, a)); + Assertions.assertEquals(file.length(), Math.min(max, a), "max=" + max + ", a=" + a); } try(SegmentedRaftLogInputStream in = SegmentedRaftLogTestUtils.newSegmentedRaftLogInputStream( file, 0, INVALID_LOG_INDEX, true)) { LogEntryProto entry = in.nextEntry(); - Assert.assertNull(entry); + Assertions.assertNull(entry); } } } @@ -321,14 +325,14 @@ public void testPreallocateSegment() throws Exception { size = LogSegment.getEntrySize(entry, Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE); out.write(entry); } - Assert.assertEquals(file.length(), + Assertions.assertEquals(file.length(), size + SegmentedRaftLogFormat.getHeaderLength()); try (SegmentedRaftLogInputStream in = SegmentedRaftLogTestUtils.newSegmentedRaftLogInputStream( file, 0, INVALID_LOG_INDEX, true)) { LogEntryProto entry = in.nextEntry(); - Assert.assertArrayEquals(content, + Assertions.assertArrayEquals(content, entry.getStateMachineLogEntry().getLogData().toByteArray()); - Assert.assertNull(in.nextEntry()); + Assertions.assertNull(in.nextEntry()); } } @@ -339,7 +343,7 @@ public void testPreallocateSegment() throws Exception { public void testPreallocationAndAppend() throws Exception { final SizeInBytes max = SizeInBytes.valueOf(2, TraditionalBinaryPrefix.MEGA); RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File file = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File file = ZERO_START_NULL_END.getFile(storage); final byte[] content = new byte[1024]; Arrays.fill(content, (byte) 1); @@ -351,37 +355,37 @@ public void testPreallocationAndAppend() throws Exception { long preallocated = 16 * 1024; try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(file, false, max.getSize(), 16 * 1024, ByteBuffer.allocateDirect(10 * 1024))) { - Assert.assertEquals(preallocated, file.length()); + Assertions.assertEquals(preallocated, file.length()); while (totalSize + entrySize < max.getSize()) { totalSize += entrySize; out.write(entry); if (totalSize > preallocated) { - Assert.assertEquals("totalSize==" + totalSize, - preallocated + 16 * 1024, file.length()); + Assertions.assertEquals(preallocated + 16 * 1024, file.length(), + "totalSize==" + totalSize); preallocated += 16 * 1024; } } } - Assert.assertEquals(totalSize, file.length()); + Assertions.assertEquals(totalSize, file.length()); } @Test public void testZeroSizeInProgressFile() throws Exception { final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File file = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File file = ZERO_START_NULL_END.getFile(storage); storage.close(); // create zero size in-progress file LOG.info("file: " + file); - Assert.assertTrue(file.createNewFile()); + Assertions.assertTrue(file.createNewFile()); final Path path = file.toPath(); - Assert.assertTrue(Files.exists(path)); - Assert.assertEquals(0, Files.size(path)); + Assertions.assertTrue(Files.exists(path)); + Assertions.assertEquals(0, Files.size(path)); // getLogSegmentPaths should remove it. final List logs = LogSegmentPath.getLogSegmentPaths(storage); - Assert.assertEquals(0, logs.size()); - Assert.assertFalse(Files.exists(path)); + Assertions.assertEquals(0, logs.size()); + Assertions.assertFalse(Files.exists(path)); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java index a020b43bd8..5600914070 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestRaftLogReadWrite.java @@ -29,10 +29,10 @@ import org.apache.ratis.thirdparty.com.google.protobuf.CodedOutputStream; import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.util.FileUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -44,6 +44,8 @@ import java.util.Collections; import java.util.List; +import static org.apache.ratis.server.raftlog.segmented.TestLogSegment.ZERO_START_NULL_END; + /** * Test basic functionality of LogReader, SegmentedRaftLogInputStream, and SegmentedRaftLogOutputStream. */ @@ -53,7 +55,7 @@ public class TestRaftLogReadWrite extends BaseTest { private long preallocatedSize; private int bufferSize; - @Before + @BeforeEach public void setup() { storageDir = getTestDir(); RaftProperties properties = new RaftProperties(); @@ -66,7 +68,7 @@ public void setup() { RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); } - @After + @AfterEach public void tearDown() throws Exception { if (storageDir != null) { FileUtils.deleteFully(storageDir.getParentFile()); @@ -105,7 +107,7 @@ private long writeMessages(LogEntryProto[] entries, SegmentedRaftLogOutputStream @Test public void testReadWriteLog() throws IOException { final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File openSegment = ZERO_START_NULL_END.getFile(storage); long size = SegmentedRaftLogFormat.getHeaderLength(); final LogEntryProto[] entries = new LogEntryProto[100]; @@ -116,16 +118,16 @@ public void testReadWriteLog() throws IOException { storage.close(); } - Assert.assertEquals(size, openSegment.length()); + Assertions.assertEquals(size, openSegment.length()); final LogEntryProto[] readEntries = readLog(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true); - Assert.assertArrayEquals(entries, readEntries); + Assertions.assertArrayEquals(entries, readEntries); } @Test public void testAppendLog() throws IOException { final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File openSegment = ZERO_START_NULL_END.getFile(storage); LogEntryProto[] entries = new LogEntryProto[200]; try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) { @@ -146,7 +148,7 @@ public void testAppendLog() throws IOException { } final LogEntryProto[] readEntries = readLog(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true); - Assert.assertArrayEquals(entries, readEntries); + Assertions.assertArrayEquals(entries, readEntries); storage.close(); } @@ -158,7 +160,7 @@ public void testAppendLog() throws IOException { @Test public void testReadWithPadding() throws IOException { final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File openSegment = ZERO_START_NULL_END.getFile(storage); long size = SegmentedRaftLogFormat.getHeaderLength(); LogEntryProto[] entries = new LogEntryProto[100]; @@ -168,16 +170,16 @@ public void testReadWithPadding() throws IOException { out.flush(); // make sure the file contains padding - Assert.assertEquals( + Assertions.assertEquals( RaftServerConfigKeys.Log.PREALLOCATED_SIZE_DEFAULT.getSize(), openSegment.length()); // check if the reader can correctly read the log file final LogEntryProto[] readEntries = readLog(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true); - Assert.assertArrayEquals(entries, readEntries); + Assertions.assertArrayEquals(entries, readEntries); out.close(); - Assert.assertEquals(size, openSegment.length()); + Assertions.assertEquals(size, openSegment.length()); } /** @@ -187,7 +189,7 @@ public void testReadWithPadding() throws IOException { @Test public void testReadWithCorruptPadding() throws IOException { final RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File openSegment = ZERO_START_NULL_END.getFile(storage); LogEntryProto[] entries = new LogEntryProto[10]; final SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, @@ -200,12 +202,12 @@ public void testReadWithCorruptPadding() throws IOException { out.flush(); // make sure the file contains padding - Assert.assertEquals(4 * 1024 * 1024, openSegment.length()); + Assertions.assertEquals(4 * 1024 * 1024, openSegment.length()); try (FileChannel fout = FileUtils.newFileChannel(openSegment, StandardOpenOption.WRITE)) { final byte[] array = {-1, 1}; final int written = fout.write(ByteBuffer.wrap(array), 16 * 1024 * 1024 - 10); - Assert.assertEquals(array.length, written); + Assertions.assertEquals(array.length, written); } List list = new ArrayList<>(); @@ -215,7 +217,7 @@ public void testReadWithCorruptPadding() throws IOException { while ((entry = in.nextEntry()) != null) { list.add(entry); } - Assert.fail("should fail since we corrupt the padding"); + Assertions.fail("should fail since we corrupt the padding"); } catch (IOException e) { boolean findVerifyTerminator = false; for (StackTraceElement s : e.getStackTrace()) { @@ -224,9 +226,9 @@ public void testReadWithCorruptPadding() throws IOException { break; } } - Assert.assertTrue(findVerifyTerminator); + Assertions.assertTrue(findVerifyTerminator); } - Assert.assertArrayEquals(entries, + Assertions.assertArrayEquals(entries, list.toArray(new LogEntryProto[list.size()])); } @@ -236,7 +238,7 @@ public void testReadWithCorruptPadding() throws IOException { @Test public void testReadWithEntryCorruption() throws IOException { RaftStorage storage = RaftStorageTestUtils.newRaftStorage(storageDir); - final File openSegment = LogSegmentStartEnd.valueOf(0).getFile(storage); + final File openSegment = ZERO_START_NULL_END.getFile(storage); try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(openSegment, false, segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) { for (int i = 0; i < 100; i++) { @@ -259,7 +261,7 @@ public void testReadWithEntryCorruption() throws IOException { try { readLog(openSegment, 0, RaftLog.INVALID_LOG_INDEX, true); - Assert.fail("The read of corrupted log file should fail"); + Assertions.fail("The read of corrupted log file should fail"); } catch (ChecksumException e) { LOG.info("Caught ChecksumException as expected", e); } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLog.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLog.java index 38fa45e6fa..9bdbded8ab 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLog.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLog.java @@ -17,8 +17,6 @@ */ package org.apache.ratis.server.raftlog.segmented; -import static org.junit.Assert.assertTrue; - import org.apache.ratis.BaseTest; import org.apache.ratis.RaftTestUtil.SimpleOperation; import org.apache.ratis.conf.RaftProperties; @@ -39,40 +37,55 @@ import org.apache.ratis.server.raftlog.RaftLog; import org.apache.ratis.server.storage.RaftStorage; import org.apache.ratis.server.storage.RaftStorageTestUtils; +import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.statemachine.impl.BaseStateMachine; +import org.apache.ratis.util.CodeInjectionForTesting; +import org.apache.ratis.util.DataBlockingQueue; import org.apache.ratis.util.LifeCycle; +import org.apache.ratis.util.ReferenceCountedObject; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.JavaUtils; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; import java.util.function.Supplier; - -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.event.Level; -@RunWith(Parameterized.class) +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; +import static org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogWorker.RUN_WORKER; +import static org.apache.ratis.server.storage.RaftStorageTestUtils.getLogUnsafe; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; + public class TestSegmentedRaftLog extends BaseTest { static { Slf4jUtils.setLogLevel(SegmentedRaftLogWorker.LOG, Level.INFO); @@ -80,27 +93,21 @@ public class TestSegmentedRaftLog extends BaseTest { Slf4jUtils.setLogLevel(SegmentedRaftLog.LOG, Level.INFO); } - private final Boolean smSyncFlush; - private final Boolean useAsyncFlush; - - public TestSegmentedRaftLog(Boolean raftLogAsync, Boolean smSync) { - this.useAsyncFlush = raftLogAsync; - this.smSyncFlush = smSync; - } - - @Parameterized.Parameters - public static Collection data() { - return Arrays.asList((new Boolean[][] {{Boolean.FALSE, Boolean.FALSE}, {Boolean.FALSE, Boolean.TRUE}, - {Boolean.TRUE, Boolean.FALSE}, {Boolean.TRUE, Boolean.TRUE}})); + public static Stream data() { + return Stream.of( + arguments(FALSE, FALSE), + arguments(FALSE, TRUE), + arguments(TRUE, FALSE), + arguments(TRUE, TRUE)); } public static long getOpenSegmentSize(RaftLog raftLog) { return ((SegmentedRaftLog)raftLog).getRaftLogCache().getOpenSegment().getTotalFileSize(); } - private static final RaftPeerId peerId = RaftPeerId.valueOf("s0"); - private static final RaftGroupId groupId = RaftGroupId.randomId(); - private static final RaftGroupMemberId memberId = RaftGroupMemberId.valueOf(peerId, groupId); + private static final RaftPeerId PEER_ID = RaftPeerId.valueOf("s0"); + private static final RaftGroupId GROUP_ID = RaftGroupId.randomId(); + private static final RaftGroupMemberId MEMBER_ID = RaftGroupMemberId.valueOf(PEER_ID, GROUP_ID); static class SegmentRange { final long start; @@ -137,7 +144,7 @@ SegmentedRaftLog newSegmentedRaftLog(LongSupplier getSnapshotIndexFromStateMachi static SegmentedRaftLog newSegmentedRaftLog(RaftStorage storage, RaftProperties properties) { return SegmentedRaftLog.newBuilder() - .setMemberId(memberId) + .setMemberId(MEMBER_ID) .setStorage(storage) .setProperties(properties) .build(); @@ -146,20 +153,18 @@ static SegmentedRaftLog newSegmentedRaftLog(RaftStorage storage, RaftProperties private SegmentedRaftLog newSegmentedRaftLogWithSnapshotIndex(RaftStorage storage, RaftProperties properties, LongSupplier getSnapshotIndexFromStateMachine) { return SegmentedRaftLog.newBuilder() - .setMemberId(memberId) + .setMemberId(MEMBER_ID) .setStorage(storage) .setSnapshotIndexSupplier(getSnapshotIndexFromStateMachine) .setProperties(properties) .build(); } - @Before + @BeforeEach public void setup() throws Exception { storageDir = getTestDir(); properties = new RaftProperties(); RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(storageDir)); - RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); - RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); storage = RaftStorageTestUtils.newRaftStorage(storageDir); this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize(); @@ -169,7 +174,7 @@ public void setup() throws Exception { RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt(); } - @After + @AfterEach public void tearDown() throws Exception { if (storageDir != null) { FileUtils.deleteFully(storageDir.getParentFile()); @@ -208,11 +213,14 @@ static List prepareRanges(int startTerm, int endTerm, int segmentS private LogEntryProto getLastEntry(SegmentedRaftLog raftLog) throws IOException { - return raftLog.get(raftLog.getLastEntryTermIndex().getIndex()); + return getLogUnsafe(raftLog, raftLog.getLastEntryTermIndex().getIndex()); } - @Test - public void testLoadLogSegments() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testLoadLogSegments(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); // first generate log files List ranges = prepareRanges(0, 5, 100, 0); LogEntryProto[] entries = prepareLog(ranges); @@ -223,23 +231,23 @@ public void testLoadLogSegments() throws Exception { // check if log entries are loaded correctly for (LogEntryProto e : entries) { LogEntryProto entry = raftLog.get(e.getIndex()); - Assert.assertEquals(e, entry); + Assertions.assertEquals(e, entry); } final LogEntryHeader[] termIndices = raftLog.getEntries(0, 500); LogEntryProto[] entriesFromLog = Arrays.stream(termIndices) .map(ti -> { try { - return raftLog.get(ti.getIndex()); + return getLogUnsafe(raftLog, ti.getIndex()); } catch (IOException e) { throw new RuntimeException(e); } }) .toArray(LogEntryProto[]::new); - Assert.assertArrayEquals(entries, entriesFromLog); - Assert.assertEquals(entries[entries.length - 1], getLastEntry(raftLog)); + Assertions.assertArrayEquals(entries, entriesFromLog); + Assertions.assertEquals(entries[entries.length - 1], getLastEntry(raftLog)); - final RatisMetricRegistry metricRegistryForLogWorker = RaftLogMetricsBase.createRegistry(memberId); + final RatisMetricRegistry metricRegistryForLogWorker = RaftLogMetricsBase.createRegistry(MEMBER_ID); final DefaultTimekeeperImpl load = (DefaultTimekeeperImpl) metricRegistryForLogWorker.timer("segmentLoadLatency"); assertTrue(load.getTimer().getMeanRate() > 0); @@ -266,7 +274,8 @@ static List prepareLogEntries(SegmentRange range, return eList; } - static LogEntryProto prepareLogEntry(long term, long index, Supplier stringSupplier, boolean hasStataMachineData) { + static LogEntryProto prepareLogEntry(long term, long index, Supplier stringSupplier, + boolean hasStataMachineData) { final SimpleOperation m = stringSupplier == null? new SimpleOperation("m" + index, hasStataMachineData): new SimpleOperation(stringSupplier.get(), hasStataMachineData); @@ -276,8 +285,11 @@ static LogEntryProto prepareLogEntry(long term, long index, Supplier str /** * Append entry one by one and check if log state is correct. */ - @Test - public void testAppendEntry() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testAppendEntry(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); List ranges = prepareRanges(0, 5, 200, 0); List entries = prepareLogEntries(ranges, null); @@ -331,8 +343,11 @@ public void testAppendEntry() throws Exception { } } - @Test - public void testAppendEntryAfterPurge() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testAppendEntryAfterPurge(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); List ranges = prepareRanges(0, 5, 200, 0); List entries = prepareLogEntries(ranges, null); @@ -362,8 +377,11 @@ public long getAsLong() { /** * Keep appending entries, make sure the rolling is correct. */ - @Test - public void testAppendAndRoll() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testAppendAndRoll(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); RaftServerConfigKeys.Log.setPreallocatedSize(properties, SizeInBytes.valueOf("16KB")); RaftServerConfigKeys.Log.setSegmentSizeMax(properties, SizeInBytes.valueOf("128KB")); @@ -382,12 +400,91 @@ public void testAppendAndRoll() throws Exception { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); // check if the raft log is correct checkEntries(raftLog, entries, 0, entries.size()); - Assert.assertEquals(9, raftLog.getRaftLogCache().getNumOfSegments()); + Assertions.assertEquals(9, raftLog.getRaftLogCache().getNumOfSegments()); } } - @Test - public void testTruncate() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testPurgeAfterAppendEntry(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); + RaftServerConfigKeys.Log.setPurgeGap(properties, 1); + RaftServerConfigKeys.Log.setForceSyncNum(properties, 128); + + int startTerm = 0; + int endTerm = 2; + int segmentSize = 10; + long endIndexOfClosedSegment = segmentSize * (endTerm - startTerm - 1); + long nextStartIndex = segmentSize * (endTerm - startTerm); + + // append entries and roll logSegment for later purge operation + List ranges0 = prepareRanges(startTerm, endTerm, segmentSize, 0); + List entries0 = prepareLogEntries(ranges0, null); + try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) { + raftLog.open(RaftLog.INVALID_LOG_INDEX, null); + entries0.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join); + } + + // test the pattern in the task queue of SegmentedRaftLogWorker: (WriteLog, ..., PurgeLog) + List ranges = prepareRanges(endTerm - 1, endTerm, 1, nextStartIndex); + List entries = prepareLogEntries(ranges, null); + + try (SegmentedRaftLog raftLog = newSegmentedRaftLog()) { + final CountDownLatch raftLogOpened = new CountDownLatch(1); + final CountDownLatch tasksAdded = new CountDownLatch(1); + + // inject test code to make the pattern (WriteLog, PurgeLog) + final ConcurrentLinkedQueue> appendFutures = new ConcurrentLinkedQueue<>(); + final AtomicReference> purgeFuture = new AtomicReference<>(); + final AtomicInteger tasksCount = new AtomicInteger(0); + CodeInjectionForTesting.put(RUN_WORKER, (localId, remoteId, args) -> { + // wait for raftLog to be opened + try { + if(!raftLogOpened.await(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit())) { + throw new TimeoutException(); + } + } catch (InterruptedException | TimeoutException e) { + LOG.error("an exception occurred", e); + throw new RuntimeException(e); + } + + // add WriteLog and PurgeLog tasks + entries.stream().map(raftLog::appendEntry).forEach(appendFutures::add); + purgeFuture.set(raftLog.purge(endIndexOfClosedSegment)); + + tasksCount.set(((DataBlockingQueue) args[0]).getNumElements()); + tasksAdded.countDown(); + return true; + }); + + // open raftLog + raftLog.open(RaftLog.INVALID_LOG_INDEX, null); + raftLogOpened.countDown(); + + // wait for all tasks to be added + if(!tasksAdded.await(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit())) { + throw new TimeoutException(); + } + Assertions.assertEquals(entries.size() + 1, tasksCount.get()); + + // check if the purge task is executed + final Long purged = purgeFuture.get().get(); + LOG.info("purgeIndex = {}, purged = {}", endIndexOfClosedSegment, purged); + Assertions.assertEquals(endIndexOfClosedSegment, raftLog.getRaftLogCache().getStartIndex()); + + // check if the appendEntry futures are done + JavaUtils.allOf(appendFutures).get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()); + } finally { + CodeInjectionForTesting.put(RUN_WORKER, (localId, remoteId, args) -> false); + } + } + + @ParameterizedTest + @MethodSource("data") + public void testTruncate(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); // prepare the log for truncation List ranges = prepareRanges(0, 5, 200, 0); List entries = prepareLogEntries(ranges, null); @@ -418,10 +515,10 @@ private void testTruncate(List entries, long fromIndex) raftLog.open(RaftLog.INVALID_LOG_INDEX, null); // check if the raft log is correct if (fromIndex > 0) { - Assert.assertEquals(entries.get((int) (fromIndex - 1)), + Assertions.assertEquals(entries.get((int) (fromIndex - 1)), getLastEntry(raftLog)); } else { - Assert.assertNull(raftLog.getLastEntryTermIndex()); + Assertions.assertNull(raftLog.getLastEntryTermIndex()); } checkEntries(raftLog, entries, 0, (int) fromIndex); } @@ -432,7 +529,7 @@ private void checkEntries(RaftLog raftLog, List expected, if (size > 0) { for (int i = offset; i < size + offset; i++) { LogEntryProto entry = raftLog.get(expected.get(i).getIndex()); - Assert.assertEquals(expected.get(i), entry); + Assertions.assertEquals(expected.get(i), entry); } final LogEntryHeader[] termIndices = raftLog.getEntries( expected.get(offset).getIndex(), @@ -440,7 +537,7 @@ private void checkEntries(RaftLog raftLog, List expected, LogEntryProto[] entriesFromLog = Arrays.stream(termIndices) .map(ti -> { try { - return raftLog.get(ti.getIndex()); + return getLogUnsafe(raftLog, ti.getIndex()); } catch (IOException e) { throw new RuntimeException(e); } @@ -448,7 +545,7 @@ private void checkEntries(RaftLog raftLog, List expected, .toArray(LogEntryProto[]::new); LogEntryProto[] expectedArray = expected.subList(offset, offset + size) .stream().toArray(LogEntryProto[]::new); - Assert.assertArrayEquals(expectedArray, entriesFromLog); + Assertions.assertArrayEquals(expectedArray, entriesFromLog); } } @@ -469,6 +566,7 @@ public void testPurgeOnOpenSegment() throws Exception { int segmentSize = 200; long beginIndexOfOpenSegment = segmentSize * (endTerm - startTerm - 1); long expectedIndex = segmentSize * (endTerm - startTerm - 1); + long purgePreservation = 0L; purgeAndVerify(startTerm, endTerm, segmentSize, 1, beginIndexOfOpenSegment, expectedIndex); } @@ -489,7 +587,7 @@ public void testPurgeLogMetric() throws Exception { int segmentSize = 200; long endIndexOfClosedSegment = segmentSize * (endTerm - startTerm - 1) - 1; long expectedIndex = segmentSize * (endTerm - startTerm - 1); - final RatisMetricRegistry metricRegistryForLogWorker = RaftLogMetricsBase.createRegistry(memberId); + final RatisMetricRegistry metricRegistryForLogWorker = RaftLogMetricsBase.createRegistry(MEMBER_ID); purgeAndVerify(startTerm, endTerm, segmentSize, 1, endIndexOfClosedSegment, expectedIndex); final DefaultTimekeeperImpl purge = (DefaultTimekeeperImpl) metricRegistryForLogWorker.timer("purgeLog"); assertTrue(purge.getTimer().getCount() > 0); @@ -505,33 +603,59 @@ public void testPurgeOnClosedSegmentsWithPurgeGap() throws Exception { purgeAndVerify(startTerm, endTerm, segmentSize, 1000, endIndexOfClosedSegment, expectedIndex); } - private void purgeAndVerify(int startTerm, int endTerm, int segmentSize, int purgeGap, long purgeIndex, long expectedIndex) throws Exception { - List ranges = prepareRanges(startTerm, endTerm, segmentSize, 0); + @Test + public void testPurgeWithLargePurgePreservationAndSmallPurgeGap() throws Exception { + int startTerm = 0; + int endTerm = 5; + int segmentSize = 200; + long endIndex = segmentSize * (endTerm - startTerm) - 1; + // start index is set so that the suggested index will not be negative, which will not trigger any purge + long startIndex = 200; + // purge preservation is larger than the total size of the log entries + // which causes suggested index to be lower than the start index + long purgePreservation = (segmentSize * (endTerm - startTerm )) + 100; + // if the suggested index is lower than the start index due to the purge preservation, we should not purge anything + purgeAndVerify(startTerm, endTerm, segmentSize, 1, endIndex, startIndex, startIndex, purgePreservation); + } + + private void purgeAndVerify(int startTerm, int endTerm, int segmentSize, int purgeGap, long purgeIndex, + long expectedIndex) throws Exception { + purgeAndVerify(startTerm, endTerm, segmentSize, purgeGap, purgeIndex, expectedIndex, 0, 0); + } + + private void purgeAndVerify(int startTerm, int endTerm, int segmentSize, int purgeGap, long purgeIndex, + long expectedIndex, long startIndex, long purgePreservation) throws Exception { + List ranges = prepareRanges(startTerm, endTerm, segmentSize, startIndex); List entries = prepareLogEntries(ranges, null); final RaftProperties p = new RaftProperties(); RaftServerConfigKeys.Log.setPurgeGap(p, purgeGap); - try (SegmentedRaftLog raftLog = newSegmentedRaftLog(storage, p)) { - raftLog.open(RaftLog.INVALID_LOG_INDEX, null); + RaftServerConfigKeys.Log.setPurgePreservationLogNum(p, purgePreservation); + try (SegmentedRaftLog raftLog = newSegmentedRaftLogWithSnapshotIndex(storage, p, () -> startIndex - 1)) { + raftLog.open(startIndex - 1, null); entries.stream().map(raftLog::appendEntry).forEach(CompletableFuture::join); final CompletableFuture f = raftLog.purge(purgeIndex); final Long purged = f.get(); LOG.info("purgeIndex = {}, purged = {}", purgeIndex, purged); - Assert.assertEquals(expectedIndex, raftLog.getRaftLogCache().getStartIndex()); + Assertions.assertEquals(expectedIndex, raftLog.getRaftLogCache().getStartIndex()); } } /** * Test append with inconsistent entries */ - @Test - public void testAppendEntriesWithInconsistency() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testAppendEntriesWithInconsistency(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); // prepare the log for truncation List ranges = prepareRanges(0, 5, 200, 0); List entries = prepareLogEntries(ranges, null); final RetryCache retryCache = RetryCacheTestUtil.createRetryCache(); - try (SegmentedRaftLog raftLog = RetryCacheTestUtil.newSegmentedRaftLog(memberId, retryCache, storage, properties)) { + try (SegmentedRaftLog raftLog = + RetryCacheTestUtil.newSegmentedRaftLog(MEMBER_ID, retryCache, storage, properties)) { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); entries.forEach(entry -> RetryCacheTestUtil.createEntry(retryCache, entry)); // append entries to the raftlog @@ -546,45 +670,50 @@ public void testAppendEntriesWithInconsistency() throws Exception { List newEntries = prepareLogEntries( Arrays.asList(r1, r2, r3), null); - try (SegmentedRaftLog raftLog = RetryCacheTestUtil.newSegmentedRaftLog(memberId, retryCache, storage, properties)) { + try (SegmentedRaftLog raftLog = + RetryCacheTestUtil.newSegmentedRaftLog(MEMBER_ID, retryCache, storage, properties)) { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); LOG.info("newEntries[0] = {}", newEntries.get(0)); final int last = newEntries.size() - 1; LOG.info("newEntries[{}] = {}", last, newEntries.get(last)); - raftLog.append(newEntries).forEach(CompletableFuture::join); + raftLog.append(ReferenceCountedObject.wrap(newEntries)).forEach(CompletableFuture::join); checkFailedEntries(entries, 650, retryCache); checkEntries(raftLog, entries, 0, 650); checkEntries(raftLog, newEntries, 100, 100); - Assert.assertEquals(newEntries.get(newEntries.size() - 1), + Assertions.assertEquals(newEntries.get(newEntries.size() - 1), getLastEntry(raftLog)); - Assert.assertEquals(newEntries.get(newEntries.size() - 1).getIndex(), + Assertions.assertEquals(newEntries.get(newEntries.size() - 1).getIndex(), raftLog.getFlushIndex()); } // load the raftlog again and check - try (SegmentedRaftLog raftLog = RetryCacheTestUtil.newSegmentedRaftLog(memberId, retryCache, storage, properties)) { + try (SegmentedRaftLog raftLog = + RetryCacheTestUtil.newSegmentedRaftLog(MEMBER_ID, retryCache, storage, properties)) { raftLog.open(RaftLog.INVALID_LOG_INDEX, null); checkEntries(raftLog, entries, 0, 650); checkEntries(raftLog, newEntries, 100, 100); - Assert.assertEquals(newEntries.get(newEntries.size() - 1), + Assertions.assertEquals(newEntries.get(newEntries.size() - 1), getLastEntry(raftLog)); - Assert.assertEquals(newEntries.get(newEntries.size() - 1).getIndex(), + Assertions.assertEquals(newEntries.get(newEntries.size() - 1).getIndex(), raftLog.getFlushIndex()); SegmentedRaftLogCache cache = raftLog.getRaftLogCache(); - Assert.assertEquals(5, cache.getNumOfSegments()); + Assertions.assertEquals(5, cache.getNumOfSegments()); } } - @Test - public void testSegmentedRaftLogStateMachineData() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testSegmentedRaftLogStateMachineData(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); final SegmentRange range = new SegmentRange(0, 10, 1, true); final List entries = prepareLogEntries(range, null, true, new ArrayList<>()); final SimpleStateMachine4Testing sm = new SimpleStateMachine4Testing(); try (SegmentedRaftLog raftLog = SegmentedRaftLog.newBuilder() - .setMemberId(memberId) + .setMemberId(MEMBER_ID) .setStateMachine(sm) .setStorage(storage) .setProperties(properties) @@ -622,8 +751,11 @@ public void testSegmentedRaftLogStateMachineData() throws Exception { } } - @Test(expected = TimeoutIOException.class) - public void testServerShutdownOnTimeoutIOException() throws Throwable { + @ParameterizedTest + @MethodSource("data") + public void testServerShutdownOnTimeoutIOException(Boolean useAsyncFlush, Boolean smSyncFlush) throws Throwable { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); RaftServerConfigKeys.Log.StateMachineData.setSync(properties, true); final TimeDuration syncTimeout = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); RaftServerConfigKeys.Log.StateMachineData.setSyncTimeout(properties, syncTimeout); @@ -633,7 +765,7 @@ public void testServerShutdownOnTimeoutIOException() throws Throwable { final LogEntryProto entry = prepareLogEntry(0, 0, null, true); final StateMachine sm = new BaseStateMachine() { @Override - public CompletableFuture write(LogEntryProto entry) { + public CompletableFuture write(ReferenceCountedObject entry, TransactionContext context) { getLifeCycle().transition(LifeCycle.State.STARTING); getLifeCycle().transition(LifeCycle.State.RUNNING); @@ -645,16 +777,16 @@ public void notifyLogFailed(Throwable cause, LogEntryProto entry) { LOG.info("Test StateMachine: Ratis log failed notification received as expected.", cause); LOG.info("Test StateMachine: Transition to PAUSED state."); - Assert.assertNotNull(entry); + Assertions.assertNotNull(entry); getLifeCycle().transition(LifeCycle.State.PAUSING); getLifeCycle().transition(LifeCycle.State.PAUSED); } }; - Throwable ex = null; // TimeoutIOException + ExecutionException ex; try (SegmentedRaftLog raftLog = SegmentedRaftLog.newBuilder() - .setMemberId(memberId) + .setMemberId(MEMBER_ID) .setStateMachine(sm) .setStorage(storage) .setProperties(properties) @@ -663,15 +795,10 @@ public void notifyLogFailed(Throwable cause, LogEntryProto entry) { // SegmentedRaftLogWorker should catch TimeoutIOException CompletableFuture f = raftLog.appendEntry(entry); // Wait for async writeStateMachineData to finish - try { - f.get(); - } catch (ExecutionException e) { - ex = e.getCause(); - } + ex = Assertions.assertThrows(ExecutionException.class, f::get); } - Assert.assertNotNull(ex); - Assert.assertSame(LifeCycle.State.PAUSED, sm.getLifeCycleState()); - throw ex; + Assertions.assertSame(LifeCycle.State.PAUSED, sm.getLifeCycleState()); + Assertions.assertInstanceOf(TimeoutIOException.class, ex.getCause()); } static Thread startAppendEntryThread(RaftLog raftLog, LogEntryProto entry) { @@ -688,18 +815,22 @@ static Thread startAppendEntryThread(RaftLog raftLog, LogEntryProto entry) { void assertIndices(RaftLog raftLog, long expectedFlushIndex, long expectedNextIndex) { LOG.info("assert expectedFlushIndex={}", expectedFlushIndex); - Assert.assertEquals(expectedFlushIndex, raftLog.getFlushIndex()); + Assertions.assertEquals(expectedFlushIndex, raftLog.getFlushIndex()); LOG.info("assert expectedNextIndex={}", expectedNextIndex); - Assert.assertEquals(expectedNextIndex, raftLog.getNextIndex()); + Assertions.assertEquals(expectedNextIndex, raftLog.getNextIndex()); } - void assertIndicesMultipleAttempts(RaftLog raftLog, long expectedFlushIndex, long expectedNextIndex) throws Exception { + void assertIndicesMultipleAttempts(RaftLog raftLog, long expectedFlushIndex, long expectedNextIndex) + throws Exception { JavaUtils.attempt(() -> assertIndices(raftLog, expectedFlushIndex, expectedNextIndex), 10, HUNDRED_MILLIS, "assertIndices", LOG); } - @Test - public void testAsyncFlushPerf1() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testAsyncFlushPerf1(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); List ranges = prepareRanges(0, 50, 20000, 0); List entries = prepareLogEntries(ranges, null); @@ -710,8 +841,9 @@ public void testAsyncFlushPerf1() throws Exception { long start = System.nanoTime(); for (int i = 0; i < entries.size(); i += 5) { // call append API - futures.add(raftLog.append(Arrays.asList( - entries.get(i), entries.get(i + 1), entries.get(i + 2), entries.get(i + 3), entries.get(i + 4)))); + List entries1 = Arrays.asList( + entries.get(i), entries.get(i + 1), entries.get(i + 2), entries.get(i + 3), entries.get(i + 4)); + futures.add(raftLog.append(ReferenceCountedObject.wrap(entries1))); } for (List> futureList: futures) { futureList.forEach(CompletableFuture::join); @@ -721,8 +853,11 @@ public void testAsyncFlushPerf1() throws Exception { } } - @Test - public void testAsyncFlushPerf2() throws Exception { + @ParameterizedTest + @MethodSource("data") + public void testAsyncFlushPerf2(Boolean useAsyncFlush, Boolean smSyncFlush) throws Exception { + RaftServerConfigKeys.Log.setAsyncFlushEnabled(properties, useAsyncFlush); + RaftServerConfigKeys.Log.StateMachineData.setSync(properties, smSyncFlush); List ranges = prepareRanges(0, 50, 20000, 0); List entries = prepareLogEntries(ranges, null); diff --git a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLogCache.java b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLogCache.java index 5be3c36578..1f5783b209 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLogCache.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/raftlog/segmented/TestSegmentedRaftLogCache.java @@ -20,7 +20,6 @@ import static org.apache.ratis.server.metrics.SegmentedRaftLogMetrics.*; import static org.apache.ratis.server.raftlog.segmented.SegmentedRaftLogTestUtils.MAX_OP_SIZE; -import java.io.IOException; import java.util.Iterator; import java.util.stream.IntStream; @@ -37,26 +36,26 @@ import org.apache.ratis.server.raftlog.segmented.LogSegment.Op; import org.apache.ratis.proto.RaftProtos.LogEntryProto; import org.apache.ratis.util.ReferenceCountedObject; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestSegmentedRaftLogCache { - private static final RaftProperties prop = new RaftProperties(); + private static final RaftProperties PROP = new RaftProperties(); private SegmentedRaftLogCache cache; private SegmentedRaftLogMetrics raftLogMetrics; private RatisMetricRegistryImpl ratisMetricRegistry; - @Before + @BeforeEach public void setup() { raftLogMetrics = new SegmentedRaftLogMetrics(RaftServerTestUtil.TEST_MEMBER_ID); ratisMetricRegistry = (RatisMetricRegistryImpl) raftLogMetrics.getRegistry(); - cache = new SegmentedRaftLogCache(null, null, prop, raftLogMetrics); + cache = new SegmentedRaftLogCache(null, null, PROP, raftLogMetrics); } - @After + @AfterEach public void clear() { raftLogMetrics.unregister(); } @@ -75,14 +74,14 @@ private LogSegment prepareLogSegment(long start, long end, boolean isOpen) { } private void checkCache(long start, long end, int segmentSize) { - Assert.assertEquals(start, cache.getStartIndex()); - Assert.assertEquals(end, cache.getEndIndex()); + Assertions.assertEquals(start, cache.getStartIndex()); + Assertions.assertEquals(end, cache.getEndIndex()); for (long index = start; index <= end; index++) { final LogSegment segment = cache.getSegment(index); final LogRecord record = segment.getLogRecord(index); - final LogEntryProto entry = segment.getEntryFromCache(record.getTermIndex()); - Assert.assertEquals(index, entry.getIndex()); + final ReferenceCountedObject entry = segment.getEntryFromCache(record.getTermIndex()); + Assertions.assertEquals(index, entry.get().getIndex()); } long[] offsets = new long[]{start, start + 1, start + (end - start) / 2, @@ -99,9 +98,9 @@ private void checkCache(long start, long end, int segmentSize) { private void checkCacheEntries(long offset, int size, long end) { final LogEntryHeader[] entries = cache.getTermIndices(offset, offset + size); long realEnd = offset + size > end + 1 ? end + 1 : offset + size; - Assert.assertEquals(realEnd - offset, entries.length); + Assertions.assertEquals(realEnd - offset, entries.length); for (long i = offset; i < realEnd; i++) { - Assert.assertEquals(i, entries[(int) (i - offset)].getIndex()); + Assertions.assertEquals(i, entries[(int) (i - offset)].getIndex()); } } @@ -114,7 +113,7 @@ public void testAddSegments() throws Exception { try { LogSegment s = prepareLogSegment(102, 103, true); cache.addSegment(s); - Assert.fail("should fail since there is gap between two segments"); + Assertions.fail("should fail since there is gap between two segments"); } catch (IllegalStateException ignored) { } @@ -125,7 +124,7 @@ public void testAddSegments() throws Exception { try { LogSegment s = prepareLogSegment(201, 202, true); cache.addSegment(s); - Assert.fail("should fail since there is still an open segment in cache"); + Assertions.fail("should fail since there is still an open segment in cache"); } catch (IllegalStateException ignored) { } @@ -135,17 +134,17 @@ public void testAddSegments() throws Exception { try { LogSegment s = prepareLogSegment(202, 203, true); cache.addSegment(s); - Assert.fail("should fail since there is gap between two segments"); + Assertions.fail("should fail since there is gap between two segments"); } catch (IllegalStateException ignored) { } LogSegment s3 = prepareLogSegment(201, 300, true); cache.addSegment(s3); - Assert.assertNotNull(cache.getOpenSegment()); + Assertions.assertNotNull(cache.getOpenSegment()); checkCache(1, 300, 100); cache.rollOpenSegment(true); - Assert.assertNotNull(cache.getOpenSegment()); + Assertions.assertNotNull(cache.getOpenSegment()); checkCache(1, 300, 100); } @@ -159,8 +158,8 @@ public void testAppendEntry() { LogEntryProto entry = LogProtoUtils.toLogEntryProto(m.getLogEntryContent(), 0, 0); cache.appendEntry(Op.WRITE_CACHE_WITHOUT_STATE_MACHINE_CACHE, ReferenceCountedObject.wrap(entry) ); - Assert.fail("the open segment is null"); - } catch (IllegalStateException ignored) { + Assertions.fail("the open segment is null"); + } catch (IllegalStateException | NullPointerException ignored) { } LogSegment openSegment = prepareLogSegment(100, 100, true); @@ -171,7 +170,7 @@ public void testAppendEntry() { ); } - Assert.assertNotNull(cache.getOpenSegment()); + Assertions.assertNotNull(cache.getOpenSegment()); checkCache(0, 199, 100); } @@ -188,7 +187,7 @@ public void testTruncate() throws Exception { cache.addSegment(s); long end = cache.getEndIndex(); - Assert.assertEquals(599, end); + Assertions.assertEquals(599, end); int numOfSegments = 6; // start truncation for (int i = 0; i < 10; i++) { // truncate 10 times @@ -200,46 +199,46 @@ public void testTruncate() throws Exception { // check TruncationSegments int currentNum= (int) (end / 100 + 1); if (currentNum < numOfSegments) { - Assert.assertEquals(1, ts.getToDelete().length); + Assertions.assertEquals(1, ts.getToDelete().length); numOfSegments = currentNum; } else { - Assert.assertEquals(0, ts.getToDelete().length); + Assertions.assertEquals(0, ts.getToDelete().length); } } // 230 entries remaining. truncate at the segment boundary TruncationSegments ts = cache.truncate(200); checkCache(0, 199, 100); - Assert.assertEquals(1, ts.getToDelete().length); - Assert.assertEquals(200, ts.getToDelete()[0].getStartIndex()); - Assert.assertEquals(229, ts.getToDelete()[0].getEndIndex()); - Assert.assertEquals(0, ts.getToDelete()[0].getTargetLength()); - Assert.assertFalse(ts.getToDelete()[0].isOpen()); - Assert.assertNull(ts.getToTruncate()); + Assertions.assertEquals(1, ts.getToDelete().length); + Assertions.assertEquals(200, ts.getToDelete()[0].getStartIndex()); + Assertions.assertEquals(229, ts.getToDelete()[0].getEndIndex()); + Assertions.assertEquals(0, ts.getToDelete()[0].getTargetLength()); + Assertions.assertFalse(ts.getToDelete()[0].isOpen()); + Assertions.assertNull(ts.getToTruncate()); // add another open segment and truncate it as a whole LogSegment newOpen = prepareLogSegment(200, 249, true); cache.addSegment(newOpen); ts = cache.truncate(200); checkCache(0, 199, 100); - Assert.assertEquals(1, ts.getToDelete().length); - Assert.assertEquals(200, ts.getToDelete()[0].getStartIndex()); - Assert.assertEquals(249, ts.getToDelete()[0].getEndIndex()); - Assert.assertEquals(0, ts.getToDelete()[0].getTargetLength()); - Assert.assertTrue(ts.getToDelete()[0].isOpen()); - Assert.assertNull(ts.getToTruncate()); + Assertions.assertEquals(1, ts.getToDelete().length); + Assertions.assertEquals(200, ts.getToDelete()[0].getStartIndex()); + Assertions.assertEquals(249, ts.getToDelete()[0].getEndIndex()); + Assertions.assertEquals(0, ts.getToDelete()[0].getTargetLength()); + Assertions.assertTrue(ts.getToDelete()[0].isOpen()); + Assertions.assertNull(ts.getToTruncate()); // add another open segment and truncate part of it newOpen = prepareLogSegment(200, 249, true); cache.addSegment(newOpen); ts = cache.truncate(220); checkCache(0, 219, 100); - Assert.assertNull(cache.getOpenSegment()); - Assert.assertEquals(0, ts.getToDelete().length); - Assert.assertTrue(ts.getToTruncate().isOpen()); - Assert.assertEquals(219, ts.getToTruncate().getNewEndIndex()); - Assert.assertEquals(200, ts.getToTruncate().getStartIndex()); - Assert.assertEquals(249, ts.getToTruncate().getEndIndex()); + Assertions.assertNull(cache.getOpenSegment()); + Assertions.assertEquals(0, ts.getToDelete().length); + Assertions.assertTrue(ts.getToTruncate().isOpen()); + Assertions.assertEquals(219, ts.getToTruncate().getNewEndIndex()); + Assertions.assertEquals(200, ts.getToTruncate().getStartIndex()); + Assertions.assertEquals(249, ts.getToTruncate().getEndIndex()); } @Test @@ -255,9 +254,9 @@ public void testOpenSegmentPurge() { int purgeIndex = sIndex; // open segment should never be purged TruncationSegments ts = cache.purge(purgeIndex); - Assert.assertNull(ts.getToTruncate()); - Assert.assertEquals(end - start, ts.getToDelete().length); - Assert.assertEquals(sIndex, cache.getStartIndex()); + Assertions.assertNull(ts.getToTruncate()); + Assertions.assertEquals(end - start, ts.getToDelete().length); + Assertions.assertEquals(sIndex, cache.getStartIndex()); } @Test @@ -272,9 +271,9 @@ public void testCloseSegmentPurge() { // overlapped close segment will not purged. Passing in index - 1 since // we purge a closed segment when end index == passed in purge index. TruncationSegments ts = cache.purge(purgeIndex - 1); - Assert.assertNull(ts.getToTruncate()); - Assert.assertEquals(end - start - 1, ts.getToDelete().length); - Assert.assertEquals(1, cache.getNumOfSegments()); + Assertions.assertNull(ts.getToTruncate()); + Assertions.assertEquals(end - start - 1, ts.getToDelete().length); + Assertions.assertEquals(1, cache.getNumOfSegments()); } private void populatedSegment(int start, int end, int segmentSize, boolean isOpen) { @@ -286,20 +285,20 @@ private void populatedSegment(int start, int end, int segmentSize, boolean isOpe }); } - private void testIterator(long startIndex) throws IOException { + private void testIterator(long startIndex) { Iterator iterator = cache.iterator(startIndex); TermIndex prev = null; while (iterator.hasNext()) { TermIndex termIndex = iterator.next(); - Assert.assertEquals(cache.getLogRecord(termIndex.getIndex()).getTermIndex(), termIndex); + Assertions.assertEquals(cache.getTermIndex(termIndex.getIndex()), termIndex); if (prev != null) { - Assert.assertEquals(prev.getIndex() + 1, termIndex.getIndex()); + Assertions.assertEquals(prev.getIndex() + 1, termIndex.getIndex()); } prev = termIndex; } if (startIndex <= cache.getEndIndex()) { - Assert.assertNotNull(prev); - Assert.assertEquals(cache.getEndIndex(), prev.getIndex()); + Assertions.assertNotNull(prev); + Assertions.assertEquals(cache.getEndIndex(), prev.getIndex()); } } @@ -321,7 +320,7 @@ public void testIterator() throws Exception { testIterator(299); Iterator iterator = cache.iterator(300); - Assert.assertFalse(iterator.hasNext()); + Assertions.assertFalse(iterator.hasNext()); } @Test @@ -332,14 +331,14 @@ public void testCacheMetric() { Long closedSegmentsNum = (Long) ratisMetricRegistry.getGauges((s, metric) -> s.contains(RAFT_LOG_CACHE_CLOSED_SEGMENTS_NUM)).values().iterator().next().getValue(); - Assert.assertEquals(2L, closedSegmentsNum.longValue()); + Assertions.assertEquals(2L, closedSegmentsNum.longValue()); Long closedSegmentsSizeInBytes = (Long) ratisMetricRegistry.getGauges((s, metric) -> s.contains(RAFT_LOG_CACHE_CLOSED_SEGMENTS_SIZE_IN_BYTES)).values().iterator().next().getValue(); - Assert.assertEquals(closedSegmentsSizeInBytes.longValue(), cache.getClosedSegmentsSizeInBytes()); + Assertions.assertEquals(closedSegmentsSizeInBytes.longValue(), cache.getClosedSegmentsSizeInBytes()); Long openSegmentSizeInBytes = (Long) ratisMetricRegistry.getGauges((s, metric) -> s.contains(RAFT_LOG_CACHE_OPEN_SEGMENT_SIZE_IN_BYTES)).values().iterator().next().getValue(); - Assert.assertEquals(openSegmentSizeInBytes.longValue(), cache.getOpenSegmentSizeInBytes()); + Assertions.assertEquals(openSegmentSizeInBytes.longValue(), cache.getOpenSegmentSizeInBytes()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java index 6e7555703f..12cd771315 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestRaftStorage.java @@ -32,10 +32,10 @@ import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.Preconditions; import org.apache.ratis.util.SizeInBytes; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.Mockito; import java.io.File; @@ -65,12 +65,12 @@ static RaftStorageImpl newRaftStorage(File dir) throws IOException { private File storageDir; - @Before + @BeforeEach public void setup() { storageDir = getTestDir(); } - @After + @AfterEach public void tearDown() throws Exception { if (storageDir != null) { FileUtils.deleteFully(storageDir.getParentFile()); @@ -87,28 +87,29 @@ static RaftStorageImpl formatRaftStorage(File dir) throws IOException { return impl; } + @SuppressWarnings({"squid:S5783"}) // Suppress same exception warning @Test public void testNotExistent() throws IOException { FileUtils.deleteFully(storageDir); // we will format the empty directory final RaftStorageImpl storage = newRaftStorage(storageDir); - Assert.assertEquals(StorageState.NORMAL, storage.getState()); + Assertions.assertEquals(StorageState.NORMAL, storage.getState()); try { formatRaftStorage(storageDir).close(); - Assert.fail("the format should fail since the storage is still locked"); + Assertions.fail("the format should fail since the storage is still locked"); } catch (IOException e) { - Assert.assertTrue(e.getMessage().contains("directory is already locked")); + Assertions.assertTrue(e.getMessage().contains("directory is already locked")); } storage.close(); FileUtils.deleteFully(storageDir); - Assert.assertTrue(storageDir.createNewFile()); + Assertions.assertTrue(storageDir.createNewFile()); try (RaftStorage ignored = newRaftStorage(storageDir)) { - Assert.fail(); + Assertions.fail(); } catch (IOException e) { - Assert.assertTrue( + Assertions.assertTrue( e.getMessage().contains(StorageState.NON_EXISTENT.name())); } } @@ -121,42 +122,42 @@ public void testStorage() throws Exception { final RaftStorageDirectoryImpl sd = new RaftStorageDirectoryImpl(storageDir, SizeInBytes.ZERO); try { StorageState state = sd.analyzeStorage(true); - Assert.assertEquals(StorageState.NOT_FORMATTED, state); - Assert.assertTrue(sd.isCurrentEmpty()); + Assertions.assertEquals(StorageState.NOT_FORMATTED, state); + Assertions.assertTrue(sd.isCurrentEmpty()); } finally { sd.unlock(); } RaftStorageImpl storage = newRaftStorage(storageDir); - Assert.assertEquals(StorageState.NORMAL, storage.getState()); + Assertions.assertEquals(StorageState.NORMAL, storage.getState()); storage.close(); - Assert.assertEquals(StorageState.NORMAL, sd.analyzeStorage(false)); + Assertions.assertEquals(StorageState.NORMAL, sd.analyzeStorage(false)); assertMetadataFile(sd.getMetaFile()); // test format storage = formatRaftStorage(storageDir); - Assert.assertEquals(StorageState.NORMAL, storage.getState()); + Assertions.assertEquals(StorageState.NORMAL, storage.getState()); final RaftStorageMetadataFile metaFile = new RaftStorageMetadataFileImpl(sd.getMetaFile()); - Assert.assertEquals(RaftStorageMetadata.getDefault(), metaFile.getMetadata()); + Assertions.assertEquals(RaftStorageMetadata.getDefault(), metaFile.getMetadata()); storage.close(); } static void assertMetadataFile(File m) throws Exception { - Assert.assertTrue(m.exists()); + Assertions.assertTrue(m.exists()); final RaftStorageMetadataFile metaFile = new RaftStorageMetadataFileImpl(m); - Assert.assertEquals(RaftStorageMetadata.getDefault(), metaFile.getMetadata()); + Assertions.assertEquals(RaftStorageMetadata.getDefault(), metaFile.getMetadata()); final RaftPeerId peer1 = RaftPeerId.valueOf("peer1"); final RaftStorageMetadata metadata = RaftStorageMetadata.valueOf(123, peer1); metaFile.persist(metadata); - Assert.assertEquals(metadata.getTerm(), 123); - Assert.assertEquals(metadata.getVotedFor(), peer1); - Assert.assertEquals(metadata, metaFile.getMetadata()); + Assertions.assertEquals(metadata.getTerm(), 123); + Assertions.assertEquals(metadata.getVotedFor(), peer1); + Assertions.assertEquals(metadata, metaFile.getMetadata()); final RaftStorageMetadataFile metaFile2 = new RaftStorageMetadataFileImpl(m); - Assert.assertNull(((AtomicReference) RaftTestUtil.getDeclaredField(metaFile2, "metadata")).get()); - Assert.assertEquals(metadata, metaFile2.getMetadata()); + Assertions.assertNull(((AtomicReference) RaftTestUtil.getDeclaredField(metaFile2, "metadata")).get()); + Assertions.assertEquals(metadata, metaFile2.getMetadata()); } @Test @@ -172,27 +173,27 @@ public void testMetaFile() throws Exception { @Test public void testCleanMetaTmpFile() throws Exception { RaftStorageImpl storage = newRaftStorage(storageDir); - Assert.assertEquals(StorageState.NORMAL, storage.getState()); + Assertions.assertEquals(StorageState.NORMAL, storage.getState()); storage.close(); final RaftStorageDirectoryImpl sd = new RaftStorageDirectoryImpl(storageDir, SizeInBytes.ZERO); File metaFile = sd.getMetaFile(); FileUtils.move(metaFile, sd.getMetaTmpFile()); - Assert.assertEquals(StorageState.NOT_FORMATTED, sd.analyzeStorage(false)); + Assertions.assertEquals(StorageState.NOT_FORMATTED, sd.analyzeStorage(false)); // RaftStorage initialization should succeed as the raft-meta.tmp is // always cleaned. newRaftStorage(storageDir).close(); - Assert.assertTrue(sd.getMetaFile().exists()); - Assert.assertTrue(sd.getMetaTmpFile().createNewFile()); - Assert.assertTrue(sd.getMetaTmpFile().exists()); + Assertions.assertTrue(sd.getMetaFile().exists()); + Assertions.assertTrue(sd.getMetaTmpFile().createNewFile()); + Assertions.assertTrue(sd.getMetaTmpFile().exists()); try { storage = newRaftStorage(storageDir); - Assert.assertEquals(StorageState.NORMAL, storage.getState()); - Assert.assertFalse(sd.getMetaTmpFile().exists()); - Assert.assertTrue(sd.getMetaFile().exists()); + Assertions.assertEquals(StorageState.NORMAL, storage.getState()); + Assertions.assertFalse(sd.getMetaTmpFile().exists()); + Assertions.assertTrue(sd.getMetaFile().exists()); } finally { storage.close(); } @@ -207,14 +208,14 @@ public void testSnapshotFileName() { final File file = new File(storageDir, name); final TermIndex ti = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(file); System.out.println("file = " + file); - Assert.assertEquals(term, ti.getTerm()); - Assert.assertEquals(index, ti.getIndex()); + Assertions.assertEquals(term, ti.getTerm()); + Assertions.assertEquals(index, ti.getIndex()); System.out.println("ti = " + ti); final File foo = new File(storageDir, "foo"); try { SimpleStateMachineStorage.getTermIndexFromSnapshotFile(foo); - Assert.fail(); + Assertions.fail(); } catch(IllegalArgumentException iae) { System.out.println("Good " + iae); } @@ -244,7 +245,7 @@ public int getNumSnapshotsRetained() { final long index = ThreadLocalRandom.current().nextLong(100, 1000L); if (termIndexSet.add(TermIndex.valueOf(term, index))) { File file = simpleStateMachineStorage.getSnapshotFile(term, index); - Assert.assertTrue(file.createNewFile()); + Assertions.assertTrue(file.createNewFile()); } } // create MD5 files that will not be deleted in older version @@ -254,7 +255,7 @@ public int getNumSnapshotsRetained() { if (termIndexSet.add(TermIndex.valueOf(term, index))) { File file = simpleStateMachineStorage.getSnapshotFile(term, index); File snapshotFile = new File(file.getParent(), file.getName() + MD5_SUFFIX); - Assert.assertTrue(snapshotFile.createNewFile()); + Assertions.assertTrue(snapshotFile.createNewFile()); } } @@ -274,7 +275,7 @@ public int getNumSnapshotsRetained() { System.out.println(file.getName()); Matcher matcher = SNAPSHOT_REGEX.matcher(file.getName()); if (matcher.matches()) { - Assert.assertTrue(remainingIndices.contains(Long.parseLong(matcher.group(2)))); + Assertions.assertTrue(remainingIndices.contains(Long.parseLong(matcher.group(2)))); } } @@ -288,7 +289,7 @@ public int getNumSnapshotsRetained() { final long term = ThreadLocalRandom.current().nextLong(1, 10L); final long index = ThreadLocalRandom.current().nextLong(1000L); File file = simpleStateMachineStorage.getSnapshotFile(term, index); - Assert.assertTrue(file.createNewFile()); + Assertions.assertTrue(file.createNewFile()); } simpleStateMachineStorage.cleanupOldSnapshots(new SnapshotRetentionPolicy() { }); @@ -297,8 +298,8 @@ public int getNumSnapshotsRetained() { private static File[] assertFileCount(File dir, int expected) { File[] files = dir.listFiles(); - Assert.assertNotNull(files); - Assert.assertEquals(Arrays.toString(files), expected, files.length); + Assertions.assertNotNull(files); + Assertions.assertEquals(expected, files.length, Arrays.toString(files)); return files; } @@ -309,6 +310,6 @@ public void testNotEnoughSpace() throws IOException { final RaftStorageDirectoryImpl sd = new RaftStorageDirectoryImpl(mockStorageDir, SizeInBytes.valueOf("100M")); StorageState state = sd.analyzeStorage(false); - Assert.assertEquals(StorageState.NO_SPACE, state); + Assertions.assertEquals(StorageState.NO_SPACE, state); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestStorageImplUtils.java b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestStorageImplUtils.java index e8661e2689..905fc0345d 100644 --- a/ratis-test/src/test/java/org/apache/ratis/server/storage/TestStorageImplUtils.java +++ b/ratis-test/src/test/java/org/apache/ratis/server/storage/TestStorageImplUtils.java @@ -22,9 +22,9 @@ import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.JavaUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -43,7 +43,7 @@ */ public class TestStorageImplUtils extends BaseTest { - private static final Supplier rootTestDir = JavaUtils.memoize( + private static final Supplier ROOT_TEST_DIR = JavaUtils.memoize( () -> new File(BaseTest.getRootTestDir(), JavaUtils.getClassSimpleName(TestStorageImplUtils.class) + Integer.toHexString(ThreadLocalRandom.current().nextInt()))); @@ -55,9 +55,9 @@ static File chooseNewStorageDir(List volumes, String sub) throws IOExcepti return new File(vol, sub); } - @AfterClass + @AfterAll public static void tearDown() throws IOException { - FileUtils.deleteFully(rootTestDir.get()); + FileUtils.deleteFully(ROOT_TEST_DIR.get()); } /** @@ -67,7 +67,7 @@ public static void tearDown() throws IOException { */ @Test public void testChooseStorageDirWithOneVolume() throws IOException { - File testDir = new File(rootTestDir.get(), UUID.randomUUID().toString()); + File testDir = new File(ROOT_TEST_DIR.get(), UUID.randomUUID().toString()); List directories = Collections.singletonList(testDir); String subDirOne = UUID.randomUUID().toString(); String subDirTwo = UUID.randomUUID().toString(); @@ -75,9 +75,9 @@ public void testChooseStorageDirWithOneVolume() throws IOException { final File storageDirTwo = chooseNewStorageDir(directories, subDirTwo); File expectedOne = new File(testDir, subDirOne); File expectedTwo = new File(testDir, subDirTwo); - Assert.assertEquals(expectedOne.getCanonicalPath(), + Assertions.assertEquals(expectedOne.getCanonicalPath(), storageDirOne.getCanonicalPath()); - Assert.assertEquals(expectedTwo.getCanonicalPath(), + Assertions.assertEquals(expectedTwo.getCanonicalPath(), storageDirTwo.getCanonicalPath()); } @@ -88,7 +88,7 @@ public void testChooseStorageDirWithOneVolume() throws IOException { */ @Test public void testChooseStorageDirWithMultipleVolumes() throws IOException { - File testDir = new File(rootTestDir.get(), UUID.randomUUID().toString()); + File testDir = new File(ROOT_TEST_DIR.get(), UUID.randomUUID().toString()); List directories = new ArrayList<>(); IntStream.range(0, 10).mapToObj((i) -> new File(testDir, Integer.toString(i))).forEach((dir) -> { @@ -113,7 +113,7 @@ public void testChooseStorageDirWithMultipleVolumes() throws IOException { String subDir = UUID.randomUUID().toString(); final File storageDirectory = chooseNewStorageDir(directories, subDir); File expected = new File(directories.get(6), subDir); - Assert.assertEquals(expected.getCanonicalPath(), + Assertions.assertEquals(expected.getCanonicalPath(), storageDirectory.getCanonicalPath()); } @@ -124,10 +124,10 @@ public void testChooseStorageDirWithMultipleVolumes() throws IOException { public void testChooseStorageDirWithNoVolume() { try { chooseNewStorageDir(Collections.emptyList(), UUID.randomUUID().toString()); - Assert.fail(); + Assertions.fail(); } catch (IOException ex) { String expectedErrMsg = "No storage directory found."; - Assert.assertEquals(expectedErrMsg, ex.getMessage()); + Assertions.assertEquals(expectedErrMsg, ex.getMessage()); } } @@ -136,7 +136,7 @@ public void testChooseStorageDirWithNoVolume() { */ @Test public void testAutoFormatSingleDirectory() throws Exception { - final File testDir = new File(rootTestDir.get(), UUID.randomUUID().toString()); + final File testDir = new File(ROOT_TEST_DIR.get(), UUID.randomUUID().toString()); FileUtils.createDirectories(testDir); final RaftProperties properties = new RaftProperties(); @@ -144,7 +144,7 @@ public void testAutoFormatSingleDirectory() throws Exception { final RaftStorageImpl storage = StorageImplUtils.initRaftStorage( "group-1", RaftStorage.StartupOption.RECOVER, properties); - Assert.assertNotNull(storage); + Assertions.assertNotNull(storage); storage.close(); } @@ -153,7 +153,7 @@ public void testAutoFormatSingleDirectory() throws Exception { */ @Test public void testAutoFormatMultiDirectories() throws Exception { - final File testDir = new File(rootTestDir.get(), UUID.randomUUID().toString()); + final File testDir = new File(ROOT_TEST_DIR.get(), UUID.randomUUID().toString()); final List directories = new ArrayList<>(); IntStream.range(0, 3).mapToObj((i) -> new File(testDir, Integer.toString(i))).forEach((dir) -> { @@ -172,6 +172,6 @@ public void testAutoFormatMultiDirectories() throws Exception { () -> StorageImplUtils.initRaftStorage( "group-1", RaftStorage.StartupOption.RECOVER, properties), IOException.class); - Assert.assertTrue(ioe.getMessage().contains("Failed to RECOVER: Storage directory not found")); + Assertions.assertTrue(ioe.getMessage().contains("Failed to RECOVER: Storage directory not found")); } } \ No newline at end of file diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/ElectionCommandIntegrationTest.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/ElectionCommandIntegrationTest.java index e708153aa5..a61bb19f73 100644 --- a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/ElectionCommandIntegrationTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/ElectionCommandIntegrationTest.java @@ -30,8 +30,8 @@ import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.util.concurrent.TimeUnit; @@ -63,13 +63,13 @@ void runTestElectionTransferCommand(MiniRaftCluster cluster) throws Exception { RaftServer.Division newLeader = cluster.getFollowers().get(0); final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); int ret = shell.run("election", "transfer", "-peers", address, "-address", newLeader.getPeer().getAddress()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); JavaUtils.attempt(() -> { - Assert.assertEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertEquals(cluster.getLeader().getId(), newLeader.getId()); }, 10, TimeDuration.valueOf(1, TimeUnit.SECONDS), "testElectionTransferCommand", LOG); } @@ -85,28 +85,28 @@ void runTestElectionTransferCommandToHigherPriority(MiniRaftCluster cluster) thr RaftServer.Division newLeader = cluster.getFollowers().get(0); final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertTrue(cluster.getFollowers().contains(newLeader)); + Assertions.assertTrue(cluster.getFollowers().contains(newLeader)); // set current leader's priority to 2 int ret = shell.run("peer", "setPriority", "-peers", address, "-addressPriority", leader.getPeer().getAddress()+ "|" + 2); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); // transfer to new leader will set its priority to 2 (with timeout 1s) ret = shell.run("election", "transfer", "-peers", address, "-address", newLeader.getPeer().getAddress(), "-timeout", "1"); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); - JavaUtils.attempt(() -> Assert.assertEquals(cluster.getLeader().getId(), newLeader.getId()), + JavaUtils.attempt(() -> Assertions.assertEquals(cluster.getLeader().getId(), newLeader.getId()), 10, TimeDuration.valueOf(1, TimeUnit.SECONDS), "testElectionTransferLeaderCommand", LOG); // verify that priorities of new leader and old leader are both 2 ret = shell.run("group", "info", "-peers", address); - Assert.assertEquals(0 , ret); + Assertions.assertEquals(0 , ret); String expected = String.format("\"%s\"%n priority: %d", newLeader.getPeer().getAddress(), 2); String expected2 = String.format("\"%s\"%n priority: %d", leader.getPeer().getAddress(), 2); - Assert.assertTrue(out.toString().contains(expected)); - Assert.assertTrue(out.toString().contains(expected2)); + Assertions.assertTrue(out.toString().contains(expected)); + Assertions.assertTrue(out.toString().contains(expected2)); } @Test @@ -120,25 +120,25 @@ void runTestElectionPauseResumeCommand(MiniRaftCluster cluster) throws Exception RaftServer.Division newLeader = cluster.getFollowers().get(0); final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); int ret = shell.run("election", "pause", "-peers", sb.toString(), "-address", newLeader.getPeer().getAddress()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); ret = shell.run("peer", "setPriority", "-peers", sb.toString(), "-addressPriority", newLeader.getPeer().getAddress() + "|" + 2); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); JavaUtils.attempt(() -> { - Assert.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); }, 10, TimeDuration.valueOf(1, TimeUnit.SECONDS), "testElectionPauseResumeCommand", LOG); ret = shell.run("election", "resume", "-peers", sb.toString(), "-address", newLeader.getPeer().getAddress()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); JavaUtils.attempt(() -> { - Assert.assertEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertEquals(cluster.getLeader().getId(), newLeader.getId()); }, 10, TimeDuration.valueOf(1, TimeUnit.SECONDS), "testElectionPauseResumeCommand", LOG); } @@ -153,10 +153,10 @@ void runTestElectionStepDownCommand(MiniRaftCluster cluster) throws Exception { RaftServer.Division newLeader = cluster.getFollowers().get(0); final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); - Assert.assertEquals(2, cluster.getFollowers().size()); + Assertions.assertNotEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertEquals(2, cluster.getFollowers().size()); int ret = shell.run("election", "stepDown", "-peers", sb.toString()); - Assert.assertEquals(0, ret); - Assert.assertEquals(3, cluster.getFollowers().size()); + Assertions.assertEquals(0, ret); + Assertions.assertEquals(3, cluster.getFollowers().size()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/GroupCommandIntegrationTest.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/GroupCommandIntegrationTest.java index c745307d2d..c020075e73 100644 --- a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/GroupCommandIntegrationTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/GroupCommandIntegrationTest.java @@ -20,6 +20,7 @@ import org.apache.ratis.RaftTestUtil; import org.apache.ratis.client.RaftClient; import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.protocol.RaftClientReply; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; import org.apache.ratis.server.impl.MiniRaftCluster; @@ -29,8 +30,8 @@ import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.SizeInBytes; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; public abstract class GroupCommandIntegrationTest @@ -64,11 +65,11 @@ void runTestGroupListCommand(MiniRaftCluster cluster) throws Exception { RatisShell shell = new RatisShell(out.getPrintStream()); int ret = shell.run("group", "list", "-peers", address, "-peerId", leader.getPeer().getId().toString()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); String info = out.toString().trim(); String expected = String.format("The peerId %s (server %s) is in 1 groups, and the groupIds is: [%s]", leader.getId(), leader.getPeer().getAddress(), leader.getGroup().getGroupId()); - Assert.assertEquals(expected, info); + Assertions.assertEquals(expected, info); } @Test @@ -82,12 +83,60 @@ void runTestGroupInfoCommand(MiniRaftCluster cluster) throws Exception { final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); int ret = shell.run("group", "info", "-peers", address); - Assert.assertEquals(0 , ret); + Assertions.assertEquals(0 , ret); String result = out.toString().trim(); String hearder = String.format("group id: %s%sleader info: %s(%s)%s%s", cluster.getGroupId().getUuid(), NEW_LINE, leader.getId(), cluster.getLeader().getPeer().getAddress(), NEW_LINE, NEW_LINE); String info = result.substring(0, hearder.length()); - Assert.assertEquals(hearder, info); + Assertions.assertEquals(hearder, info); } + + @Test + public void testGroupInfoCommandIncludesCorrectLogInfo() throws Exception { + // set number of server to 1 so that we can make sure which server returns the LogInfoProto + // since information of applied index, snapshot index, and last entry index are not shared between servers + runWithNewCluster(1, this::runTestGroupInfoCommandWithLogInfoVerification); + } + + void runTestGroupInfoCommandWithLogInfoVerification(MiniRaftCluster cluster) throws Exception { + RaftServer.Division leader = RaftTestUtil.waitForLeader(cluster); + + try (final RaftClient client = cluster.createClient(leader.getId())) { + for (int i = 0; i < RaftServerConfigKeys.Snapshot.creationGap(getProperties()); i++) { + RaftClientReply + reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i)); + Assertions.assertTrue(reply.isSuccess()); + } + } + + leader.getStateMachine().takeSnapshot(); + + final String address = getClusterAddress(cluster); + final StringPrintStream out = new StringPrintStream(); + RatisShell shell = new RatisShell(out.getPrintStream()); + int ret = shell.run("group", "info", "-peers", address); + Assertions.assertEquals(0 , ret); + String result = out.toString().trim(); + String hearder = String.format("group id: %s%sleader info: %s(%s)%s%s", + cluster.getGroupId().getUuid(), NEW_LINE, leader.getId(), + cluster.getLeader().getPeer().getAddress(), NEW_LINE, NEW_LINE); + String info = result.substring(0, hearder.length()); + Assertions.assertEquals(hearder, info); + long currentTerm = leader.getInfo().getCurrentTerm(); + String logInfoProtoFormat = "%s {" + NEW_LINE + " term: " + currentTerm + NEW_LINE + " index: %s"; + Assertions.assertTrue(result.contains( + String.format(logInfoProtoFormat, "applied", + leader.getStateMachine().getLastAppliedTermIndex().getIndex()))); + Assertions.assertTrue(result.contains( + String.format(logInfoProtoFormat, "committed", + leader.getRaftLog().getLastCommittedIndex()))); + Assertions.assertTrue(result.contains( + String.format(logInfoProtoFormat, "lastSnapshot", + leader.getStateMachine().getLatestSnapshot().getIndex()))); + Assertions.assertTrue(result.contains( + String.format(logInfoProtoFormat, "lastEntry", + leader.getRaftLog().getLastCommittedIndex()))); + } + } diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/LocalCommandIntegrationTest.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/LocalCommandIntegrationTest.java new file mode 100644 index 0000000000..afc13837c5 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/LocalCommandIntegrationTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.shell.cli.sh; + +import org.apache.ratis.proto.RaftProtos.RaftPeerRole; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto; +import org.apache.ratis.proto.RaftProtos.RaftPeerProto; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class LocalCommandIntegrationTest { + + private static final String RAFT_META_CONF = "raft-meta.conf"; + private static final String NEW_RAFT_META_CONF = "new-raft-meta.conf"; + private static Pattern p = Pattern.compile("(?:\\w+\\|\\w+:\\d+,?)+"); + + + @Test + public void testDuplicatedPeerAddresses() throws Exception { + String[] duplicatedAddressesList = {"peer1_ID1|host1:9872,peer2_ID|host2:9872,peer1_ID2|host1:9872", + "host1:9872,host2:9872,host1:9872"}; + + testDuplicatedPeers(duplicatedAddressesList, "address", "host1:9872"); + } + + @Test + public void testDuplicatedPeerIds() throws Exception { + String[] duplicatedIdsList = {"peer1_ID1|host1:9872,peer2_ID|host2:9872,peer1_ID1|host3:9872"}; + + testDuplicatedPeers(duplicatedIdsList, "ID", "peer1_ID1"); + } + + private void testDuplicatedPeers(String[] peersList, String expectedErrorMessagePart, String expectedDuplicatedValue) + throws Exception { + for (String peersStr : peersList) { + StringPrintStream out = new StringPrintStream(); + RatisShell shell = new RatisShell(out.getPrintStream()); + int ret = shell.run("local", "raftMetaConf", "-peers", peersStr, "-path", "test"); + Assertions.assertEquals(-1, ret); + String message = out.toString().trim(); + Assertions.assertEquals( + String.format("Found duplicated %s: %s. Please make sure the %s of peer have no duplicated value.", + expectedErrorMessagePart, expectedDuplicatedValue, expectedErrorMessagePart), message); + } + } + + @Test + public void testRunMethod(@TempDir Path tempDir) throws Exception { + int index = 1; + generateRaftConf(tempDir.resolve(RAFT_META_CONF), index); + + String[] testPeersListArray = {"peer1_ID|localhost:9872,peer2_ID|host2:9872,peer3_ID|host3:9872", + "host1:9872,host2:9872,host3:9872"}; + + for (String peersListStr : testPeersListArray) { + generateRaftConf(tempDir.resolve(RAFT_META_CONF), index); + StringPrintStream out = new StringPrintStream(); + RatisShell shell = new RatisShell(out.getPrintStream()); + int ret = shell.run("local", "raftMetaConf", "-peers", peersListStr, "-path", tempDir.toString()); + Assertions.assertEquals(0, ret); + + // read & verify the contents of the new-raft-meta.conf file + long indexFromNewConf; + List peers; + try (InputStream in = Files.newInputStream(tempDir.resolve(NEW_RAFT_META_CONF))) { + LogEntryProto logEntry = LogEntryProto.newBuilder().mergeFrom(in).build(); + indexFromNewConf = logEntry.getIndex(); + peers = logEntry.getConfigurationEntry().getPeersList(); + } + + Assertions.assertEquals(index + 1, indexFromNewConf); + + String addressRegex = "^[a-zA-Z0-9.-]+:\\d+$"; + Pattern pattern = Pattern.compile(addressRegex); + peers.forEach(peerProto -> Assertions.assertTrue( + pattern.matcher(peerProto.getAddress()).matches())); + + String peersListStrFromNewMetaConf; + if (containsPeerId(peersListStr)) { + peersListStrFromNewMetaConf = peers.stream() + .map(peer -> peer.getId().toStringUtf8() + "|" + peer.getAddress()) + .collect(Collectors.joining(",")); + } else { + peersListStrFromNewMetaConf = peers.stream().map(RaftPeerProto::getAddress) + .collect(Collectors.joining(",")); + } + + Assertions.assertEquals(peersListStr, peersListStrFromNewMetaConf); + } + } + + + private void generateRaftConf(Path path, int index) throws IOException { + Map map = new HashMap<>(); + map.put("peer1_ID", "host1:9872"); + map.put("peer2_ID", "host2:9872"); + map.put("peer3_ID", "host3:9872"); + map.put("peer4_ID", "host4:9872"); + List raftPeerProtos = new ArrayList<>(); + for (Map.Entry en : map.entrySet()) { + raftPeerProtos.add(RaftPeerProto.newBuilder() + .setId(ByteString.copyFrom(en.getKey().getBytes(StandardCharsets.UTF_8))).setAddress(en.getValue()) + .setStartupRole(RaftPeerRole.FOLLOWER).build()); + } + + LogEntryProto generateLogEntryProto = LogEntryProto.newBuilder() + .setConfigurationEntry(RaftConfigurationProto.newBuilder().addAllPeers(raftPeerProtos).build()) + .setIndex(index).build(); + try (OutputStream out = Files.newOutputStream(path)) { + generateLogEntryProto.writeTo(out); + } + } + + private boolean containsPeerId(String str) { + return p.matcher(str).find(); + } + +} diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/PeerCommandIntegrationTest.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/PeerCommandIntegrationTest.java index 80400309ec..b3c35147a9 100644 --- a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/PeerCommandIntegrationTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/PeerCommandIntegrationTest.java @@ -33,8 +33,8 @@ import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.util.concurrent.TimeUnit; @@ -68,11 +68,11 @@ void runTestPeerRemoveCommand(MiniRaftCluster cluster) throws Exception { RaftPeer[] peers = new RaftPeer[]{cluster.getFollowers().get(1).getPeer(), leader.getPeer()}; final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertTrue(cluster.getFollowers().contains(toRemove)); + Assertions.assertTrue(cluster.getFollowers().contains(toRemove)); int ret = shell.run("peer", "remove", "-peers", address, "-peerId", toRemove.getPeer().getId().toString()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); RaftServerTestUtil.waitAndCheckNewConf(cluster, peers,1, null); } @@ -95,7 +95,7 @@ void runTestPeerAddCommand(MiniRaftCluster cluster) throws Exception { int ret = shell.run("peer", "add", "-peers", sb.toString(), "-address", newPeers[0].getAdminAddress(), "-peerId", newPeers[0].getId().toString()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); RaftServerTestUtil.waitAndCheckNewConf(cluster, ObjectArrays.concat(peers, newPeers[0]), 0, null); } @@ -112,12 +112,12 @@ void runTestPeerSetPriorityCommand(MiniRaftCluster cluster) throws Exception { RaftServer.Division newLeader = cluster.getFollowers().get(0); final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertTrue(cluster.getFollowers().contains(newLeader)); + Assertions.assertTrue(cluster.getFollowers().contains(newLeader)); int ret = shell.run("peer", "setPriority", "-peers", address, "-addressPriority", newLeader.getPeer().getAddress()+ "|" + 2); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); JavaUtils.attempt(() -> { - Assert.assertEquals(cluster.getLeader().getId(), newLeader.getId()); + Assertions.assertEquals(cluster.getLeader().getId(), newLeader.getId()); }, 10, TimeDuration.valueOf(1, TimeUnit.SECONDS), "testPeerSetPriorityCommand", LOG); } diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/SnapshotCommandIntegrationTest.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/SnapshotCommandIntegrationTest.java index 3c0ee9772c..f3f4e43077 100644 --- a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/SnapshotCommandIntegrationTest.java +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/SnapshotCommandIntegrationTest.java @@ -30,8 +30,8 @@ import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.util.Slf4jUtils; import org.apache.ratis.util.SizeInBytes; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.io.File; @@ -67,7 +67,7 @@ void runTestSnapshotCreateCommand(MiniRaftCluster cluster) throws Exception { try (final RaftClient client = cluster.createClient(leaderId)) { for (int i = 0; i < RaftServerConfigKeys.Snapshot.creationGap(getProperties()); i++) { RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } final String address = getClusterAddress(cluster); @@ -75,14 +75,14 @@ void runTestSnapshotCreateCommand(MiniRaftCluster cluster) throws Exception { RatisShell shell = new RatisShell(out.getPrintStream()); int ret = shell.run("snapshot", "create", "-peers", address, "-peerId", leader.getPeer().getId().toString()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); String[] str = out.toString().trim().split(" "); int snapshotIndex = Integer.parseInt(str[str.length-1]); LOG.info("snapshotIndex = {}", snapshotIndex); final File snapshotFile = SimpleStateMachine4Testing.get(leader) .getStateMachineStorage().getSnapshotFile(leader.getInfo().getCurrentTerm(), snapshotIndex); - Assert.assertTrue(snapshotFile.exists()); + Assertions.assertTrue(snapshotFile.exists()); } void runTestSnapshotCreateCommandOnSpecificServer(MiniRaftCluster cluster) throws Exception { @@ -91,16 +91,16 @@ void runTestSnapshotCreateCommandOnSpecificServer(MiniRaftCluster cluster) throw try (final RaftClient client = cluster.createClient(leaderId)) { for (int i = 0; i < RaftServerConfigKeys.Snapshot.creationGap(getProperties()); i++) { RaftClientReply reply = client.io().send(new RaftTestUtil.SimpleMessage("m" + i)); - Assert.assertTrue(reply.isSuccess()); + Assertions.assertTrue(reply.isSuccess()); } } final String address = getClusterAddress(cluster); final StringPrintStream out = new StringPrintStream(); RatisShell shell = new RatisShell(out.getPrintStream()); - Assert.assertEquals(2, cluster.getFollowers().size()); + Assertions.assertEquals(2, cluster.getFollowers().size()); int ret = shell.run("snapshot", "create", "-peers", address, "-peerId", cluster.getFollowers().get(0).getId().toString()); - Assert.assertEquals(0, ret); + Assertions.assertEquals(0, ret); String[] str = out.toString().trim().split(" "); int snapshotIndex = Integer.parseInt(str[str.length-1]); LOG.info("snapshotIndex = {}", snapshotIndex); @@ -108,7 +108,7 @@ void runTestSnapshotCreateCommandOnSpecificServer(MiniRaftCluster cluster) throw final File snapshotFile = SimpleStateMachine4Testing.get(cluster.getFollowers().get(0)) .getStateMachineStorage() .getSnapshotFile(cluster.getFollowers().get(0).getInfo().getCurrentTerm(), snapshotIndex); - Assert.assertTrue(snapshotFile.exists()); + Assertions.assertTrue(snapshotFile.exists()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestRatisShell.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestRatisShell.java index 6e22276475..21f4685121 100644 --- a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestRatisShell.java +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestRatisShell.java @@ -27,8 +27,8 @@ import org.apache.ratis.shell.cli.sh.command.PeerCommand; import org.apache.ratis.shell.cli.sh.command.SnapshotCommand; import org.apache.ratis.util.ReflectionUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.reflections.Reflections; import java.io.PrintStream; @@ -49,12 +49,12 @@ public class TestRatisShell extends BaseTest { static final Class[] ARG_CLASSES = new Class[] {Context.class}; static void assertCommand(String message, Command expected, Command computed) { - Assert.assertEquals(message, expected.getClass(), computed.getClass()); - Assert.assertEquals(message, expected.getCommandName(), computed.getCommandName()); + Assertions.assertEquals(expected.getClass(), computed.getClass(), message); + Assertions.assertEquals(expected.getCommandName(), computed.getCommandName(), message); } static void assertCommands(List expected, List computed) { - Assert.assertEquals(expected.size(), computed.size()); + Assertions.assertEquals(expected.size(), computed.size()); for(int i = 0; i < expected.size(); i++) { assertCommand("Command " + i, expected.get(i), computed.get(i)); } @@ -103,7 +103,7 @@ static void runTestFullCommandList(Function pare final List computed = new ArrayList<>(parent.getSubCommands().values()); Collections.sort(computed); - Assert.assertFalse(computed.isEmpty()); + Assertions.assertFalse(computed.isEmpty()); final Package pkg = computed.iterator().next().getClass().getPackage(); final List expected = new ArrayList<>(loadCommands(pkg)); diff --git a/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestSecureRatisShell.java b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestSecureRatisShell.java new file mode 100644 index 0000000000..785cd2e829 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/shell/cli/sh/TestSecureRatisShell.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.shell.cli.sh; + +import org.apache.ratis.BaseTest; +import org.apache.ratis.client.RaftClientConfigKeys; +import org.apache.ratis.conf.Parameters; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.grpc.GrpcConfigKeys; +import org.apache.ratis.grpc.GrpcTlsConfig; +import org.apache.ratis.grpc.MiniRaftClusterWithGrpc; +import org.apache.ratis.netty.NettyUtils; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.security.SecurityTestUtils; +import org.apache.ratis.util.Slf4jUtils; +import org.apache.ratis.util.TimeDuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.slf4j.event.Level; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.TrustManager; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.List; + +public class TestSecureRatisShell extends BaseTest { + { + Slf4jUtils.setLogLevel(NettyUtils.LOG, Level.DEBUG); + } + + private static final Parameters SERVER_PARAMETERS = new Parameters(); + private static final Parameters CLIENT_PARAMETERS = new Parameters(); + + static { + final TrustManager emptyTrustManager = SecurityTestUtils.emptyTrustManager(); + try { + final KeyManager serverKeyManager = SecurityTestUtils.getKeyManager(SecurityTestUtils::getServerKeyStore); + final GrpcTlsConfig serverConfig = new GrpcTlsConfig(serverKeyManager, emptyTrustManager, true); + GrpcConfigKeys.Server.setTlsConf(SERVER_PARAMETERS, serverConfig); + GrpcConfigKeys.Admin.setTlsConf(SERVER_PARAMETERS, serverConfig); + GrpcConfigKeys.Client.setTlsConf(SERVER_PARAMETERS, serverConfig); + } catch (Exception e) { + throw new IllegalStateException("Failed to init SERVER_PARAMETERS", e); + } + + try { + final KeyManager clientKeyManager = SecurityTestUtils.getKeyManager(SecurityTestUtils::getClientKeyStore); + final GrpcTlsConfig clientConfig = new GrpcTlsConfig(clientKeyManager, emptyTrustManager, true); + GrpcConfigKeys.Admin.setTlsConf(CLIENT_PARAMETERS, clientConfig); + GrpcConfigKeys.Client.setTlsConf(CLIENT_PARAMETERS, clientConfig); + } catch (Exception e) { + throw new IllegalStateException("Failed to init CLIENT_PARAMETERS", e); + } + } + + @Test + public void testRatisShell() throws Exception { + final String[] ids = {"s0"}; + final RaftProperties properties = new RaftProperties(); + RaftClientConfigKeys.Rpc.setRequestTimeout(properties, TimeDuration.ONE_MINUTE); + GrpcConfigKeys.TLS.setEnabled(properties, true); + GrpcConfigKeys.TLS.setMutualAuthnEnabled(properties, true); + + try(MiniRaftClusterWithGrpc cluster = new MiniRaftClusterWithGrpc(ids, properties, SERVER_PARAMETERS)) { + cluster.start(); + + runTestRatisShell(cluster, true); + runTestRatisShell(cluster, false); + } + } + + void runTestRatisShell(MiniRaftClusterWithGrpc cluster, boolean secure) throws Exception { + try(ByteArrayOutputStream out = new ByteArrayOutputStream(1 << 16); + RatisShell shell = newRatisShell(out, cluster.getProperties(), secure)) { + shell.run("group", "info", "-peers", toCliArg(cluster.getPeers())); + final String output = out.toString(); + LOG.info("output (secure? {}):\n{}", secure, output); + final String gid = cluster.getGroup().getGroupId().getUuid().toString(); + if (secure) { + Assertions.assertTrue(output.contains(gid), () -> gid + " not found for secure shell"); + } else { + Assertions.assertTrue(output.contains("Failed to get group ID"), "Unexpected output for unsecure shell"); + } + } + } + + static RatisShell newRatisShell(OutputStream out, RaftProperties properties, boolean secure) { + final PrintStream printStream = new PrintStream(out, true); + if (!secure) { + return new RatisShell(printStream); + } + return RatisShell.newBuilder() + .setPrintStream(printStream) + .setProperties(properties) + .setParameters(CLIENT_PARAMETERS) + .build(); + } + + static String toCliArg(List peers) { + final StringBuilder b = new StringBuilder(); + for(RaftPeer peer : peers) { + b.append(peer.getAdminAddress()).append(","); + } + return b.substring(0, b.length() - 1); + } +} diff --git a/ratis-test/src/test/java/org/apache/ratis/statemachine/TestStateMachine.java b/ratis-test/src/test/java/org/apache/ratis/statemachine/TestStateMachine.java index 8e37947d3b..0c49b31af2 100644 --- a/ratis-test/src/test/java/org/apache/ratis/statemachine/TestStateMachine.java +++ b/ratis-test/src/test/java/org/apache/ratis/statemachine/TestStateMachine.java @@ -35,7 +35,8 @@ import org.apache.ratis.server.simulation.MiniRaftClusterWithSimulatedRpc; import org.apache.ratis.statemachine.impl.SimpleStateMachine4Testing; import org.apache.ratis.util.Slf4jUtils; -import org.junit.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.slf4j.event.Level; import java.util.Collections; @@ -48,7 +49,10 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test StateMachine related functionality @@ -87,8 +91,8 @@ public TransactionContext startTransaction(RaftClientRequest request) { @Override public CompletableFuture applyTransaction(TransactionContext trx) { try { - assertNotNull(trx.getLogEntry()); - assertNotNull(trx.getStateMachineLogEntry()); + assertNotNull(trx.getLogEntryUnsafe()); + assertNotNull(trx.getLogEntryUnsafe().getStateMachineLogEntry()); Object context = trx.getStateMachineContext(); if (isLeader.get()) { assertNotNull(trx.getClientRequest()); @@ -162,9 +166,9 @@ static void runTestTransactionContextIsPassedBack(MiniRaftCluster cluster) throw final SMTransactionContext sm = SMTransactionContext.get(raftServer); final List ll = new ArrayList<>(sm.applied); Collections.sort(ll); - assertEquals(ll.toString(), ll.size(), numTrx); + assertEquals(ll.size(), numTrx, ll.toString()); for (int i=0; i < numTrx; i++) { - assertEquals(ll.toString(), Long.valueOf(i+1), ll.get(i)); + assertEquals(Long.valueOf(i+1), ll.get(i), ll.toString()); } } @@ -193,7 +197,7 @@ public void testStateMachineRegistry() throws Throwable { final RaftServer server = cluster.getServer(id); for(Map.Entry e: registry.entrySet()) { - Assert.assertSame(e.getValue(), server.getDivision(e.getKey()).getStateMachine()); + Assertions.assertSame(e.getValue(), server.getDivision(e.getKey()).getStateMachine()); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestDataBlockingQueue.java b/ratis-test/src/test/java/org/apache/ratis/util/TestDataBlockingQueue.java index df0dd2f8ed..3ad40d4ca7 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestDataBlockingQueue.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestDataBlockingQueue.java @@ -17,8 +17,9 @@ */ package org.apache.ratis.util; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,34 +37,38 @@ public class TestDataBlockingQueue { final TimeDuration slow = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); final TimeDuration fast = TimeDuration.valueOf(10, TimeUnit.MILLISECONDS); - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testElementLimit() { TestDataQueue.runTestElementLimit(q); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testByteLimit() { TestDataQueue.runTestByteLimit(q); } - @Test(timeout = 10_000) + @Test + @Timeout(value = 10) public void testSlowOfferFastPoll() throws Exception { runTestBlockingCalls(slow, fast, q); } - @Test(timeout = 10_000) + @Test + @Timeout(value = 10) public void testFastOfferSlowPoll() throws Exception { runTestBlockingCalls(fast, slow, q); } static void assertOfferPull(long offering, long polled, long elementLimit) { - Assert.assertTrue(offering >= polled); - Assert.assertTrue(offering - polled <= elementLimit + 1); + Assertions.assertTrue(offering >= polled); + Assertions.assertTrue(offering - polled <= elementLimit + 1); } static void runTestBlockingCalls(TimeDuration offerSleepTime, TimeDuration pollSleepTime, DataBlockingQueue q) throws Exception { - Assert.assertTrue(q.isEmpty()); + Assertions.assertTrue(q.isEmpty()); ExitUtils.disableSystemExit(); final int elementLimit = q.getElementLimit(); final TimeDuration timeout = CollectionUtils.min(offerSleepTime, pollSleepTime); @@ -74,11 +79,11 @@ static void runTestBlockingCalls(TimeDuration offerSleepTime, TimeDuration pollS final Thread pollThread = new Thread(() -> { try { - for(; polledValue.get() < endValue;) { + while (polledValue.get() < endValue) { pollSleepTime.sleep(); final Long polled = q.poll(timeout); if (polled != null) { - Assert.assertEquals(polledValue.incrementAndGet(), polled.intValue()); + Assertions.assertEquals(polledValue.incrementAndGet(), polled.intValue()); LOG.info("polled {}", polled); } assertOfferPull(offeringValue.get(), polledValue.get(), elementLimit); @@ -109,10 +114,10 @@ static void runTestBlockingCalls(TimeDuration offerSleepTime, TimeDuration pollS offerThread.join(); pollThread.join(); - Assert.assertEquals(endValue + 1, offeringValue.get()); - Assert.assertEquals(endValue, polledValue.get()); + Assertions.assertEquals(endValue + 1, offeringValue.get()); + Assertions.assertEquals(endValue, polledValue.get()); - Assert.assertTrue(q.isEmpty()); + Assertions.assertTrue(q.isEmpty()); ExitUtils.assertNotTerminated(); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestDataQueue.java b/ratis-test/src/test/java/org/apache/ratis/util/TestDataQueue.java index fe8a7ac246..42a55d90ff 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestDataQueue.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestDataQueue.java @@ -18,8 +18,9 @@ package org.apache.ratis.util; import org.apache.ratis.util.function.TriConsumer; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.ArrayList; import java.util.Collections; @@ -37,15 +38,16 @@ static TriConsumer getTimeoutHandler(bool } static void assertSizes(long expectedNumElements, long expectedNumBytes, DataQueue q) { - Assert.assertEquals(expectedNumElements, q.getNumElements()); - Assert.assertEquals(expectedNumBytes, q.getNumBytes()); + Assertions.assertEquals(expectedNumElements, q.getNumElements()); + Assertions.assertEquals(expectedNumBytes, q.getNumBytes()); } final SizeInBytes byteLimit = SizeInBytes.valueOf(100); final int elementLimit = 5; final DataQueue q = new DataQueue<>(null, byteLimit, elementLimit, Long::longValue); - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testElementLimit() { runTestElementLimit(q); } @@ -56,30 +58,31 @@ static void runTestElementLimit(DataQueue q) { final int elementLimit = q.getElementLimit(); long numBytes = 0; for (long i = 0; i < elementLimit; i++) { - Assert.assertEquals(i, q.getNumElements()); - Assert.assertEquals(numBytes, q.getNumBytes()); + Assertions.assertEquals(i, q.getNumElements()); + Assertions.assertEquals(numBytes, q.getNumBytes()); final boolean offered = q.offer(i); - Assert.assertTrue(offered); + Assertions.assertTrue(offered); numBytes += i; assertSizes(i+1, numBytes, q); } { final boolean offered = q.offer(0L); - Assert.assertFalse(offered); + Assertions.assertFalse(offered); assertSizes(elementLimit, numBytes, q); } { // poll all elements final List polled = q.pollList(100, (i, timeout) -> i, getTimeoutHandler(false)); - Assert.assertEquals(elementLimit, polled.size()); + Assertions.assertEquals(elementLimit, polled.size()); for (int i = 0; i < polled.size(); i++) { - Assert.assertEquals(i, polled.get(i).intValue()); + Assertions.assertEquals(i, polled.get(i).intValue()); } } assertSizes(0, 0, q); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testByteLimit() { runTestByteLimit(q); } @@ -90,53 +93,54 @@ static void runTestByteLimit(DataQueue q) { final long byteLimit = q.getByteLimit(); try { q.offer(byteLimit + 1); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException ignored) { } final long halfBytes = byteLimit / 2; { final boolean offered = q.offer(halfBytes); - Assert.assertTrue(offered); + Assertions.assertTrue(offered); assertSizes(1, halfBytes, q); } { final boolean offered = q.offer(halfBytes + 1); - Assert.assertFalse(offered); + Assertions.assertFalse(offered); assertSizes(1, halfBytes, q); } { final boolean offered = q.offer(halfBytes); - Assert.assertTrue(offered); + Assertions.assertTrue(offered); assertSizes(2, byteLimit, q); } { final boolean offered = q.offer(1L); - Assert.assertFalse(offered); + Assertions.assertFalse(offered); assertSizes(2, byteLimit, q); } { final boolean offered = q.offer(0L); - Assert.assertTrue(offered); + Assertions.assertTrue(offered); assertSizes(3, byteLimit, q); } { // poll all elements final List polled = q.pollList(100, (i, timeout) -> i, getTimeoutHandler(false)); - Assert.assertEquals(3, polled.size()); - Assert.assertEquals(halfBytes, polled.get(0).intValue()); - Assert.assertEquals(halfBytes, polled.get(1).intValue()); - Assert.assertEquals(0, polled.get(2).intValue()); + Assertions.assertEquals(3, polled.size()); + Assertions.assertEquals(halfBytes, polled.get(0).intValue()); + Assertions.assertEquals(halfBytes, polled.get(1).intValue()); + Assertions.assertEquals(0, polled.get(2).intValue()); } assertSizes(0, 0, q); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testIteratorAndRemove() { runTestIteratorAndRemove(q); } @@ -149,7 +153,7 @@ static void runTestIteratorAndRemove(DataQueue q) { long numBytes = 0; for(long i = 0; i < elementLimit; i++) { final boolean offered = q.offer(i); - Assert.assertTrue(offered); + Assertions.assertTrue(offered); numElements++; numBytes += i; assertSizes(numElements, numBytes, q); @@ -158,7 +162,7 @@ static void runTestIteratorAndRemove(DataQueue q) { { // test iterator() final Iterator i = q.iterator(); for (long expected = 0; expected < elementLimit; expected++) { - Assert.assertEquals(expected, i.next().longValue()); + Assertions.assertEquals(expected, i.next().longValue()); } } @@ -180,23 +184,24 @@ static void runTestIteratorAndRemove(DataQueue q) { assertSizes(0, 0, q); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testTimeout() { assertSizes(0, 0, q); long numBytes = 0; for (long i = 0; i < elementLimit; i++) { - Assert.assertEquals(i, q.getNumElements()); - Assert.assertEquals(numBytes, q.getNumBytes()); + Assertions.assertEquals(i, q.getNumElements()); + Assertions.assertEquals(numBytes, q.getNumBytes()); final boolean offered = q.offer(i); - Assert.assertTrue(offered); + Assertions.assertTrue(offered); numBytes += i; assertSizes(i+1, numBytes, q); } { // poll with zero time final List polled = q.pollList(0, (i, timeout) -> i, getTimeoutHandler(false)); - Assert.assertTrue(polled.isEmpty()); + Assertions.assertTrue(polled.isEmpty()); assertSizes(elementLimit, numBytes, q); } @@ -209,9 +214,9 @@ public void testTimeout() { } return i; }, getTimeoutHandler(true)); - Assert.assertEquals(halfElements, polled.size()); + Assertions.assertEquals(halfElements, polled.size()); for (int i = 0; i < polled.size(); i++) { - Assert.assertEquals(i, polled.get(i).intValue()); + Assertions.assertEquals(i, polled.get(i).intValue()); numBytes -= i; } assertSizes(elementLimit - halfElements, numBytes, q); @@ -219,9 +224,9 @@ public void testTimeout() { { // poll the remaining elements final List polled = q.pollList(100, (i, timeout) -> i, getTimeoutHandler(false)); - Assert.assertEquals(elementLimit - halfElements, polled.size()); + Assertions.assertEquals(elementLimit - halfElements, polled.size()); for (int i = 0; i < polled.size(); i++) { - Assert.assertEquals(halfElements + i, polled.get(i).intValue()); + Assertions.assertEquals(halfElements + i, polled.get(i).intValue()); } } assertSizes(0, 0, q); diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestExitUtils.java b/ratis-test/src/test/java/org/apache/ratis/util/TestExitUtils.java index 25b05fd0df..3c7cc025c9 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestExitUtils.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestExitUtils.java @@ -19,15 +19,17 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.util.ExitUtils.ExitException; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; public class TestExitUtils extends BaseTest { /** Test if {@link BaseTest} can handle uncaught exception. */ - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testUncaughtException() throws Exception { - Assert.assertFalse(ExitUtils.isTerminated()); - Assert.assertFalse(ExitUtils.clear()); + Assertions.assertFalse(ExitUtils.isTerminated()); + Assertions.assertFalse(ExitUtils.clear()); final Thread t = new Thread(null, () -> { throw new AssertionError("Testing"); @@ -35,25 +37,26 @@ public void testUncaughtException() throws Exception { t.start(); t.join(); - Assert.assertTrue(ExitUtils.isTerminated()); - Assert.assertTrue(ExitUtils.clear()); + Assertions.assertTrue(ExitUtils.isTerminated()); + Assertions.assertTrue(ExitUtils.clear()); } /** Test if {@link BaseTest} can handle ExitUtils.terminate(..). */ - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testExitStatus() { - Assert.assertFalse(ExitUtils.isTerminated()); - Assert.assertFalse(ExitUtils.clear()); + Assertions.assertFalse(ExitUtils.isTerminated()); + Assertions.assertFalse(ExitUtils.clear()); final int status = -1; try { ExitUtils.terminate(status, "testExitStatus", LOG); - Assert.fail(); + Assertions.fail(); } catch (ExitException e) { - Assert.assertEquals(status, e.getStatus()); + Assertions.assertEquals(status, e.getStatus()); } - Assert.assertTrue(ExitUtils.isTerminated()); - Assert.assertTrue(ExitUtils.clear()); + Assertions.assertTrue(ExitUtils.isTerminated()); + Assertions.assertTrue(ExitUtils.clear()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestLifeCycle.java b/ratis-test/src/test/java/org/apache/ratis/util/TestLifeCycle.java index 3faf2a4959..201b510571 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestLifeCycle.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestLifeCycle.java @@ -18,14 +18,19 @@ package org.apache.ratis.util; import org.apache.ratis.util.function.TriConsumer; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.Arrays; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; import static org.apache.ratis.util.LifeCycle.State.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.*; public class TestLifeCycle { /** @@ -33,7 +38,8 @@ public class TestLifeCycle { * {@link LifeCycle} uses predecessors to validate transitions * while this test uses successors. */ - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testIsValid() { final Map> successors = new EnumMap<>(LifeCycle.State.class); @@ -49,9 +55,8 @@ public void testIsValid() { final List states = Arrays.asList(LifeCycle.State.values()); states.forEach( from -> states.forEach( - to -> assertEquals(from + " -> " + to, - successors.get(from).contains(to), - isValid(from, to)))); + to -> assertEquals(successors.get(from).contains(to), + isValid(from, to), from + " -> " + to))); } @Test @@ -81,7 +86,8 @@ public void invalidTransitions() { testInvalidTransition((from, subject, to) -> subject.transitionAndGet(any -> to), true); } - private static void testInvalidTransition(TriConsumer op, boolean shouldThrow) { + private static void testInvalidTransition(TriConsumer op, + boolean shouldThrow) { LifeCycle subject = new LifeCycle("subject"); for (LifeCycle.State to : new LifeCycle.State[] { RUNNING, EXCEPTION, CLOSING }) { LifeCycle.State from = subject.getCurrentState(); @@ -90,7 +96,7 @@ private static void testInvalidTransition(TriConsumer n).limit(10)); @@ -41,13 +43,14 @@ public void testMinMax() { static void runTestMinMax(LongStream stream) { final List list = stream.collect(ArrayList::new, List::add, List::addAll); - final LongMinMax longMinMax = toLongStream(list).collect(LongMinMax::new, LongMinMax::accumulate, LongMinMax::combine); + final LongMinMax longMinMax = toLongStream(list) + .collect(LongMinMax::new, LongMinMax::accumulate, LongMinMax::combine); if (longMinMax.isInitialized()) { - Assert.assertEquals(toLongStream(list).min().getAsLong(), longMinMax.getMin()); - Assert.assertEquals(toLongStream(list).max().getAsLong(), longMinMax.getMax()); + Assertions.assertEquals(toLongStream(list).min().getAsLong(), longMinMax.getMin()); + Assertions.assertEquals(toLongStream(list).max().getAsLong(), longMinMax.getMax()); } else { - Assert.assertEquals(OptionalLong.empty(), toLongStream(list).min()); - Assert.assertEquals(OptionalLong.empty(), toLongStream(list).max()); + Assertions.assertEquals(OptionalLong.empty(), toLongStream(list).min()); + Assertions.assertEquals(OptionalLong.empty(), toLongStream(list).max()); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestPeerProxyMap.java b/ratis-test/src/test/java/org/apache/ratis/util/TestPeerProxyMap.java index 549fbc53f8..a2255041f8 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestPeerProxyMap.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestPeerProxyMap.java @@ -20,8 +20,9 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.io.Closeable; import java.io.IOException; @@ -51,7 +52,8 @@ public String toString() { } } - @Test(timeout = 10_000) + @Test + @Timeout(value = 10) public void testCloseDeadLock() throws Exception { final PeerProxyMap map = new PeerProxyMap<>("test", DummyProxy::new); final RaftPeerId id = RaftPeerId.valueOf("s0"); @@ -68,7 +70,7 @@ public void testCloseDeadLock() throws Exception { HUNDRED_MILLIS.sleep(); LOG.info("Try getProxy"); final DummyProxy newProxy = map.getProxy(id); - Assert.assertNotSame(proxy, newProxy); + Assertions.assertNotSame(proxy, newProxy); } catch (Exception e) { setFirstException(e); } @@ -115,7 +117,8 @@ public String toString() { } } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testStackTrace() { final RaftPeerId id = RaftPeerId.valueOf("s0"); final RaftPeer peer = RaftPeer.newBuilder().setId(id).build(); @@ -123,7 +126,7 @@ public void testStackTrace() { final ExceptionProxy ignored = map.computeIfAbsent(peer).get()) { } catch (IOException e) { assertThrowable("closeProxy", e, AnnotatedConnectException.class, LOG, ConnectException.class); - Assert.assertEquals(0, e.getStackTrace().length); + Assertions.assertEquals(0, e.getStackTrace().length); } } } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestPreconditions.java b/ratis-test/src/test/java/org/apache/ratis/util/TestPreconditions.java index 884c1e5d59..73cf4c1676 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestPreconditions.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestPreconditions.java @@ -18,7 +18,8 @@ package org.apache.ratis.util; import org.apache.ratis.BaseTest; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.Arrays; import java.util.Collections; @@ -28,7 +29,8 @@ import java.util.Set; public class TestPreconditions extends BaseTest { - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testAssertUnique() { final Set empty = Collections.emptySet(); Preconditions.assertUnique(empty); @@ -53,7 +55,8 @@ public void testAssertUnique() { Preconditions.assertUnique(three, Arrays.asList(4, 5, 6)); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testAssertNull() { final Map map = new HashMap<>(); final String key = "abc1234"; diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestRaftIdCache.java b/ratis-test/src/test/java/org/apache/ratis/util/TestRaftIdCache.java new file mode 100644 index 0000000000..16d5cd6526 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestRaftIdCache.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.util; + +import org.apache.ratis.BaseTest; +import org.apache.ratis.RaftTestUtil; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.TestRaftId; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** Testing {@link WeakValueCache}. */ +public class TestRaftIdCache extends BaseTest { + static WeakValueCache CACHE = TestRaftId.getClientIdCache(); + + static String dumpCache() { + final List values = CACHE.getValues(); + values.sort(Comparator.comparing(ClientId::getUuid)); + String header = CACHE + ": " + values.size(); + System.out.println(header); + System.out.println(" " + values); + return header; + } + + static void assertCache(IDs expectedIDs) { + final List computed = CACHE.getValues(); + computed.sort(Comparator.comparing(ClientId::getUuid)); + + final List expected = expectedIDs.getIds(); + expected.sort(Comparator.comparing(ClientId::getUuid)); + + assertEquals(expected, computed, TestRaftIdCache::dumpCache); + } + + void assertCacheSizeWithGC(IDs expectedIDs) throws Exception{ + JavaUtils.attempt(() -> { + RaftTestUtil.gc(); + assertCache(expectedIDs); + }, 5, HUNDRED_MILLIS, "assertCacheSizeWithGC", LOG); + } + + class IDs { + private final List ids = new LinkedList<>(); + + List getIds() { + return new ArrayList<>(ids); + } + + int size() { + return ids.size(); + } + + ClientId allocate() { + final ClientId id = ClientId.randomId(); + LOG.info("allocate {}", id); + ids.add(id); + return id; + } + + void release() { + final int r = ThreadLocalRandom.current().nextInt(size()); + final ClientId removed = ids.remove(r); + LOG.info("release {}", removed); + } + } + + @Test + public void testCaching() throws Exception { + final int n = 100; + final IDs ids = new IDs(); + assertEquals(0, ids.size()); + assertCache(ids); + + for(int i = 0; i < n; i++) { + final ClientId id = ids.allocate(); + assertSame(id, ClientId.valueOf(id.getUuid())); + assertCache(ids); + } + + for(int i = 0; i < n/2; i++) { + ids.release(); + if (ThreadLocalRandom.current().nextInt(10) == 0) { + assertCacheSizeWithGC(ids); + } + } + assertCacheSizeWithGC(ids); + + for(int i = 0; i < n/2; i++) { + final ClientId id = ids.allocate(); + assertSame(id, ClientId.valueOf(id.getUuid())); + assertCache(ids); + } + + + for(int i = 0; i < n; i++) { + ids.release(); + if (ThreadLocalRandom.current().nextInt(10) == 0) { + assertCacheSizeWithGC(ids); + } + } + assertCacheSizeWithGC(ids); + + assertEquals(0, ids.size()); + } +} diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestReferenceCountedObject.java b/ratis-test/src/test/java/org/apache/ratis/util/TestReferenceCountedObject.java index fe58b92da6..2436310141 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestReferenceCountedObject.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestReferenceCountedObject.java @@ -18,8 +18,9 @@ package org.apache.ratis.util; import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.concurrent.atomic.AtomicInteger; @@ -27,8 +28,8 @@ public class TestReferenceCountedObject { static void assertValues( AtomicInteger retained, int expectedRetained, AtomicInteger released, int expectedReleased) { - Assert.assertEquals("retained", expectedRetained, retained.get()); - Assert.assertEquals("released", expectedReleased, released.get()); + Assertions.assertEquals(expectedRetained, retained.get(), "retained"); + Assertions.assertEquals(expectedReleased, released.get(), "retained"); } static void assertRelease(ReferenceCountedObject ref, @@ -36,10 +37,11 @@ static void assertRelease(ReferenceCountedObject ref, AtomicInteger released, int expectedReleased) { final boolean returned = ref.release(); assertValues(retained, expectedRetained, released, expectedReleased); - Assert.assertEquals(expectedRetained == expectedReleased, returned); + Assertions.assertEquals(expectedRetained == expectedReleased, returned); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testWrap() { final String value = "testWrap"; final AtomicInteger retained = new AtomicInteger(); @@ -50,19 +52,19 @@ public void testWrap() { assertValues(retained, 0, released, 0); try { ref.get(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } assertValues(retained, 0, released, 0); - Assert.assertEquals(value, ref.retain()); + Assertions.assertEquals(value, ref.retain()); assertValues(retained, 1, released, 0); try(UncheckedAutoCloseableSupplier auto = ref.retainAndReleaseOnClose()) { final String got = auto.get(); - Assert.assertEquals(value, got); - Assert.assertSame(got, auto.get()); // it should return the same object. + Assertions.assertEquals(value, got); + Assertions.assertSame(got, auto.get()); // it should return the same object. assertValues(retained, 2, released, 0); } catch (IllegalStateException e) { e.printStackTrace(System.out); @@ -70,12 +72,12 @@ public void testWrap() { assertValues(retained, 2, released, 1); final UncheckedAutoCloseableSupplier notClosing = ref.retainAndReleaseOnClose(); - Assert.assertEquals(value, notClosing.get()); + Assertions.assertEquals(value, notClosing.get()); assertValues(retained, 3, released, 1); assertRelease(ref, retained, 3, released, 2); final UncheckedAutoCloseableSupplier auto = ref.retainAndReleaseOnClose(); - Assert.assertEquals(value, auto.get()); + Assertions.assertEquals(value, auto.get()); assertValues(retained, 4, released, 2); auto.close(); assertValues(retained, 4, released, 3); @@ -87,59 +89,60 @@ public void testWrap() { try { ref.get(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } try { ref.retain(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } try(UncheckedAutoCloseable ignore = ref.retainAndReleaseOnClose()) { - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } try { ref.release(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testReleaseWithoutRetaining() { final ReferenceCountedObject ref = ReferenceCountedObject.wrap(""); try { ref.release(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } try { ref.get(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } try { ref.retain(); - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } try(UncheckedAutoCloseable ignore = ref.retainAndReleaseOnClose()) { - Assert.fail(); + Assertions.fail(); } catch (IllegalStateException e) { e.printStackTrace(System.out); } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestResourceSemaphore.java b/ratis-test/src/test/java/org/apache/ratis/util/TestResourceSemaphore.java index 6fe1aed7ef..72c811d877 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestResourceSemaphore.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestResourceSemaphore.java @@ -19,15 +19,17 @@ import org.apache.ratis.BaseTest; import org.apache.ratis.RaftTestUtil; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import java.util.concurrent.TimeoutException; import static org.apache.ratis.util.ResourceSemaphore.Group.SUCCESS; public class TestResourceSemaphore extends BaseTest { - @Test(timeout = 5000) + @Test + @Timeout(value = 5) public void testGroup() throws InterruptedException, TimeoutException { final int FAILED_IN_ELEMENT_LIMIT = 0; final int FAILED_IN_BYTE_SIZE_LIMIT = 1; @@ -80,15 +82,15 @@ public void testGroup() throws InterruptedException, TimeoutException { } static void assertUsed(ResourceSemaphore.Group g, int... expected) { - Assert.assertEquals(expected.length, g.resourceSize()); + Assertions.assertEquals(expected.length, g.resourceSize()); for(int i = 0; i < expected.length; i++) { - Assert.assertEquals(expected[i], g.get(i).used()); + Assertions.assertEquals(expected[i], g.get(i).used()); } } static void assertAcquire(ResourceSemaphore.Group g, int expected, int... permits) { final int computed = g.tryAcquire(permits); - Assert.assertEquals(expected, computed); + Assertions.assertEquals(expected, computed); } static Runnable acquire(ResourceSemaphore.Group g, int... permits) { diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestTermIndex.java b/ratis-test/src/test/java/org/apache/ratis/util/TestTermIndex.java new file mode 100644 index 0000000000..678d7afe68 --- /dev/null +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestTermIndex.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.util; + +import org.apache.ratis.BaseTest; +import org.apache.ratis.RaftTestUtil; +import org.apache.ratis.server.protocol.ProtocolTestUtils; +import org.apache.ratis.server.protocol.TermIndex; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** Testing {@link BiWeakValueCache}. */ +public class TestTermIndex extends BaseTest { + static BiWeakValueCache CACHE = ProtocolTestUtils.getTermIndexCache(); + + static void dumpCache(Integer expectedEmptyCount) { + final int computed = CACHE.dump(System.out::print); + if (expectedEmptyCount != null) { + assertEquals(expectedEmptyCount, computed); + } + System.out.flush(); + } + + static void assertCacheSize(int expectedSize, long term) { + final int computed = CACHE.count(term); + if (computed != expectedSize) { + dumpCache(null); + } + assertEquals(expectedSize, computed); + } + + void assertCacheSizeWithGC(int expectedSize, long term) throws Exception{ + JavaUtils.attempt(() -> { + RaftTestUtil.gc(); + assertCacheSize(expectedSize, term); + }, 5, HUNDRED_MILLIS, "assertCacheSizeWithGC", LOG); + } + + static void initTermIndex(TermIndex[][] ti, int term, int index) { + ti[term][index] = TermIndex.valueOf(term, index); + } + + @Test + public void testCaching() throws Exception { + final int n = 9; + final TermIndex[][] ti = new TermIndex[n][n]; + final long[] terms = new long[n]; + final long[] indices = new long[n]; + for(int j = 0; j < n; j++) { + terms[j] = j; + indices[j] = j; + } + + assertCacheSize(0, terms[1]); + initTermIndex(ti, 1, 1); + assertSame(ti[1][1], TermIndex.valueOf(terms[1], indices[1])); + assertCacheSize(1, terms[1]); + + initTermIndex(ti, 1, 2); + assertSame(ti[1][1], TermIndex.valueOf(terms[1], indices[1])); + assertSame(ti[1][2], TermIndex.valueOf(terms[1], indices[2])); + assertCacheSize(2, terms[1]); + dumpCache(0); + + initTermIndex(ti, 2, 2); + assertSame(ti[1][1], TermIndex.valueOf(terms[1], indices[1])); + assertSame(ti[1][2], TermIndex.valueOf(terms[1], indices[2])); + assertSame(ti[2][2], TermIndex.valueOf(terms[2], indices[2])); + assertCacheSize(2, terms[1]); + assertCacheSize(1, terms[2]); + dumpCache(0); + + ti[1][1] = null; // release ti[1][1]; + assertCacheSizeWithGC(1, terms[1]); + dumpCache(0); + + ti[1][2] = null; // release ti[1][2]; + assertCacheSizeWithGC(0, terms[1]); + dumpCache(1); + + CACHE.cleanupEmptyInnerMaps(); + dumpCache(0); + } +} diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestTimeDuration.java b/ratis-test/src/test/java/org/apache/ratis/util/TestTimeDuration.java index a2c180d922..9ba5e57235 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestTimeDuration.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestTimeDuration.java @@ -17,8 +17,9 @@ */ package org.apache.ratis.util; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.event.Level; import java.util.Arrays; @@ -28,17 +29,18 @@ import static org.apache.ratis.util.TimeDuration.Abbreviation; import static org.apache.ratis.util.TimeDuration.parse; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestTimeDuration { { Slf4jUtils.setLogLevel(TimeDuration.LOG, Level.DEBUG); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testAbbreviation() { Arrays.asList(TimeUnit.values()) .forEach(a -> assertNotNull(Abbreviation.valueOf(a.name()))); @@ -51,10 +53,11 @@ public void testAbbreviation() { Arrays.asList(TimeUnit.values()).forEach(unit -> allSymbols.stream() .map(s -> "0" + s) - .forEach(s -> assertEquals(s, 0L, parse(s, unit)))); + .forEach(s -> assertEquals(0L, parse(s, unit), s))); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testParse() { assertEquals(1L, parse("1_000_000 ns", TimeUnit.MILLISECONDS)); assertEquals(10L, parse("10_000_000 nanos", TimeUnit.MILLISECONDS)); @@ -93,7 +96,8 @@ public void testParse() { assertEquals(2400, parse("100 days", TimeUnit.HOURS)); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testRoundUp() { final long nanosPerSecond = 1_000_000_000L; final TimeDuration oneSecond = TimeDuration.valueOf(1, TimeUnit.SECONDS); @@ -108,7 +112,8 @@ public void testRoundUp() { assertEquals(2*nanosPerSecond, oneSecond.roundUpNanos(nanosPerSecond + 1)); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testTo() { final TimeDuration oneSecond = TimeDuration.valueOf(1, TimeUnit.SECONDS); assertTo(1000, "1000ms", oneSecond, TimeUnit.MILLISECONDS); @@ -131,7 +136,8 @@ static TimeDuration assertTo(long expected, String expectedString, TimeDuration return computed; } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testAddAndSubtract() { final TimeDuration oneSecond = TimeDuration.valueOf(1, TimeUnit.SECONDS); final TimeDuration tenSecond = TimeDuration.valueOf(10, TimeUnit.SECONDS); @@ -184,28 +190,30 @@ public void testAddAndSubtract() { } } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testNegate() { assertNegate(0); assertNegate(1); assertNegate(-1); assertNegate(Long.MAX_VALUE); - Assert.assertEquals( + Assertions.assertEquals( TimeDuration.valueOf(Long.MAX_VALUE, TimeUnit.SECONDS), TimeDuration.valueOf(Long.MIN_VALUE, TimeUnit.SECONDS).negate()); } private static void assertNegate(long n) { - Assert.assertEquals( + Assertions.assertEquals( TimeDuration.valueOf(-n, TimeUnit.SECONDS), TimeDuration.valueOf(n, TimeUnit.SECONDS).negate()); - Assert.assertEquals( + Assertions.assertEquals( TimeDuration.valueOf(n, TimeUnit.SECONDS), TimeDuration.valueOf(-n, TimeUnit.SECONDS).negate()); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testMultiply() { assertMultiply(0, TimeDuration.ONE_SECOND, TimeDuration.valueOf(0, TimeUnit.SECONDS)); assertMultiply(0.001, TimeDuration.ONE_SECOND, TimeDuration.ONE_MILLISECOND); @@ -230,12 +238,13 @@ private static void assertMultiply(double multiplier, TimeDuration t, TimeDurati private static void assertMultiply(TimeDuration t, double multiplier, TimeDuration expected) { final TimeDuration computed = t.multiply(multiplier); TimeDuration.LOG.info("assertMultiply: {} x {} = {} ?= {}\n\n", t, multiplier, computed, expected); - Assert.assertEquals(expected.getUnit(), computed.getUnit()); + Assertions.assertEquals(expected.getUnit(), computed.getUnit()); final long d = Math.abs(computed.getDuration() - expected.getDuration()); - Assert.assertTrue(d <= Math.abs(expected.getDuration()) * TimeDuration.ERROR_THRESHOLD); + Assertions.assertTrue(d <= Math.abs(expected.getDuration()) * TimeDuration.ERROR_THRESHOLD); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testHigherLower() { final TimeUnit[] units = {TimeUnit.NANOSECONDS, TimeUnit.MICROSECONDS, TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS}; @@ -243,11 +252,12 @@ public void testHigherLower() { assertHigherLower(units[i-1], units[i]); } - Assert.assertSame(TimeUnit.NANOSECONDS, TimeDuration.lowerUnit(TimeUnit.NANOSECONDS)); - Assert.assertSame(TimeUnit.DAYS, TimeDuration.higherUnit(TimeUnit.DAYS)); + Assertions.assertSame(TimeUnit.NANOSECONDS, TimeDuration.lowerUnit(TimeUnit.NANOSECONDS)); + Assertions.assertSame(TimeUnit.DAYS, TimeDuration.higherUnit(TimeUnit.DAYS)); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testCompareTo() { assertTimeDurationCompareTo(TimeDuration.ONE_MINUTE, TimeDuration.ONE_SECOND); @@ -281,7 +291,7 @@ static void assertTimeDurationCompareTo(TimeDuration larger, TimeDuration smalle } private static void assertHigherLower(TimeUnit lower, TimeUnit higher) { - Assert.assertSame(lower, TimeDuration.lowerUnit(higher)); - Assert.assertSame(higher, TimeDuration.higherUnit(lower)); + Assertions.assertSame(lower, TimeDuration.lowerUnit(higher)); + Assertions.assertSame(higher, TimeDuration.higherUnit(lower)); } } diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestTimeoutScheduler.java b/ratis-test/src/test/java/org/apache/ratis/util/TestTimeoutScheduler.java index 848428f5b8..e8e9d6eb0c 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestTimeoutScheduler.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestTimeoutScheduler.java @@ -18,8 +18,9 @@ package org.apache.ratis.util; import org.apache.ratis.BaseTest; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.slf4j.event.Level; import java.util.concurrent.TimeUnit; @@ -41,52 +42,54 @@ public void accept(RuntimeException e) { } void assertNoError() { - Assert.assertFalse(hasError.get()); + Assertions.assertFalse(hasError.get()); } } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testSingleTask() throws Exception { final TimeoutScheduler scheduler = TimeoutScheduler.newInstance(); final TimeDuration grace = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); scheduler.setGracePeriod(grace); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertFalse(scheduler.hasScheduler()); final ErrorHandler errorHandler = new ErrorHandler(); final AtomicBoolean fired = new AtomicBoolean(false); scheduler.onTimeout(TimeDuration.valueOf(250, TimeUnit.MILLISECONDS), () -> { - Assert.assertFalse(fired.get()); + Assertions.assertFalse(fired.get()); fired.set(true); }, errorHandler); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertFalse(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertFalse(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertFalse(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertFalse(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired.get()); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertTrue(fired.get()); + Assertions.assertFalse(scheduler.hasScheduler()); errorHandler.assertNoError(); scheduler.setGracePeriod(grace); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testMultipleTasks() throws Exception { final TimeoutScheduler scheduler = TimeoutScheduler.newInstance(); final TimeDuration grace = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); scheduler.setGracePeriod(grace); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertFalse(scheduler.hasScheduler()); final ErrorHandler errorHandler = new ErrorHandler(); @@ -94,126 +97,129 @@ public void testMultipleTasks() throws Exception { for(int i = 0; i < fired.length; i++) { final AtomicBoolean f = fired[i] = new AtomicBoolean(false); scheduler.onTimeout(TimeDuration.valueOf(100*i + 50, TimeUnit.MILLISECONDS), () -> { - Assert.assertFalse(f.get()); + Assertions.assertFalse(f.get()); f.set(true); }, errorHandler); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(scheduler.hasScheduler()); } Thread.sleep(100); - Assert.assertTrue(fired[0].get()); - Assert.assertFalse(fired[1].get()); - Assert.assertFalse(fired[2].get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired[0].get()); + Assertions.assertFalse(fired[1].get()); + Assertions.assertFalse(fired[2].get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired[0].get()); - Assert.assertTrue(fired[1].get()); - Assert.assertFalse(fired[2].get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired[0].get()); + Assertions.assertTrue(fired[1].get()); + Assertions.assertFalse(fired[2].get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired[0].get()); - Assert.assertTrue(fired[1].get()); - Assert.assertTrue(fired[2].get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired[0].get()); + Assertions.assertTrue(fired[1].get()); + Assertions.assertTrue(fired[2].get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired[0].get()); - Assert.assertTrue(fired[1].get()); - Assert.assertTrue(fired[2].get()); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertTrue(fired[0].get()); + Assertions.assertTrue(fired[1].get()); + Assertions.assertTrue(fired[2].get()); + Assertions.assertFalse(scheduler.hasScheduler()); errorHandler.assertNoError(); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testExtendingGracePeriod() throws Exception { final TimeoutScheduler scheduler = TimeoutScheduler.newInstance(); final TimeDuration grace = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); scheduler.setGracePeriod(grace); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertFalse(scheduler.hasScheduler()); final ErrorHandler errorHandler = new ErrorHandler(); { final AtomicBoolean fired = new AtomicBoolean(false); scheduler.onTimeout(TimeDuration.valueOf(150, TimeUnit.MILLISECONDS), () -> { - Assert.assertFalse(fired.get()); + Assertions.assertFalse(fired.get()); fired.set(true); }, errorHandler); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertFalse(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertFalse(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); } { // submit another task during grace period final AtomicBoolean fired2 = new AtomicBoolean(false); scheduler.onTimeout(TimeDuration.valueOf(150, TimeUnit.MILLISECONDS), () -> { - Assert.assertFalse(fired2.get()); + Assertions.assertFalse(fired2.get()); fired2.set(true); }, errorHandler); Thread.sleep(100); - Assert.assertFalse(fired2.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertFalse(fired2.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired2.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired2.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired2.get()); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertTrue(fired2.get()); + Assertions.assertFalse(scheduler.hasScheduler()); } errorHandler.assertNoError(); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testRestartingScheduler() throws Exception { final TimeoutScheduler scheduler = TimeoutScheduler.newInstance(); final TimeDuration grace = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); scheduler.setGracePeriod(grace); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertFalse(scheduler.hasScheduler()); final ErrorHandler errorHandler = new ErrorHandler(); for(int i = 0; i < 2; i++) { final AtomicBoolean fired = new AtomicBoolean(false); scheduler.onTimeout(TimeDuration.valueOf(150, TimeUnit.MILLISECONDS), () -> { - Assert.assertFalse(fired.get()); + Assertions.assertFalse(fired.get()); fired.set(true); }, errorHandler); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertFalse(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertFalse(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired.get()); - Assert.assertTrue(scheduler.hasScheduler()); + Assertions.assertTrue(fired.get()); + Assertions.assertTrue(scheduler.hasScheduler()); Thread.sleep(100); - Assert.assertTrue(fired.get()); - Assert.assertFalse(scheduler.hasScheduler()); + Assertions.assertTrue(fired.get()); + Assertions.assertFalse(scheduler.hasScheduler()); } errorHandler.assertNoError(); } - @Test(timeout = 10_000) + @Test + @Timeout(value = 10) public void testShutdown() throws Exception { final TimeoutScheduler scheduler = TimeoutScheduler.newInstance(); - Assert.assertEquals(TimeoutScheduler.DEFAULT_GRACE_PERIOD, scheduler.getGracePeriod()); + Assertions.assertEquals(TimeoutScheduler.DEFAULT_GRACE_PERIOD, scheduler.getGracePeriod()); final ErrorHandler errorHandler = new ErrorHandler(); final int numTasks = 100; @@ -223,7 +229,7 @@ public void testShutdown() throws Exception { } HUNDRED_MILLIS.sleep(); HUNDRED_MILLIS.sleep(); - JavaUtils.attempt(() -> Assert.assertEquals(1, scheduler.getTaskCount()), + JavaUtils.attempt(() -> Assertions.assertEquals(1, scheduler.getTaskCount()), 10, HUNDRED_MILLIS, "only 1 shutdown task is scheduled", LOG); final TimeDuration oneMillis = TimeDuration.valueOf(1, TimeUnit.MILLISECONDS); @@ -234,7 +240,7 @@ public void testShutdown() throws Exception { oneMillis.sleep(); } HUNDRED_MILLIS.sleep(); - JavaUtils.attempt(() -> Assert.assertEquals(1, scheduler.getTaskCount()), + JavaUtils.attempt(() -> Assertions.assertEquals(1, scheduler.getTaskCount()), 10, HUNDRED_MILLIS, "only 1 shutdown task is scheduled", LOG); errorHandler.assertNoError(); diff --git a/ratis-test/src/test/java/org/apache/ratis/util/TestTraditionalBinaryPrefix.java b/ratis-test/src/test/java/org/apache/ratis/util/TestTraditionalBinaryPrefix.java index 8b6b7a378f..a699b91c81 100644 --- a/ratis-test/src/test/java/org/apache/ratis/util/TestTraditionalBinaryPrefix.java +++ b/ratis-test/src/test/java/org/apache/ratis/util/TestTraditionalBinaryPrefix.java @@ -17,15 +17,18 @@ */ package org.apache.ratis.util; -import org.junit.Test; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import static org.apache.ratis.util.TraditionalBinaryPrefix.long2String; import static org.apache.ratis.util.TraditionalBinaryPrefix.string2long; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; public class TestTraditionalBinaryPrefix { - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testTraditionalBinaryPrefix() { //test string2long(..) String[] symbol = {"k", "m", "g", "t", "p", "e"}; @@ -108,19 +111,19 @@ public void testTraditionalBinaryPrefix() { { // n = 2^e final long n = 1L << e; final String expected = (n/p.getValue()) + " " + p.getSymbol(); - assertEquals("n=" + n, expected, long2String(n, null, 2)); + assertEquals(expected, long2String(n, null, 2), "n=" + n); } { // n = 2^e + 1 final long n = (1L << e) + 1; final String expected = (n/p.getValue()) + trailingZeros + p.getSymbol(); - assertEquals("n=" + n, expected, long2String(n, null, decimalPlace)); + assertEquals(expected, long2String(n, null, decimalPlace), "n=" + n); } { // n = 2^e - 1 final long n = (1L << e) - 1; final String expected = ((n+1)/p.getValue()) + trailingZeros + p.getSymbol(); - assertEquals("n=" + n, expected, long2String(n, null, decimalPlace)); + assertEquals(expected, long2String(n, null, decimalPlace), "n=" + n); } } } @@ -143,7 +146,8 @@ private static String byteDescription(long len) { return long2String(len, "B", 2); } - @Test(timeout = 1000) + @Test + @Timeout(value = 1) public void testUnderscore() { final SizeInBytes value = SizeInBytes.valueOf("1_000_000_000_000_000"); assertEquals(1_000_000_000_000_000L, value.getSize()); diff --git a/ratis-tools/pom.xml b/ratis-tools/pom.xml index c1971ced96..f3dbb31882 100644 --- a/ratis-tools/pom.xml +++ b/ratis-tools/pom.xml @@ -17,12 +17,17 @@ ratis org.apache.ratis - 3.1.0-SNAPSHOT + 3.2.0-SNAPSHOT ratis-tools Apache Ratis Tools + + + true + + org.apache.ratis diff --git a/src/main/resources/ratis-version.properties b/src/main/resources/ratis-version.properties new file mode 100644 index 0000000000..f34dc73dc7 --- /dev/null +++ b/src/main/resources/ratis-version.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name=${project.name} +version=${project.version} +revision=${version-info.scm.commit}