From a433fa2c1b5dd44aa742c08745b69f50053a82c2 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 13:59:06 +0300 Subject: [PATCH 01/26] build with custom graph --- .github/actions/test_ya/action.yml | 9 ++-- .github/scripts/graph_compare.py | 75 ++++++++---------------------- 2 files changed, 22 insertions(+), 62 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 0d13d9af5b91..78d893161de1 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -252,13 +252,12 @@ runs: echo "::debug::get version" ./ya --version - export YA_MAKE_COMMAND="./ya make ${params[@]}" if [ "${{ inputs.increment }}" = "true" ]; then GRAPH_COMPARE_OUTPUT="$PUBLIC_DIR/graph_compare_log.txt" GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt" set +e - ./.github/scripts/graph_compare.py $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + ./.github/scripts/graph_compare.py --ya-make-command "./ya make ${params[@]}" --result-graph-path graph.json -result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e @@ -271,7 +270,7 @@ runs: fi git checkout $ORIGINAL_HEAD - YA_MAKE_TARGET=. + YA_MAKE_TARGET="--build-custom-json graph.json --custom-context context.json" else YA_MAKE_TARGET="" for TARGET in ${{ inputs.build_target }}; do @@ -297,8 +296,6 @@ runs: params+=(--retest) fi - export YA_MAKE_COMMAND="./ya make ${params[@]}" - YA_MAKE_OUT_DIR=$TMP_DIR/out YA_MAKE_OUTPUT="$PUBLIC_DIR/ya_make_output.txt" @@ -369,7 +366,7 @@ runs: MONITOR_PID=$! set +e - ($YA_MAKE_COMMAND $YA_MAKE_TARGET \ + (./ya make ${params[@]} $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index e6bbe9940073..9233989e4908 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -6,8 +6,8 @@ import os import tempfile -import sys import json +import argparse def exec(command: str): @@ -22,17 +22,11 @@ def log(msg: str): print(msg) -def do_compare(): - if len(sys.argv) < 3: - print('base or head commit not set') - exit(1) - base_commit = sys.argv[1] - head_commit = sys.argv[2] - - ya_make_command = os.getenv('YA_MAKE_COMMAND') - if not ya_make_command: - print('YA_MAKE_COMMAND not set') +def do_compare(opts): + if opts.ya_make_command: + print('--ya-make-command not set') exit(1) + ya = opts.ya_make_command.split(' ')[0] workdir = os.getenv('workdir') if not workdir: @@ -40,59 +34,28 @@ def do_compare(): log(f'Workdir: {workdir}') log('Checkout base commit...') - exec(f'git checkout {base_commit}') + exec(f'git checkout {opts.base_commit}') log('Build graph for base commit...') - exec(f'{ya_make_command} ydb -k -A --cache-tests -Gj0 > {workdir}/graph_base.json') + exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_base.json --save-context-to {workdir}/context_base.json') log('Checkout head commit...') - exec(f'git checkout {head_commit}') + exec(f'git checkout {opts.head_commit}') log('Build graph for head commit...') - exec(f'{ya_make_command} ydb -k -A --cache-tests -Gj0 > {workdir}/graph_head.json') + exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'./ya tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {workdir}/graph_diff.json --dump-uids-for-affected-nodes {workdir}/affected_uids.json') - - log('Read diff graph...') - with open(f'{workdir}/graph_diff.json', 'r') as f: - diff_graph = json.load(f) - - with open(f'{workdir}/affected_uids.json', 'r') as f: - uids = set(json.load(f)) - - tests = set() - modules = set() - - log('Scan diff graph...') - for target in diff_graph.get('graph', []): - if target.get('uid') not in uids: - continue - if target.get('node-type') == 'test': - path = target.get('kv', {}).get('path') - if path is not None: - tests.add(os.path.dirname(path)) - tp = target.get('target_properties') - if ( - tp is not None - and tp.get('module_type') is not None - and tp.get('module_dir', '').startswith('ydb') - and tp.get('module_tag', '').find('proto') < 0 - ): - modules.add(tp.get('module_dir')) - - log('Create ya.make') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.result_graph_path} --dump-uids {workdir}/uids.json') - with open('ya.make', 'w') as ya_make: - ya_make.write('RECURSE_FOR_TESTS(\n') - for test in sorted(tests): - ya_make.write(f' {test}\n') - ya_make.write(')\n\nRECURSE (\n') - for module in sorted(modules): - ya_make.write(f' {module}\n') - ya_make.write(')\n') - log('ya.make content:') - exec('cat ya.make') + log('Generate diff context...') + exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.result_context_path} {workdir}/uids.json {opts.result_graph_path}') exit(0) if __name__ == '__main__': - do_compare() + parser = argparse.ArgumentParser() + parser.add_argument('--result-graph-path', '-g', type=str, help='Path for result graph', dest='graph_path', required=True) + parser.add_argument('--result-context-path', '-c', type=str, help='Path for result context', dest='context_path', required=True) + parser.add_argument('--ya-make-command', '-y', type=str, help='Ya make command', dest='ya_make_command', required=True) + parser.add_argument(dest='base_commit', help='Base commit') + parser.add_argument(dest='head_commit', help='Head commit') + do_compare(parser.parse_args()) From 4d7fda755175dfab6f6f7a139a03e0f274666425 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 14:07:26 +0300 Subject: [PATCH 02/26] fix --- .github/actions/test_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 78d893161de1..297f03334bd8 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -257,7 +257,7 @@ runs: GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt" set +e - ./.github/scripts/graph_compare.py --ya-make-command "./ya make ${params[@]}" --result-graph-path graph.json -result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + ./.github/scripts/graph_compare.py --ya-make-command "./ya make ${params[@]}" --result-graph-path graph.json --result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e From 86a977ee72b411b32800672da8d3c845cae6c3d4 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 14:17:04 +0300 Subject: [PATCH 03/26] fix cmd line --- .github/actions/test_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 297f03334bd8..3e310c8202ed 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -257,7 +257,7 @@ runs: GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt" set +e - ./.github/scripts/graph_compare.py --ya-make-command "./ya make ${params[@]}" --result-graph-path graph.json --result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + ./.github/scripts/graph_compare.py --ya-make-command "'./ya make ${params[@]}'" --result-graph-path graph.json --result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e From c40537072a6f39ff128fc8632b8012d92cfed20e Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 14:34:06 +0300 Subject: [PATCH 04/26] fix --- .github/actions/test_ya/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 3e310c8202ed..7453fbc5e24b 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -257,7 +257,8 @@ runs: GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt" set +e - ./.github/scripts/graph_compare.py --ya-make-command "'./ya make ${params[@]}'" --result-graph-path graph.json --result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + YA_MAKE_COMMAD="./ya make ${params[@]}" + ./.github/scripts/graph_compare.py --ya-make-command "$YA_MAKE_COMMAD" --result-graph-path graph.json --result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e From 41cdc6a439cbee06df6d4e2dcf93d24b3657b76e Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 14:39:46 +0300 Subject: [PATCH 05/26] fix --- .github/actions/test_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 7453fbc5e24b..6021bea51a08 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -258,7 +258,7 @@ runs: set +e YA_MAKE_COMMAD="./ya make ${params[@]}" - ./.github/scripts/graph_compare.py --ya-make-command "$YA_MAKE_COMMAD" --result-graph-path graph.json --result-context-path context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + ./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAD" --result-graph-path=graph.json --result-context-path=context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e From e80bb3c0cbfdc772f8a3f82fbaf05b304a2ccd3c Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 14:45:33 +0300 Subject: [PATCH 06/26] fix --- .github/scripts/graph_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index 9233989e4908..08c8839356b3 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -23,7 +23,7 @@ def log(msg: str): def do_compare(opts): - if opts.ya_make_command: + if not opts.ya_make_command: print('--ya-make-command not set') exit(1) ya = opts.ya_make_command.split(' ')[0] From 3f47bc20d0ec97bd75e54753a98006eac0765700 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 14:54:33 +0300 Subject: [PATCH 07/26] fix --- .github/scripts/graph_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index 08c8839356b3..bc736e29bd5e 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -44,10 +44,10 @@ def do_compare(opts): exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.result_graph_path} --dump-uids {workdir}/uids.json') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json') log('Generate diff context...') - exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.result_context_path} {workdir}/uids.json {opts.result_graph_path}') + exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.context_path} {workdir}/uids.json {opts.graph_path}') exit(0) From 0eaf3fa9d7bf751a67644a791ab27cfe51e2efe4 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 18:20:07 +0300 Subject: [PATCH 08/26] fix --- .github/actions/test_ya/action.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 6021bea51a08..e16d0e3f88bc 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -257,8 +257,11 @@ runs: GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt" set +e - YA_MAKE_COMMAD="./ya make ${params[@]}" - ./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAD" --result-graph-path=graph.json --result-context-path=context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + YA_MAKE_COMMAND="./ya make ${params[@]}" + if [ true = ${{ inputs.run_tests }} ]; then + YA_MAKE_COMMAND="$YA_MAKE_COMMAND -A" + fi + ./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAND" --result-graph-path=graph.json --result-context-path=context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e @@ -271,7 +274,7 @@ runs: fi git checkout $ORIGINAL_HEAD - YA_MAKE_TARGET="--build-custom-json graph.json --custom-context context.json" + YA_MAKE_TARGET="--build-custom-json graph.json --custom-context context.json ydb" else YA_MAKE_TARGET="" for TARGET in ${{ inputs.build_target }}; do From 9af27829550bb1471f6f6e56ce581e6e06276c22 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 19:57:21 +0300 Subject: [PATCH 09/26] use absolute paths --- .github/actions/test_ya/action.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index e16d0e3f88bc..910677c23cf4 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -261,7 +261,9 @@ runs: if [ true = ${{ inputs.run_tests }} ]; then YA_MAKE_COMMAND="$YA_MAKE_COMMAND -A" fi - ./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAND" --result-graph-path=graph.json --result-context-path=context.json $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + GRAPH_PATH=$(realpath graph.json) + CONTEXT_PATH=$(realpath context.json) + ./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAND" --result-graph-path=$GRAPH_PATH --result-context-path=$CONTEXT_PATH $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e @@ -274,7 +276,11 @@ runs: fi git checkout $ORIGINAL_HEAD - YA_MAKE_TARGET="--build-custom-json graph.json --custom-context context.json ydb" + params+=( + --build-custom-json=$GRAPH_PATH + --custom-context=$CONTEXT_PATH + ) + YA_MAKE_TARGET="ydb" else YA_MAKE_TARGET="" for TARGET in ${{ inputs.build_target }}; do From cbe48b4e737ee071371ad3a9d6db03aba6bbde08 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 21:45:06 +0300 Subject: [PATCH 10/26] no put in cach --- .github/actions/test_ya/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 910677c23cf4..77bb005ec665 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -295,11 +295,11 @@ runs: params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") fi - if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then - params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") - params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") - params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) - fi + # if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then + # params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") + # params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") + # params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) + # fi if [ true = ${{ inputs.run_tests }} ]; then params+=(-A) From 39f17bd032ba4d050fe0137185d6243c93cdd2af Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Fri, 31 Oct 2025 22:00:46 +0300 Subject: [PATCH 11/26] fix put-cache --- .github/actions/test_ya/action.yml | 10 +++++----- .github/workflows/pr_check.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 77bb005ec665..910677c23cf4 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -295,11 +295,11 @@ runs: params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") fi - # if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then - # params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") - # params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") - # params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) - # fi + if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then + params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") + params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") + params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) + fi if [ true = ${{ inputs.run_tests }} ]; then params+=(-A) diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 383ba72117bb..9d084e849d0d 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -252,7 +252,7 @@ jobs: run_tests: ${{ contains(fromJSON('["relwithdebinfo", "release-asan", "release-tsan", "release-msan"]'), matrix.build_preset) }} test_size: ${{ matrix.test_size }} test_threads: ${{ matrix.threads_count }} - put_build_results_to_cache: true + put_build_results_to_cache: false additional_ya_make_args: -DDEBUGINFO_LINES_ONLY # we don't need full symbols in CI checks secs: ${{ format('{{"AWS_KEY_ID":"{0}","AWS_KEY_VALUE":"{1}","REMOTE_CACHE_USERNAME":"{2}","REMOTE_CACHE_PASSWORD":"{3}","TELEGRAM_YDBOT_TOKEN":"{4}"}}', secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD, secrets.TELEGRAM_YDBOT_TOKEN ) }} From 0fe716a2f74fdd7df3e33d42354da63b0a273624 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Sat, 8 Nov 2025 09:44:31 +0300 Subject: [PATCH 12/26] patch graph when rerun --- .github/actions/test_ya/action.yml | 12 ++- .github/scripts/graph_compare.py | 5 +- .github/scripts/graph_patch.py | 160 +++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 4 deletions(-) create mode 100755 .github/scripts/graph_patch.py diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 910677c23cf4..d3d19bcf4e6e 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -354,13 +354,16 @@ runs: RERUN_FAILED_OPT="" else CURRENT_MESSAGE="$CURRENT_MESSAGE (failed tests rerun, try $RETRY)" - RERUN_FAILED_OPT="-X" + if [ -z "$GRAPH_PATH" ]; then + RERUN_FAILED_OPT="-X" + fi fi echo $CURRENT_MESSAGE | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py CURRENT_JUNIT_XML_PATH=$CURRENT_PUBLIC_DIR/junit.xml CURRENT_REPORT=$CURRENT_PUBLIC_DIR/report.json + CURRENT_JSONL_REPORT=$CURRENT_PUBLIC_DIR/report.jsonl monitor_memory() { set +x @@ -379,6 +382,7 @@ runs: (./ya make ${params[@]} $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ + ---jsnol-report "$CURRENT_JSONL_REPORT" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT set -e RC=`cat exit_code` @@ -496,6 +500,12 @@ runs: if [ $IS_LAST_RETRY = 1 ]; then break fi + if [ -n "$GRAPH_PATH" ] && [ -n "$CONTEXT_PATH" ]; then + .github/scripts/graph_patch.py \ + --in-graph="$GRAPH_PATH" --in-context="$CONTEXT_PATH" \ + --out-graph="$GRAPH_PATH" --out-context="$CONTEXT_PATH" \ + --report="$CURRENT_JSONL_REPORT" --muted=.github/config/muted_ya.txt + fi done; if [ $BUILD_FAILED = 0 ]; then diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index bc736e29bd5e..5e1b3cb5e199 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -6,7 +6,6 @@ import os import tempfile -import json import argparse @@ -53,8 +52,8 @@ def do_compare(opts): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--result-graph-path', '-g', type=str, help='Path for result graph', dest='graph_path', required=True) - parser.add_argument('--result-context-path', '-c', type=str, help='Path for result context', dest='context_path', required=True) + parser.add_argument('--result-graph-path', '-g', type=str, help='Path to result graph', dest='graph_path', required=True) + parser.add_argument('--result-context-path', '-c', type=str, help='Path to result context', dest='context_path', required=True) parser.add_argument('--ya-make-command', '-y', type=str, help='Ya make command', dest='ya_make_command', required=True) parser.add_argument(dest='base_commit', help='Base commit') parser.add_argument(dest='head_commit', help='Head commit') diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py new file mode 100755 index 000000000000..6cc007b49eb9 --- /dev/null +++ b/.github/scripts/graph_patch.py @@ -0,0 +1,160 @@ +#! /usr/bin/python3 -u + +from __future__ import annotations +import argparse +import json +import re +import pickle +import six + + +class MuteTestCheck: + def __pattern_to_re(self, pattern): + res = [] + for c in pattern: + if c == '*': + res.append('.*') + else: + res.append(re.escape(c)) + + return f"(?:^{''.join(res)}$)" + + def __init__(self, fn): + self.regexps = [] + + with open(fn, 'r') as fp: + for line in fp: + line = line.strip() + pattern = self.__pattern_to_re(line) + + try: + self.regexps.append(re.compile(pattern)) + except re.error: + print(f"Unable to compile regex {pattern!r}") + raise + + def __call__(self, fullname): + for r in self.regexps: + if r.match(fullname): + return True + return False + +def _patch_uid(uid: str) -> str: + return uid # f'{uid}-I' + +def get_failed_uids(opts) -> set[str]: + print('Load failed uids..') + result = set() + mute_check = MuteTestCheck(opts.muted) if opts.muted else None + with open(opts.report) as report_file: + for line in report_file.readlines(): + record = json.loads(line).get('data', {}) + if record.get('status', 'OK') == 'OK' or record.get('suite', False): + continue + if mute_check is not None: + test_name = f'{record.get("path", "")} {record.get("name", "")}.{record.get("subtest_name", "")}' + if mute_check(test_name): + continue + uid = record.get('uid') + if uid: + result.add(uid) + print(f'{len(result)} uids loaded') + return result + + +def _strip_graph(graph: dict, uids_filter: set[str]) -> dict: + result = {uid for uid in graph['result'] if uid in uids_filter} + nodes = _strip_unused_nodes(graph['graph'], result) + + conf = graph.get('conf', {}).copy() + conf['resources'] = _filter_duplicate_resources(conf.get('resources', [])) + + return {'conf': conf, 'inputs': graph.get('inputs', {}), 'result': [_patch_uid(uid) for uid in result], 'graph': nodes} + + +def _strip_unused_nodes(graph_nodes: list, result: set[str]) -> list[dict]: + by_uid = {n['uid']: n for n in graph_nodes} + + def visit(uid): + if uid in by_uid: + node = by_uid.pop(uid) + yield node + for dep in node['deps']: + yield from visit(dep) + + result_nodes: list[dict] = [] + for uid in result: + for node in visit(uid): + if node['uid'] == uid: + node['uid'] = _patch_uid(uid) + result_nodes.append(node) + + return result_nodes + + +def _filter_duplicate_resources(resources): + v = set() + result = [] + for x in resources: + if x['pattern'] not in v: + v.add(x['pattern']) + result.append(x) + return result + + +def process_graph(opts, uids_filter: set[str]) -> None: + print('Load graph...') + with open(opts.in_graph) as f: + in_graph = json.load(f) + print('Strip graph...') + out_graph = _strip_graph(in_graph, uids_filter) + print('Save graph...') + with open(opts.out_graph, 'w') as f: + json.dump(out_graph, f, indent=2) + print('Process graph...OK') + + +def process_context(opts, uids_filter: set[str]) -> None: + print('Load context...') + with open(opts.in_context) as f: + in_context = json.load(f) + out_context = {} + print('Strip context...') + for k, v in in_context.items(): + if k == 'tests': + new_tests = {} + for uid in v.keys(): + if uid in uids_filter: + pathced_uid = _patch_uid(uid) +# replaces = {uid: pathced_uid} +# test = pickle.loads(six.ensure_binary(v[uid], encoding='latin-1'), encoding='utf-8') +# test.uid = pathced_uid +# test._result_uids = [replaces.get(dep_uid, dep_uid) for dep_uid in test._result_uids] +# test._output_uids = [replaces.get(dep_uid, dep_uid) for dep_uid in test._output_uids] +# test.dep_uids = [replaces.get(dep_uid, dep_uid) for dep_uid in test.dep_uids] +# test.change_build_dep_uids({uid: pathced_uid}) +# new_tests[pathced_uid] = six.ensure_str(pickle.dumps(test), encoding='latin-1') + new_tests[pathced_uid] = v[uid] + out_context[k] = new_tests + elif k == 'graph': + out_context[k] = _strip_graph(v, uids_filter) + else: + out_context[k] = v + print('Save context...') + with open(opts.out_context, 'w') as f: + json.dump(out_context, f, indent=2) + print('Process context...OK') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--in-graph', '-G', type=str, help='Path to input graph', dest='in_graph', required=True) + parser.add_argument('--in-context', '-C', type=str, help='Path to input context', dest='in_context', required=True) + parser.add_argument('--out-graph', '-g', type=str, help='Path to result graph', dest='out_graph', required=True) + parser.add_argument('--out-context', '-c', type=str, help='Path to result context', dest='out_context', required=True) + parser.add_argument('--report', '-r', type=str, help='Path to jsonl report', dest='report', required=True) + parser.add_argument('--muted', '-m', type=str, help='Path to muted tests', dest='muted') + opts = parser.parse_args() + uids = get_failed_uids(opts) + process_graph(opts, uids) + process_context(opts, uids) From c5aea5c1f255050e897b4148b53976b5ff352b7b Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Sat, 8 Nov 2025 09:54:40 +0300 Subject: [PATCH 13/26] fix --- .github/actions/test_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index d3d19bcf4e6e..96e53cda3b36 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -382,7 +382,7 @@ runs: (./ya make ${params[@]} $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ - ---jsnol-report "$CURRENT_JSONL_REPORT" \ + --jsnol-report "$CURRENT_JSONL_REPORT" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT set -e RC=`cat exit_code` From d1111ccef2377793dab91be737c70bbd9af0edac Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Sat, 8 Nov 2025 10:21:26 +0300 Subject: [PATCH 14/26] fix --- .github/actions/test_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 96e53cda3b36..14b88d63023b 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -382,7 +382,7 @@ runs: (./ya make ${params[@]} $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ - --jsnol-report "$CURRENT_JSONL_REPORT" \ + --jsonl-report "$CURRENT_JSONL_REPORT" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT set -e RC=`cat exit_code` From ff20c05a2a599cfae57102a8eecef64210a915d9 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 06:54:45 +0300 Subject: [PATCH 15/26] Uncache for retest --- .github/scripts/graph_patch.py | 1 + .github/workflows/pr_check.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index 6cc007b49eb9..4a36f259cc85 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -87,6 +87,7 @@ def visit(uid): for node in visit(uid): if node['uid'] == uid: node['uid'] = _patch_uid(uid) + node['cache'] = False result_nodes.append(node) return result_nodes diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 9d084e849d0d..383ba72117bb 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -252,7 +252,7 @@ jobs: run_tests: ${{ contains(fromJSON('["relwithdebinfo", "release-asan", "release-tsan", "release-msan"]'), matrix.build_preset) }} test_size: ${{ matrix.test_size }} test_threads: ${{ matrix.threads_count }} - put_build_results_to_cache: false + put_build_results_to_cache: true additional_ya_make_args: -DDEBUGINFO_LINES_ONLY # we don't need full symbols in CI checks secs: ${{ format('{{"AWS_KEY_ID":"{0}","AWS_KEY_VALUE":"{1}","REMOTE_CACHE_USERNAME":"{2}","REMOTE_CACHE_PASSWORD":"{3}","TELEGRAM_YDBOT_TOKEN":"{4}"}}', secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD, secrets.TELEGRAM_YDBOT_TOKEN ) }} From 1b09b454fc40119a66acf1ed03fad0525a9a96f4 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 07:12:15 +0300 Subject: [PATCH 16/26] no patch uids --- .github/scripts/graph_patch.py | 45 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index 4a36f259cc85..83127e064fbc 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -4,8 +4,6 @@ import argparse import json import re -import pickle -import six class MuteTestCheck: @@ -39,8 +37,6 @@ def __call__(self, fullname): return True return False -def _patch_uid(uid: str) -> str: - return uid # f'{uid}-I' def get_failed_uids(opts) -> set[str]: print('Load failed uids..') @@ -69,7 +65,7 @@ def _strip_graph(graph: dict, uids_filter: set[str]) -> dict: conf = graph.get('conf', {}).copy() conf['resources'] = _filter_duplicate_resources(conf.get('resources', [])) - return {'conf': conf, 'inputs': graph.get('inputs', {}), 'result': [_patch_uid(uid) for uid in result], 'graph': nodes} + return {'conf': conf, 'inputs': graph.get('inputs', {}), 'result': [uid for uid in result], 'graph': nodes} def _strip_unused_nodes(graph_nodes: list, result: set[str]) -> list[dict]: @@ -85,8 +81,7 @@ def visit(uid): result_nodes: list[dict] = [] for uid in result: for node in visit(uid): - if node['uid'] == uid: - node['uid'] = _patch_uid(uid) + if uid in result: node['cache'] = False result_nodes.append(node) @@ -126,16 +121,7 @@ def process_context(opts, uids_filter: set[str]) -> None: new_tests = {} for uid in v.keys(): if uid in uids_filter: - pathced_uid = _patch_uid(uid) -# replaces = {uid: pathced_uid} -# test = pickle.loads(six.ensure_binary(v[uid], encoding='latin-1'), encoding='utf-8') -# test.uid = pathced_uid -# test._result_uids = [replaces.get(dep_uid, dep_uid) for dep_uid in test._result_uids] -# test._output_uids = [replaces.get(dep_uid, dep_uid) for dep_uid in test._output_uids] -# test.dep_uids = [replaces.get(dep_uid, dep_uid) for dep_uid in test.dep_uids] -# test.change_build_dep_uids({uid: pathced_uid}) -# new_tests[pathced_uid] = six.ensure_str(pickle.dumps(test), encoding='latin-1') - new_tests[pathced_uid] = v[uid] + new_tests[uid] = v[uid] out_context[k] = new_tests elif k == 'graph': out_context[k] = _strip_graph(v, uids_filter) @@ -149,11 +135,26 @@ def process_context(opts, uids_filter: set[str]) -> None: if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--in-graph', '-G', type=str, help='Path to input graph', dest='in_graph', required=True) - parser.add_argument('--in-context', '-C', type=str, help='Path to input context', dest='in_context', required=True) - parser.add_argument('--out-graph', '-g', type=str, help='Path to result graph', dest='out_graph', required=True) - parser.add_argument('--out-context', '-c', type=str, help='Path to result context', dest='out_context', required=True) - parser.add_argument('--report', '-r', type=str, help='Path to jsonl report', dest='report', required=True) + parser.add_argument( + '--in-graph', '-G', type=str, dest='in_graph', required=True, + help='Path to input graph' + ) + parser.add_argument( + '--in-context', '-C', type=str, dest='in_context', required=True, + help='Path to input context' + ) + parser.add_argument( + '--out-graph', '-g', type=str, dest='out_graph', required=True, + help='Path to result graph' + ) + parser.add_argument( + '--out-context', '-c', type=str, dest='out_context', required=True, + help='Path to result context' + ) + parser.add_argument( + '--report', '-r', type=str, dest='report', required=True, + help='Path to jsonl report' + ) parser.add_argument('--muted', '-m', type=str, help='Path to muted tests', dest='muted') opts = parser.parse_args() uids = get_failed_uids(opts) From 8ac3abb8bfe6382fa4e9f28fbb4c602431746e2e Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 07:37:43 +0300 Subject: [PATCH 17/26] --no-cache-for-affected-nodes --- .github/scripts/graph_compare.py | 2 +- .github/scripts/graph_patch.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index 5e1b3cb5e199..9bee9ee64240 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -43,7 +43,7 @@ def do_compare(opts): exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') log('Generate diff context...') exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.context_path} {workdir}/uids.json {opts.graph_path}') diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index 83127e064fbc..5cd1a4c05cc5 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -81,8 +81,6 @@ def visit(uid): result_nodes: list[dict] = [] for uid in result: for node in visit(uid): - if uid in result: - node['cache'] = False result_nodes.append(node) return result_nodes From eb59e2d0e02a60bb5299e5c6a4bb2588981b54ac Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 09:08:19 +0300 Subject: [PATCH 18/26] do not use cache on retest --- .github/actions/test_ya/action.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 14b88d63023b..42d5b57cc484 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -290,17 +290,6 @@ runs: done fi - if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then - params+=(--bazel-remote-store) - params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") - fi - - if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then - params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") - params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") - params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) - fi - if [ true = ${{ inputs.run_tests }} ]; then params+=(-A) params+=(--retest) @@ -349,9 +338,20 @@ runs: mkdir $TEST_META_INFO CURRENT_MESSAGE="ya make is running..." + CACHE_OPTS="" if [ $IS_RETRY = 0 ]; then CURRENT_MESSAGE="$CURRENT_MESSAGE" RERUN_FAILED_OPT="" + if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then + CACHE_OPTS+=(--bazel-remote-store) + CACHE_OPTS+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") + fi + + if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then + CACHE_OPTS+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") + CACHE_OPTS+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") + CACHE_OPTS+=(--bazel-remote-put --dist-cache-max-file-size=209715200) + fi else CURRENT_MESSAGE="$CURRENT_MESSAGE (failed tests rerun, try $RETRY)" if [ -z "$GRAPH_PATH" ]; then @@ -379,7 +379,7 @@ runs: MONITOR_PID=$! set +e - (./ya make ${params[@]} $YA_MAKE_TARGET \ + (./ya make ${params[@]} $CACHE_OPTS $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ --jsonl-report "$CURRENT_JSONL_REPORT" \ From 1ce10d36cba612b040e52dab3e81aa5c9133286a Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 09:57:27 +0300 Subject: [PATCH 19/26] use json report instead jsonl one --- .github/actions/test_ya/action.yml | 4 +--- .github/scripts/graph_patch.py | 28 ++++++++++++---------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 6b199fbea471..2306ac64e29f 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -377,7 +377,6 @@ runs: CURRENT_JUNIT_XML_PATH=$CURRENT_PUBLIC_DIR/junit.xml CURRENT_REPORT=$CURRENT_PUBLIC_DIR/report.json - CURRENT_JSONL_REPORT=$CURRENT_PUBLIC_DIR/report.jsonl monitor_memory() { set +x @@ -396,7 +395,6 @@ runs: (./ya make ${params[@]} $CACHE_OPTS $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ - --jsonl-report "$CURRENT_JSONL_REPORT" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT set -e RC=`cat exit_code` @@ -540,7 +538,7 @@ runs: .github/scripts/graph_patch.py \ --in-graph="$GRAPH_PATH" --in-context="$CONTEXT_PATH" \ --out-graph="$GRAPH_PATH" --out-context="$CONTEXT_PATH" \ - --report="$CURRENT_JSONL_REPORT" --muted=.github/config/muted_ya.txt + --report="$CURRENT_REPORT" --muted=.github/config/muted_ya.txt fi done; diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index 5cd1a4c05cc5..55cc7096dc03 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -43,17 +43,17 @@ def get_failed_uids(opts) -> set[str]: result = set() mute_check = MuteTestCheck(opts.muted) if opts.muted else None with open(opts.report) as report_file: - for line in report_file.readlines(): - record = json.loads(line).get('data', {}) - if record.get('status', 'OK') == 'OK' or record.get('suite', False): + report = json.load(report_file).get('results', []) + for record in report: + if record.get('status', 'OK') == 'OK' or record.get('suite', False): + continue + if mute_check is not None: + test_name = f'{record.get("path", "")} {record.get("name", "")}.{record.get("subtest_name", "")}' + if mute_check(test_name): continue - if mute_check is not None: - test_name = f'{record.get("path", "")} {record.get("name", "")}.{record.get("subtest_name", "")}' - if mute_check(test_name): - continue - uid = record.get('uid') - if uid: - result.add(uid) + uid = record.get('uid') + if uid: + result.add(uid) print(f'{len(result)} uids loaded') return result @@ -116,11 +116,7 @@ def process_context(opts, uids_filter: set[str]) -> None: print('Strip context...') for k, v in in_context.items(): if k == 'tests': - new_tests = {} - for uid in v.keys(): - if uid in uids_filter: - new_tests[uid] = v[uid] - out_context[k] = new_tests + out_context[k] = {uid: v[uid] for uid in v.keys() if uid in uids_filter} elif k == 'graph': out_context[k] = _strip_graph(v, uids_filter) else: @@ -151,7 +147,7 @@ def process_context(opts, uids_filter: set[str]) -> None: ) parser.add_argument( '--report', '-r', type=str, dest='report', required=True, - help='Path to jsonl report' + help='Path to json build report' ) parser.add_argument('--muted', '-m', type=str, help='Path to muted tests', dest='muted') opts = parser.parse_args() From 4db394ed6c33abf9042cc2919a887661b3697883 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 10:46:36 +0300 Subject: [PATCH 20/26] do not check muted tests on retry --- .github/actions/test_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 2306ac64e29f..86659e1856f6 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -538,7 +538,7 @@ runs: .github/scripts/graph_patch.py \ --in-graph="$GRAPH_PATH" --in-context="$CONTEXT_PATH" \ --out-graph="$GRAPH_PATH" --out-context="$CONTEXT_PATH" \ - --report="$CURRENT_REPORT" --muted=.github/config/muted_ya.txt + --report="$CURRENT_REPORT" fi done; From 5dec8e3cad2da7040b273f79d84bbd6de60b192a Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 11:23:28 +0300 Subject: [PATCH 21/26] fix Copilot issues --- .github/actions/test_ya/action.yml | 6 ++++-- .github/scripts/graph_compare.py | 2 +- .github/scripts/graph_patch.py | 6 +++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 86659e1856f6..94a3493a7e6e 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -352,7 +352,7 @@ runs: mkdir $TEST_META_INFO CURRENT_MESSAGE="ya make is running..." - CACHE_OPTS="" + CACHE_OPTS=() if [ $IS_RETRY = 0 ]; then CURRENT_MESSAGE="$CURRENT_MESSAGE" RERUN_FAILED_OPT="" @@ -370,6 +370,8 @@ runs: CURRENT_MESSAGE="$CURRENT_MESSAGE (failed tests rerun, try $RETRY)" if [ -z "$GRAPH_PATH" ]; then RERUN_FAILED_OPT="-X" + else + RERUN_FAILED_OPT="" fi fi @@ -392,7 +394,7 @@ runs: MONITOR_PID=$! set +e - (./ya make ${params[@]} $CACHE_OPTS $YA_MAKE_TARGET \ + (./ya make ${params[@]} ${CACHE_OPTS[@]} $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index 9bee9ee64240..5e1b3cb5e199 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -43,7 +43,7 @@ def do_compare(opts): exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json') log('Generate diff context...') exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.context_path} {workdir}/uids.json {opts.graph_path}') diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index 55cc7096dc03..dc607e66e0f6 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -23,6 +23,8 @@ def __init__(self, fn): with open(fn, 'r') as fp: for line in fp: line = line.strip() + if not line: + continue pattern = self.__pattern_to_re(line) try: @@ -81,12 +83,14 @@ def visit(uid): result_nodes: list[dict] = [] for uid in result: for node in visit(uid): + if node['uid'] in result: + node['cache'] = False result_nodes.append(node) return result_nodes -def _filter_duplicate_resources(resources): +def _filter_duplicate_resources(resources: list[dict]) -> list[dict]: v = set() result = [] for x in resources: From 2dcca65aeee00ea612b7c6e3a06e2259c9a6ae81 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 11:38:20 +0300 Subject: [PATCH 22/26] fix --- .github/scripts/graph_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index 5e1b3cb5e199..258830979698 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -43,7 +43,7 @@ def do_compare(opts): exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') log('Generate diff context...') exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.context_path} {workdir}/uids.json {opts.graph_path}') From e49e330535455703088f27affdbfa3276a66a6ba Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 14:18:31 +0300 Subject: [PATCH 23/26] fix issues --- .github/scripts/graph_compare.py | 44 +++++++++++++-------- .github/scripts/graph_patch.py | 67 ++++++++++++-------------------- 2 files changed, 52 insertions(+), 59 deletions(-) diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index 258830979698..d3b3c3fb997e 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -21,11 +21,8 @@ def log(msg: str): print(msg) -def do_compare(opts): - if not opts.ya_make_command: - print('--ya-make-command not set') - exit(1) - ya = opts.ya_make_command.split(' ')[0] +def main(ya_make_command: str, graph_path: str, context_path: str, base_commit: str, head_commit: str) -> None: + ya = ya_make_command.split(' ')[0] workdir = os.getenv('workdir') if not workdir: @@ -33,28 +30,43 @@ def do_compare(opts): log(f'Workdir: {workdir}') log('Checkout base commit...') - exec(f'git checkout {opts.base_commit}') + exec(f'git checkout {base_commit}') log('Build graph for base commit...') - exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_base.json --save-context-to {workdir}/context_base.json') + exec(f'{ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_base.json --save-context-to {workdir}/context_base.json') log('Checkout head commit...') - exec(f'git checkout {opts.head_commit}') + exec(f'git checkout {head_commit}') log('Build graph for head commit...') - exec(f'{opts.ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') + exec(f'{ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {opts.graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') log('Generate diff context...') - exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {opts.context_path} {workdir}/uids.json {opts.graph_path}') - exit(0) + exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {context_path} {workdir}/uids.json {opts.graph_path}') if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--result-graph-path', '-g', type=str, help='Path to result graph', dest='graph_path', required=True) - parser.add_argument('--result-context-path', '-c', type=str, help='Path to result context', dest='context_path', required=True) - parser.add_argument('--ya-make-command', '-y', type=str, help='Ya make command', dest='ya_make_command', required=True) + parser.add_argument( + '--result-graph-path', '-g', type=str, dest='graph_path', required=True, + help='Path to result graph' + ) + parser.add_argument( + '--result-context-path', '-c', type=str, dest='context_path', required=True, + help='Path to result context' + ) + parser.add_argument( + '--ya-make-command', '-y', type=str, dest='ya_make_command', required=True, + help='Ya make command' + ) parser.add_argument(dest='base_commit', help='Base commit') parser.add_argument(dest='head_commit', help='Head commit') - do_compare(parser.parse_args()) + opts = parser.parse_args() + main( + ya_make_command=opts.ya_make_command, + graph_path=opts.graph_path, + context_path=opts.context_path, + base_commit=opts.base_commit, + head_commit=opts.head_commit + ) diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index dc607e66e0f6..cbb239d7e8cd 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -3,48 +3,29 @@ from __future__ import annotations import argparse import json -import re +import importlib.util +import sys +import os -class MuteTestCheck: - def __pattern_to_re(self, pattern): - res = [] - for c in pattern: - if c == '*': - res.append('.*') - else: - res.append(re.escape(c)) +def __import_from_path(module_name, file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module - return f"(?:^{''.join(res)}$)" - def __init__(self, fn): - self.regexps = [] +self_path = os.path.dirname(__file__) +sys.path.append(os.path.join(self_path, 'tests')) +mute_utils = __import_from_path('mute_util', os.path.join(self_path, 'tests', 'mute_utils.py')) - with open(fn, 'r') as fp: - for line in fp: - line = line.strip() - if not line: - continue - pattern = self.__pattern_to_re(line) - try: - self.regexps.append(re.compile(pattern)) - except re.error: - print(f"Unable to compile regex {pattern!r}") - raise - - def __call__(self, fullname): - for r in self.regexps: - if r.match(fullname): - return True - return False - - -def get_failed_uids(opts) -> set[str]: +def get_failed_uids(muted_path: str, report_path: str) -> set[str]: print('Load failed uids..') result = set() - mute_check = MuteTestCheck(opts.muted) if opts.muted else None - with open(opts.report) as report_file: + mute_check = mute_utils.MuteTestCheck(muted_path) if muted_path else None + with open(report_path) as report_file: report = json.load(report_file).get('results', []) for record in report: if record.get('status', 'OK') == 'OK' or record.get('suite', False): @@ -100,21 +81,21 @@ def _filter_duplicate_resources(resources: list[dict]) -> list[dict]: return result -def process_graph(opts, uids_filter: set[str]) -> None: +def process_graph(in_graph_path: str, out_graph_path: str, uids_filter: set[str]) -> None: print('Load graph...') - with open(opts.in_graph) as f: + with open(in_graph_path) as f: in_graph = json.load(f) print('Strip graph...') out_graph = _strip_graph(in_graph, uids_filter) print('Save graph...') - with open(opts.out_graph, 'w') as f: + with open(out_graph_path, 'w') as f: json.dump(out_graph, f, indent=2) print('Process graph...OK') -def process_context(opts, uids_filter: set[str]) -> None: +def process_context(in_context_path: str, out_context_path: str, uids_filter: set[str]) -> None: print('Load context...') - with open(opts.in_context) as f: + with open(in_context_path) as f: in_context = json.load(f) out_context = {} print('Strip context...') @@ -126,7 +107,7 @@ def process_context(opts, uids_filter: set[str]) -> None: else: out_context[k] = v print('Save context...') - with open(opts.out_context, 'w') as f: + with open(out_context_path, 'w') as f: json.dump(out_context, f, indent=2) print('Process context...OK') @@ -155,6 +136,6 @@ def process_context(opts, uids_filter: set[str]) -> None: ) parser.add_argument('--muted', '-m', type=str, help='Path to muted tests', dest='muted') opts = parser.parse_args() - uids = get_failed_uids(opts) - process_graph(opts, uids) - process_context(opts, uids) + uids = get_failed_uids(muted_path=opts.muted, report_path=opts.report) + process_graph(in_graph_path=opts.in_graph, out_graph_path=opts.out_graph, uids_filter=uids) + process_context(in_context_path=opts.in_context, out_context_path=opts.out_context, uids_filter=uids) From 2e0fc0e7ed09a57ee5765db4ad7ef3d33474cd01 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 14:22:07 +0300 Subject: [PATCH 24/26] simplify import --- .github/scripts/graph_patch.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index cbb239d7e8cd..06543dd31d0a 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -3,22 +3,11 @@ from __future__ import annotations import argparse import json -import importlib.util import sys import os - -def __import_from_path(module_name, file_path): - spec = importlib.util.spec_from_file_location(module_name, file_path) - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - return module - - -self_path = os.path.dirname(__file__) -sys.path.append(os.path.join(self_path, 'tests')) -mute_utils = __import_from_path('mute_util', os.path.join(self_path, 'tests', 'mute_utils.py')) +sys.path.append(os.path.join(os.path.dirname(__file__), 'tests')) +import mute_utils def get_failed_uids(muted_path: str, report_path: str) -> set[str]: From b84f6b0d72a6f4ad86e616a2fe61e9ff9b0ef9a4 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 19:08:34 +0300 Subject: [PATCH 25/26] add comments, fix naming --- .github/actions/test_ya/action.yml | 8 ++++++-- .github/scripts/graph_compare.py | 8 ++++---- .github/scripts/graph_patch.py | 5 +++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index 94a3493a7e6e..1d21b17781c2 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -537,10 +537,14 @@ runs: break fi if [ -n "$GRAPH_PATH" ] && [ -n "$CONTEXT_PATH" ]; then + NEW_GRAPH_PATH=$(realpath "graph-$RETRY.json") + NEW_CONTEXT_PATH=$(realpath "context-$RETRY.json") .github/scripts/graph_patch.py \ - --in-graph="$GRAPH_PATH" --in-context="$CONTEXT_PATH" \ - --out-graph="$GRAPH_PATH" --out-context="$CONTEXT_PATH" \ + --in-graph "$GRAPH_PATH" --in-context "$CONTEXT_PATH" \ + --out-graph "$NEW_GRAPH_PATH" --out-context "$NEW_GRAPH_PATH" \ --report="$CURRENT_REPORT" + GRAPH_PATH=$NEW_GRAPH_PATH + CONTEXT_PATH=$NEW_CONTEXT_PATH fi done; diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index d3b3c3fb997e..d1db325cac74 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -49,11 +49,11 @@ def main(ya_make_command: str, graph_path: str, context_path: str, base_commit: if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--result-graph-path', '-g', type=str, dest='graph_path', required=True, + '--result-graph-path', '-g', type=str, dest='result_graph_path', required=True, help='Path to result graph' ) parser.add_argument( - '--result-context-path', '-c', type=str, dest='context_path', required=True, + '--result-context-path', '-c', type=str, dest='result_context_path', required=True, help='Path to result context' ) parser.add_argument( @@ -65,8 +65,8 @@ def main(ya_make_command: str, graph_path: str, context_path: str, base_commit: opts = parser.parse_args() main( ya_make_command=opts.ya_make_command, - graph_path=opts.graph_path, - context_path=opts.context_path, + graph_path=opts.result_graph_path, + context_path=opts.result_context_path, base_commit=opts.base_commit, head_commit=opts.head_commit ) diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py index 06543dd31d0a..c54d558da8fe 100755 --- a/.github/scripts/graph_patch.py +++ b/.github/scripts/graph_patch.py @@ -30,6 +30,7 @@ def get_failed_uids(muted_path: str, report_path: str) -> set[str]: return result +# See https://nda.ya.ru/t/pjXsFHg-7Mj5ty def _strip_graph(graph: dict, uids_filter: set[str]) -> dict: result = {uid for uid in graph['result'] if uid in uids_filter} nodes = _strip_unused_nodes(graph['graph'], result) @@ -40,6 +41,7 @@ def _strip_graph(graph: dict, uids_filter: set[str]) -> dict: return {'conf': conf, 'inputs': graph.get('inputs', {}), 'result': [uid for uid in result], 'graph': nodes} +# See https://nda.ya.ru/t/SnKqrM137Mj5kA def _strip_unused_nodes(graph_nodes: list, result: set[str]) -> list[dict]: by_uid = {n['uid']: n for n in graph_nodes} @@ -60,6 +62,7 @@ def visit(uid): return result_nodes +# See https://nda.ya.ru/t/NGk-S38S7Mj5q5 def _filter_duplicate_resources(resources: list[dict]) -> list[dict]: v = set() result = [] @@ -71,6 +74,7 @@ def _filter_duplicate_resources(resources: list[dict]) -> list[dict]: def process_graph(in_graph_path: str, out_graph_path: str, uids_filter: set[str]) -> None: + # This method loads graph from file, strips it using uids_filter, and saves to another file print('Load graph...') with open(in_graph_path) as f: in_graph = json.load(f) @@ -83,6 +87,7 @@ def process_graph(in_graph_path: str, out_graph_path: str, uids_filter: set[str] def process_context(in_context_path: str, out_context_path: str, uids_filter: set[str]) -> None: + # This method loads context from file, strips it using uids_filter, and saves to another file print('Load context...') with open(in_context_path) as f: in_context = json.load(f) From 9cb8c7d7a38a7985b9687ae85b5d017d37ba1c32 Mon Sep 17 00:00:00 2001 From: Oleg Geller Date: Mon, 10 Nov 2025 19:36:57 +0300 Subject: [PATCH 26/26] fix --- .github/scripts/graph_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index d1db325cac74..c72a7ab1ec15 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -43,7 +43,7 @@ def main(ya_make_command: str, graph_path: str, context_path: str, base_commit: exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') log('Generate diff context...') - exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {context_path} {workdir}/uids.json {opts.graph_path}') + exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {context_path} {workdir}/uids.json {graph_path}') if __name__ == '__main__':