diff --git a/.github/actions/test_ya/action.yml b/.github/actions/test_ya/action.yml index c32c488bde40..94a3493a7e6e 100644 --- a/.github/actions/test_ya/action.yml +++ b/.github/actions/test_ya/action.yml @@ -254,13 +254,18 @@ runs: echo "::debug::get version" ./ya --version - export YA_MAKE_COMMAND="./ya make ${params[@]}" if [ "${{ inputs.increment }}" = "true" ]; then GRAPH_COMPARE_OUTPUT="$PUBLIC_DIR/graph_compare_log.txt" GRAPH_COMPARE_OUTPUT_URL="$PUBLIC_DIR_URL/graph_compare_log.txt" set +e - ./.github/scripts/graph_compare.py $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT + YA_MAKE_COMMAND="./ya make ${params[@]}" + if [ true = ${{ inputs.run_tests }} ]; then + YA_MAKE_COMMAND="$YA_MAKE_COMMAND -A" + fi + GRAPH_PATH=$(realpath graph.json) + CONTEXT_PATH=$(realpath context.json) + ./.github/scripts/graph_compare.py --ya-make-command="$YA_MAKE_COMMAND" --result-graph-path=$GRAPH_PATH --result-context-path=$CONTEXT_PATH $ORIGINAL_HEAD~1 $ORIGINAL_HEAD |& tee $GRAPH_COMPARE_OUTPUT RC=${PIPESTATUS[0]} set -e @@ -273,7 +278,11 @@ runs: fi git checkout $ORIGINAL_HEAD - YA_MAKE_TARGET=. + params+=( + --build-custom-json=$GRAPH_PATH + --custom-context=$CONTEXT_PATH + ) + YA_MAKE_TARGET="ydb" else YA_MAKE_TARGET="" for TARGET in ${{ inputs.build_target }}; do @@ -313,24 +322,11 @@ runs: # Check YDB configuration (after checkout from main) python3 .github/scripts/check_ydb_qa_config.py || true - if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then - params+=(--bazel-remote-store) - params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") - fi - - if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then - params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") - params+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") - params+=(--bazel-remote-put --dist-cache-max-file-size=209715200) - fi - if [ true = ${{ inputs.run_tests }} ]; then params+=(-A) params+=(--retest) fi - export YA_MAKE_COMMAND="./ya make ${params[@]}" - YA_MAKE_OUT_DIR=$TMP_DIR/out YA_MAKE_OUTPUT="$PUBLIC_DIR/ya_make_output.txt" @@ -356,12 +352,27 @@ runs: mkdir $TEST_META_INFO CURRENT_MESSAGE="ya make is running..." + CACHE_OPTS=() if [ $IS_RETRY = 0 ]; then CURRENT_MESSAGE="$CURRENT_MESSAGE" RERUN_FAILED_OPT="" + if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then + CACHE_OPTS+=(--bazel-remote-store) + CACHE_OPTS+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}") + fi + + if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then + CACHE_OPTS+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}") + CACHE_OPTS+=(--bazel-remote-password-file "$BAZEL_REMOTE_PASSWORD_FILE") + CACHE_OPTS+=(--bazel-remote-put --dist-cache-max-file-size=209715200) + fi else CURRENT_MESSAGE="$CURRENT_MESSAGE (failed tests rerun, try $RETRY)" - RERUN_FAILED_OPT="-X" + if [ -z "$GRAPH_PATH" ]; then + RERUN_FAILED_OPT="-X" + else + RERUN_FAILED_OPT="" + fi fi echo $CURRENT_MESSAGE | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py @@ -383,7 +394,7 @@ runs: MONITOR_PID=$! set +e - ($YA_MAKE_COMMAND $YA_MAKE_TARGET \ + (./ya make ${params[@]} ${CACHE_OPTS[@]} $YA_MAKE_TARGET \ $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.txt" \ --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \ --junit "$CURRENT_JUNIT_XML_PATH" --build-results-report "$CURRENT_REPORT" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT @@ -525,6 +536,12 @@ runs: if [ $IS_LAST_RETRY = 1 ]; then break fi + if [ -n "$GRAPH_PATH" ] && [ -n "$CONTEXT_PATH" ]; then + .github/scripts/graph_patch.py \ + --in-graph="$GRAPH_PATH" --in-context="$CONTEXT_PATH" \ + --out-graph="$GRAPH_PATH" --out-context="$CONTEXT_PATH" \ + --report="$CURRENT_REPORT" + fi done; if [ $BUILD_FAILED = 0 ]; then diff --git a/.github/scripts/graph_compare.py b/.github/scripts/graph_compare.py index e6bbe9940073..d3b3c3fb997e 100755 --- a/.github/scripts/graph_compare.py +++ b/.github/scripts/graph_compare.py @@ -6,8 +6,7 @@ import os import tempfile -import sys -import json +import argparse def exec(command: str): @@ -22,17 +21,8 @@ def log(msg: str): print(msg) -def do_compare(): - if len(sys.argv) < 3: - print('base or head commit not set') - exit(1) - base_commit = sys.argv[1] - head_commit = sys.argv[2] - - ya_make_command = os.getenv('YA_MAKE_COMMAND') - if not ya_make_command: - print('YA_MAKE_COMMAND not set') - exit(1) +def main(ya_make_command: str, graph_path: str, context_path: str, base_commit: str, head_commit: str) -> None: + ya = ya_make_command.split(' ')[0] workdir = os.getenv('workdir') if not workdir: @@ -42,57 +32,41 @@ def do_compare(): log('Checkout base commit...') exec(f'git checkout {base_commit}') log('Build graph for base commit...') - exec(f'{ya_make_command} ydb -k -A --cache-tests -Gj0 > {workdir}/graph_base.json') + exec(f'{ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_base.json --save-context-to {workdir}/context_base.json') log('Checkout head commit...') exec(f'git checkout {head_commit}') log('Build graph for head commit...') - exec(f'{ya_make_command} ydb -k -A --cache-tests -Gj0 > {workdir}/graph_head.json') + exec(f'{ya_make_command} ydb -k --cache-tests --save-graph-to {workdir}/graph_head.json --save-context-to {workdir}/context_head.json') log('Generate diff graph...') - exec(f'./ya tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {workdir}/graph_diff.json --dump-uids-for-affected-nodes {workdir}/affected_uids.json') - - log('Read diff graph...') - with open(f'{workdir}/graph_diff.json', 'r') as f: - diff_graph = json.load(f) - - with open(f'{workdir}/affected_uids.json', 'r') as f: - uids = set(json.load(f)) - - tests = set() - modules = set() - - log('Scan diff graph...') - for target in diff_graph.get('graph', []): - if target.get('uid') not in uids: - continue - if target.get('node-type') == 'test': - path = target.get('kv', {}).get('path') - if path is not None: - tests.add(os.path.dirname(path)) - tp = target.get('target_properties') - if ( - tp is not None - and tp.get('module_type') is not None - and tp.get('module_dir', '').startswith('ydb') - and tp.get('module_tag', '').find('proto') < 0 - ): - modules.add(tp.get('module_dir')) - - log('Create ya.make') + exec(f'{ya} tool ygdiff --old {workdir}/graph_base.json --new {workdir}/graph_head.json --cut {graph_path} --dump-uids {workdir}/uids.json --no-cache-for-affected-nodes') - with open('ya.make', 'w') as ya_make: - ya_make.write('RECURSE_FOR_TESTS(\n') - for test in sorted(tests): - ya_make.write(f' {test}\n') - ya_make.write(')\n\nRECURSE (\n') - for module in sorted(modules): - ya_make.write(f' {module}\n') - ya_make.write(')\n') - log('ya.make content:') - exec('cat ya.make') - exit(0) + log('Generate diff context...') + exec(f'{ya} tool context_difference {workdir}/context_base.json {workdir}/context_head.json {context_path} {workdir}/uids.json {opts.graph_path}') if __name__ == '__main__': - do_compare() + parser = argparse.ArgumentParser() + parser.add_argument( + '--result-graph-path', '-g', type=str, dest='graph_path', required=True, + help='Path to result graph' + ) + parser.add_argument( + '--result-context-path', '-c', type=str, dest='context_path', required=True, + help='Path to result context' + ) + parser.add_argument( + '--ya-make-command', '-y', type=str, dest='ya_make_command', required=True, + help='Ya make command' + ) + parser.add_argument(dest='base_commit', help='Base commit') + parser.add_argument(dest='head_commit', help='Head commit') + opts = parser.parse_args() + main( + ya_make_command=opts.ya_make_command, + graph_path=opts.graph_path, + context_path=opts.context_path, + base_commit=opts.base_commit, + head_commit=opts.head_commit + ) diff --git a/.github/scripts/graph_patch.py b/.github/scripts/graph_patch.py new file mode 100755 index 000000000000..06543dd31d0a --- /dev/null +++ b/.github/scripts/graph_patch.py @@ -0,0 +1,130 @@ +#! /usr/bin/python3 -u + +from __future__ import annotations +import argparse +import json +import sys +import os + +sys.path.append(os.path.join(os.path.dirname(__file__), 'tests')) +import mute_utils + + +def get_failed_uids(muted_path: str, report_path: str) -> set[str]: + print('Load failed uids..') + result = set() + mute_check = mute_utils.MuteTestCheck(muted_path) if muted_path else None + with open(report_path) as report_file: + report = json.load(report_file).get('results', []) + for record in report: + if record.get('status', 'OK') == 'OK' or record.get('suite', False): + continue + if mute_check is not None: + test_name = f'{record.get("path", "")} {record.get("name", "")}.{record.get("subtest_name", "")}' + if mute_check(test_name): + continue + uid = record.get('uid') + if uid: + result.add(uid) + print(f'{len(result)} uids loaded') + return result + + +def _strip_graph(graph: dict, uids_filter: set[str]) -> dict: + result = {uid for uid in graph['result'] if uid in uids_filter} + nodes = _strip_unused_nodes(graph['graph'], result) + + conf = graph.get('conf', {}).copy() + conf['resources'] = _filter_duplicate_resources(conf.get('resources', [])) + + return {'conf': conf, 'inputs': graph.get('inputs', {}), 'result': [uid for uid in result], 'graph': nodes} + + +def _strip_unused_nodes(graph_nodes: list, result: set[str]) -> list[dict]: + by_uid = {n['uid']: n for n in graph_nodes} + + def visit(uid): + if uid in by_uid: + node = by_uid.pop(uid) + yield node + for dep in node['deps']: + yield from visit(dep) + + result_nodes: list[dict] = [] + for uid in result: + for node in visit(uid): + if node['uid'] in result: + node['cache'] = False + result_nodes.append(node) + + return result_nodes + + +def _filter_duplicate_resources(resources: list[dict]) -> list[dict]: + v = set() + result = [] + for x in resources: + if x['pattern'] not in v: + v.add(x['pattern']) + result.append(x) + return result + + +def process_graph(in_graph_path: str, out_graph_path: str, uids_filter: set[str]) -> None: + print('Load graph...') + with open(in_graph_path) as f: + in_graph = json.load(f) + print('Strip graph...') + out_graph = _strip_graph(in_graph, uids_filter) + print('Save graph...') + with open(out_graph_path, 'w') as f: + json.dump(out_graph, f, indent=2) + print('Process graph...OK') + + +def process_context(in_context_path: str, out_context_path: str, uids_filter: set[str]) -> None: + print('Load context...') + with open(in_context_path) as f: + in_context = json.load(f) + out_context = {} + print('Strip context...') + for k, v in in_context.items(): + if k == 'tests': + out_context[k] = {uid: v[uid] for uid in v.keys() if uid in uids_filter} + elif k == 'graph': + out_context[k] = _strip_graph(v, uids_filter) + else: + out_context[k] = v + print('Save context...') + with open(out_context_path, 'w') as f: + json.dump(out_context, f, indent=2) + print('Process context...OK') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--in-graph', '-G', type=str, dest='in_graph', required=True, + help='Path to input graph' + ) + parser.add_argument( + '--in-context', '-C', type=str, dest='in_context', required=True, + help='Path to input context' + ) + parser.add_argument( + '--out-graph', '-g', type=str, dest='out_graph', required=True, + help='Path to result graph' + ) + parser.add_argument( + '--out-context', '-c', type=str, dest='out_context', required=True, + help='Path to result context' + ) + parser.add_argument( + '--report', '-r', type=str, dest='report', required=True, + help='Path to json build report' + ) + parser.add_argument('--muted', '-m', type=str, help='Path to muted tests', dest='muted') + opts = parser.parse_args() + uids = get_failed_uids(muted_path=opts.muted, report_path=opts.report) + process_graph(in_graph_path=opts.in_graph, out_graph_path=opts.out_graph, uids_filter=uids) + process_context(in_context_path=opts.in_context, out_context_path=opts.out_context, uids_filter=uids)