From 5057914685d9a5a6c572cab32110e2d417c8428f Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Wed, 9 Jul 2025 14:46:37 +0100 Subject: [PATCH 1/4] Add DebugifyBuilder for debug location coverage testing --- zorg/buildbot/builders/DebugifyBuilder.py | 82 ++++++++++++++++++++++ zorg/buildbot/builders/TestSuiteBuilder.py | 17 +++-- 2 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 zorg/buildbot/builders/DebugifyBuilder.py diff --git a/zorg/buildbot/builders/DebugifyBuilder.py b/zorg/buildbot/builders/DebugifyBuilder.py new file mode 100644 index 000000000..bcb54f981 --- /dev/null +++ b/zorg/buildbot/builders/DebugifyBuilder.py @@ -0,0 +1,82 @@ +from buildbot.plugins import util +from buildbot.steps.shell import ShellCommand +from zorg.buildbot.builders import TestSuiteBuilder +from zorg.buildbot.commands.CmakeCommand import CmakeCommand + +def addCheckDebugifyStep( + f, + debugify_output_path, + compiler_dir = '.', + env = None): + script = util.Interpolate(f'%(prop:builddir)s/{compiler_dir}/llvm/utils/llvm-original-di-preservation.py') + f.add_step(ShellCommand(name='check debugify output', + command=["python3", script, debugify_output_path, "--error-test"], + description='check debugify output', + env=env)) + +def getDebugifyBuildFactory( + depends_on_projects = None, + enable_runtimes = "auto", + targets = None, + llvm_srcdir = None, + obj_dir = None, + checks = None, + install_dir = None, + clean = False, + test_suite_build_flags = '-O2 -g -DNDEBUG', + extra_configure_args = None, + enable_origin_tracking = True, + extra_test_suite_configure_args = None, + env = None, + **kwargs): + + # Make a local copy of the LLVM configure args, as we are going to modify that. + if extra_configure_args is not None: + llvm_cmake_args = extra_configure_args[:] + else: + llvm_cmake_args = list() + + tracking_mode = "COVERAGE_AND_ORIGIN" if enable_origin_tracking else "COVERAGE" + CmakeCommand.applyRequiredOptions(llvm_cmake_args, [ + ('-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=', tracking_mode) + ]) + + debugify_output_path = util.Interpolate(f'%(prop:builddir)s/debugify-report.json') + + # Make a local copy of the test suite configure args, as we are going to modify that. + if extra_test_suite_configure_args is not None: + test_suite_cmake_args = extra_test_suite_configure_args[:] + else: + test_suite_cmake_args = list() + + CmakeCommand.applyDefaultOptions(test_suite_cmake_args, [ + ('-DTEST_SUITE_SUBDIRS=', 'CTMark'), + ('-DTEST_SUITE_RUN_BENCHMARKS=', 'false'), + ('-DTEST_SUITE_COLLECT_CODE_SIZE=', 'false'), + ]) + # The only configuration that currently makes sense for Debugify builds is optimized debug info builds; any build + # configuration adjustments can be made through the test_suite_build_flags arg. + build_flags = f'{test_suite_build_flags} -Xclang -fverify-debuginfo-preserve -Xclang -fverify-debuginfo-preserve-export={debugify_output_path} -mllvm --debugify-quiet -mllvm -debugify-level=locations' + CmakeCommand.applyRequiredOptions(test_suite_cmake_args, [ + ('-DCMAKE_BUILD_TYPE=', 'RelWithDebInfo'), + ('-DCMAKE_C_FLAGS_RELWITHDEBINFO=', build_flags), + ('-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=', build_flags), + ]) + + f = TestSuiteBuilder.getTestSuiteBuildFactory( + depends_on_projects=depends_on_projects, + enable_runtimes=enable_runtimes, + targets=targets, + llvm_srcdir=llvm_srcdir, + obj_dir=obj_dir, + checks=checks, + install_dir=install_dir, + clean=clean, + extra_configure_args=llvm_cmake_args, + extra_test_suite_configure_args=test_suite_cmake_args, + **kwargs + ) + + addCheckDebugifyStep(f, debugify_output_path, compiler_dir=f.monorepo_dir, env=env) + + return f diff --git a/zorg/buildbot/builders/TestSuiteBuilder.py b/zorg/buildbot/builders/TestSuiteBuilder.py index fb9c3bb49..f192b2b04 100644 --- a/zorg/buildbot/builders/TestSuiteBuilder.py +++ b/zorg/buildbot/builders/TestSuiteBuilder.py @@ -16,6 +16,7 @@ def addTestSuiteStep( compiler_dir = '.', env = None, lit_args = None, + extra_configure_args = None, **kwargs): # Set defaults @@ -24,15 +25,19 @@ def addTestSuiteStep( if lit_args is None: lit_args = [] - cc = util.Interpolate('-DCMAKE_C_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang') - cxx = util.Interpolate('-DCMAKE_CXX_COMPILER=' + '%(prop:builddir)s/'+compiler_dir+'/bin/clang++') + cc = ('-DCMAKE_C_COMPILER=', util.Interpolate('%(prop:builddir)s/'+compiler_dir+'/bin/clang')) + cxx = ('-DCMAKE_CXX_COMPILER=', util.Interpolate('%(prop:builddir)s/'+compiler_dir+'/bin/clang++')) lit = util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') test_suite_base_dir = util.Interpolate('%(prop:builddir)s/' + 'test') test_suite_src_dir = util.Interpolate('%(prop:builddir)s/' + 'test/test-suite') test_suite_workdir = util.Interpolate('%(prop:builddir)s/' + 'test/build-test-suite') - cmake_lit_arg = util.Interpolate('-DTEST_SUITE_LIT:FILEPATH=' + '%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') + cmake_lit_arg = ('-DTEST_SUITE_LIT:FILEPATH=', util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit')) # used for cmake building test-suite step - options = [cc, cxx, cmake_lit_arg] + if extra_configure_args is not None: + cmake_args = extra_configure_args[:] + else: + cmake_args = list() + CmakeCommand.applyRequiredOptions(cmake_args, [cc, cxx, cmake_lit_arg]) # always clobber the build directory to test each new compiler f.addStep(ShellCommand(name='Clean Test Suite Build dir', @@ -51,7 +56,7 @@ def addTestSuiteStep( haltOnFailure=True, description='Running cmake on Test Suite dir', workdir=test_suite_workdir, - options=options, + options=cmake_args, path=test_suite_src_dir, generator='Ninja')) @@ -80,6 +85,7 @@ def getTestSuiteBuildFactory( install_dir = None, clean = False, extra_configure_args = None, + extra_test_suite_configure_args = None, env = None, **kwargs): @@ -109,6 +115,7 @@ def getTestSuiteBuildFactory( compiler_dir=f.obj_dir, env=env, lit_args=lit_args, + extra_configure_args=extra_test_suite_configure_args, **kwargs) return f From 91e4caf5d1de39189e82c1b0428c2bfc3d04a1fd Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Mon, 14 Jul 2025 15:38:20 +0100 Subject: [PATCH 2/4] Don't pass Interpolate to applyRequiredOptions, fix addStep typo --- zorg/buildbot/builders/DebugifyBuilder.py | 8 ++++---- zorg/buildbot/builders/TestSuiteBuilder.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/zorg/buildbot/builders/DebugifyBuilder.py b/zorg/buildbot/builders/DebugifyBuilder.py index bcb54f981..57c4dd27c 100644 --- a/zorg/buildbot/builders/DebugifyBuilder.py +++ b/zorg/buildbot/builders/DebugifyBuilder.py @@ -9,10 +9,10 @@ def addCheckDebugifyStep( compiler_dir = '.', env = None): script = util.Interpolate(f'%(prop:builddir)s/{compiler_dir}/llvm/utils/llvm-original-di-preservation.py') - f.add_step(ShellCommand(name='check debugify output', - command=["python3", script, debugify_output_path, "--error-test"], - description='check debugify output', - env=env)) + f.addStep(ShellCommand(name='check debugify output', + command=["python3", script, debugify_output_path, "--error-test"], + description='check debugify output', + env=env)) def getDebugifyBuildFactory( depends_on_projects = None, diff --git a/zorg/buildbot/builders/TestSuiteBuilder.py b/zorg/buildbot/builders/TestSuiteBuilder.py index f192b2b04..719315b48 100644 --- a/zorg/buildbot/builders/TestSuiteBuilder.py +++ b/zorg/buildbot/builders/TestSuiteBuilder.py @@ -25,19 +25,19 @@ def addTestSuiteStep( if lit_args is None: lit_args = [] - cc = ('-DCMAKE_C_COMPILER=', util.Interpolate('%(prop:builddir)s/'+compiler_dir+'/bin/clang')) - cxx = ('-DCMAKE_CXX_COMPILER=', util.Interpolate('%(prop:builddir)s/'+compiler_dir+'/bin/clang++')) + cc = util.Interpolate('-DCMAKE_C_COMPILER=%(prop:builddir)s/'+compiler_dir+'/bin/clang') + cxx = util.Interpolate('-DCMAKE_CXX_COMPILER=%(prop:builddir)s/'+compiler_dir+'/bin/clang++') lit = util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') test_suite_base_dir = util.Interpolate('%(prop:builddir)s/' + 'test') test_suite_src_dir = util.Interpolate('%(prop:builddir)s/' + 'test/test-suite') test_suite_workdir = util.Interpolate('%(prop:builddir)s/' + 'test/build-test-suite') - cmake_lit_arg = ('-DTEST_SUITE_LIT:FILEPATH=', util.Interpolate('%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit')) + cmake_lit_arg = util.Interpolate('-DTEST_SUITE_LIT:FILEPATH=%(prop:builddir)s/' + compiler_dir + '/bin/llvm-lit') # used for cmake building test-suite step if extra_configure_args is not None: cmake_args = extra_configure_args[:] else: cmake_args = list() - CmakeCommand.applyRequiredOptions(cmake_args, [cc, cxx, cmake_lit_arg]) + cmake_args.extend([cc, cxx, cmake_lit_arg]) # always clobber the build directory to test each new compiler f.addStep(ShellCommand(name='Clean Test Suite Build dir', From 69cd3bd6c90c20f901bd0951b2fc4e6859499c5e Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Fri, 18 Jul 2025 10:24:44 +0100 Subject: [PATCH 3/4] Apply darker, update debugify-script argument name --- zorg/buildbot/builders/DebugifyBuilder.py | 24 +++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/zorg/buildbot/builders/DebugifyBuilder.py b/zorg/buildbot/builders/DebugifyBuilder.py index 57c4dd27c..66df0f34f 100644 --- a/zorg/buildbot/builders/DebugifyBuilder.py +++ b/zorg/buildbot/builders/DebugifyBuilder.py @@ -3,16 +3,20 @@ from zorg.buildbot.builders import TestSuiteBuilder from zorg.buildbot.commands.CmakeCommand import CmakeCommand -def addCheckDebugifyStep( - f, - debugify_output_path, - compiler_dir = '.', - env = None): - script = util.Interpolate(f'%(prop:builddir)s/{compiler_dir}/llvm/utils/llvm-original-di-preservation.py') - f.addStep(ShellCommand(name='check debugify output', - command=["python3", script, debugify_output_path, "--error-test"], - description='check debugify output', - env=env)) + +def addCheckDebugifyStep(f, debugify_output_path, compiler_dir=".", env=None): + script = util.Interpolate( + f"%(prop:builddir)s/{compiler_dir}/llvm/utils/llvm-original-di-preservation.py" + ) + f.addStep( + ShellCommand( + name="check debugify output", + command=["python3", script, debugify_output_path, "--acceptance-test"], + description="check debugify output", + env=env, + ) + ) + def getDebugifyBuildFactory( depends_on_projects = None, From f3cef5b8917f186a16e54109d6ce24f8542156d8 Mon Sep 17 00:00:00 2001 From: Stephen Tozer Date: Fri, 18 Jul 2025 16:44:55 +0100 Subject: [PATCH 4/4] Fixup some errors relating to passing args, apply darker --- zorg/buildbot/builders/DebugifyBuilder.py | 47 ++++++++++++++--------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/zorg/buildbot/builders/DebugifyBuilder.py b/zorg/buildbot/builders/DebugifyBuilder.py index 66df0f34f..2d94856f1 100644 --- a/zorg/buildbot/builders/DebugifyBuilder.py +++ b/zorg/buildbot/builders/DebugifyBuilder.py @@ -4,14 +4,19 @@ from zorg.buildbot.commands.CmakeCommand import CmakeCommand -def addCheckDebugifyStep(f, debugify_output_path, compiler_dir=".", env=None): +def addCheckDebugifyStep(f, debugify_output_path, compiler_dir=".", env={}): script = util.Interpolate( f"%(prop:builddir)s/{compiler_dir}/llvm/utils/llvm-original-di-preservation.py" ) f.addStep( ShellCommand( name="check debugify output", - command=["python3", script, debugify_output_path, "--acceptance-test"], + command=[ + "python3", + script, + util.Interpolate(debugify_output_path), + "--acceptance-test", + ], description="check debugify output", env=env, ) @@ -19,20 +24,21 @@ def addCheckDebugifyStep(f, debugify_output_path, compiler_dir=".", env=None): def getDebugifyBuildFactory( - depends_on_projects = None, - enable_runtimes = "auto", - targets = None, - llvm_srcdir = None, - obj_dir = None, - checks = None, - install_dir = None, - clean = False, - test_suite_build_flags = '-O2 -g -DNDEBUG', - extra_configure_args = None, - enable_origin_tracking = True, - extra_test_suite_configure_args = None, - env = None, - **kwargs): + depends_on_projects=None, + enable_runtimes="auto", + targets=None, + llvm_srcdir=None, + obj_dir=None, + checks=None, + install_dir=None, + clean=False, + test_suite_build_flags="-O2 -g -DNDEBUG", + extra_configure_args=None, + enable_origin_tracking=True, + extra_test_suite_configure_args=None, + env={}, + **kwargs, +): # Make a local copy of the LLVM configure args, as we are going to modify that. if extra_configure_args is not None: @@ -45,7 +51,8 @@ def getDebugifyBuildFactory( ('-DLLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING=', tracking_mode) ]) - debugify_output_path = util.Interpolate(f'%(prop:builddir)s/debugify-report.json') + # This path will be passed through to util.Interpolate, so we leave it in this format. + debugify_output_path = f"%(prop:builddir)s/debugify-report.json" # Make a local copy of the test suite configure args, as we are going to modify that. if extra_test_suite_configure_args is not None: @@ -63,9 +70,11 @@ def getDebugifyBuildFactory( build_flags = f'{test_suite_build_flags} -Xclang -fverify-debuginfo-preserve -Xclang -fverify-debuginfo-preserve-export={debugify_output_path} -mllvm --debugify-quiet -mllvm -debugify-level=locations' CmakeCommand.applyRequiredOptions(test_suite_cmake_args, [ ('-DCMAKE_BUILD_TYPE=', 'RelWithDebInfo'), - ('-DCMAKE_C_FLAGS_RELWITHDEBINFO=', build_flags), - ('-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=', build_flags), ]) + test_suite_cmake_args += [ + util.Interpolate(f"-DCMAKE_C_FLAGS_RELWITHDEBINFO={build_flags}"), + util.Interpolate(f"-DCMAKE_CXX_FLAGS_RELWITHDEBINFO={build_flags}"), + ] f = TestSuiteBuilder.getTestSuiteBuildFactory( depends_on_projects=depends_on_projects,