From e415409ca0832edf3f12f5c20d303d821787be23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Oct 2025 03:10:11 +0300 Subject: [PATCH 1/6] Implement automatic test suite skipping facility into the browser test harness, so that users do not need to manually maintain skip lists of features. Paves the way toward test harness working for users out of the box. --- test/test_browser.py | 36 +++++++++++++++++++++++++++++------- tools/feature_matrix.py | 23 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index a0ca3839bab5c..201b5c1ab4d87 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -28,7 +28,7 @@ from common import HttpServerThread, requires_dev_dependency from tools import shared from tools import ports -from tools.feature_matrix import UNSUPPORTED +from tools.feature_matrix import UNSUPPORTED, min_browser_versions, Feature from tools.shared import EMCC, WINDOWS, FILE_PACKAGER, PIPE, DEBUG from tools.utils import delete_dir, memoize @@ -158,6 +158,23 @@ def get_safari_version(): return parts[0] * 10000 + parts[1] * 100 + parts[2] +@memoize +def get_firefox_version(): + if not is_firefox(): + return UNSUPPORTED + exe_path = shlex.split(common.EMTEST_BROWSER)[0] + ini_path = os.path.join(os.path.dirname(exe_path), "platform.ini") + # Extract the first numeric part before any dot (e.g. "Milestone=102.15.1" → 102) + m = re.search(r"^Milestone=([^\n\r]+)", open(ini_path).read(), re.MULTILINE) + milestone = m.group(1).strip() + version = int(re.match(r"(\d+)", milestone).group(1)) + # On Nightly and BEta, e.g. 145.0a1, pretend it to still mean version 144, + # since it is a pre-release version + if any(c in milestone for c in ("a", "b")): + version -= 1 + return version + + no_swiftshader = skip_if_simple('not compatible with swiftshader', is_swiftshader) no_chrome = skip_if('no_chrome', lambda _: is_chrome(), 'chrome is not supported') @@ -214,11 +231,16 @@ def decorated(self, *args, **kwargs): def webgl2_disabled(): - return os.getenv('EMTEST_LACKS_WEBGL2') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') + return os.getenv('EMTEST_LACKS_WEBGL2') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') or current_browser_lacks_feature(Feature.WEBGL2) def webgpu_disabled(): - return os.getenv('EMTEST_LACKS_WEBGPU') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') + return os.getenv('EMTEST_LACKS_WEBGPU') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') or current_browser_lacks_feature(Feature.WEBGPU) + + +def current_browser_lacks_feature(feature): + min_required = min_browser_versions[feature] + return get_firefox_version() < min_required['firefox'] or get_safari_version() < min_required['safari'] requires_graphics_hardware = skipExecIf(os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE'), 'This test requires graphics hardware') @@ -226,13 +248,13 @@ def webgpu_disabled(): requires_webgpu = unittest.skipIf(webgpu_disabled(), "This test requires WebGPU to be available") requires_sound_hardware = skipExecIf(os.getenv('EMTEST_LACKS_SOUND_HARDWARE'), 'This test requires sound hardware') requires_microphone_access = skipExecIf(os.getenv('EMTEST_LACKS_MICROPHONE_ACCESS'), 'This test accesses microphone, which may need accepting a user prompt to enable it.') -requires_offscreen_canvas = unittest.skipIf(os.getenv('EMTEST_LACKS_OFFSCREEN_CANVAS'), 'This test requires a browser with OffscreenCanvas') -requires_es6_workers = unittest.skipIf(os.getenv('EMTEST_LACKS_ES6_WORKERS'), 'This test requires a browser with ES6 Module Workers support') -requires_growable_arraybuffers = unittest.skipIf(os.getenv('EMTEST_LACKS_GROWABLE_ARRAYBUFFERS'), 'This test requires a browser that supports growable ArrayBuffers') +requires_offscreen_canvas = unittest.skipIf(os.getenv('EMTEST_LACKS_OFFSCREEN_CANVAS') or current_browser_lacks_feature(Feature.OFFSCREENCANVAS_SUPPORT), 'This test requires a browser with OffscreenCanvas') +requires_es6_workers = unittest.skipIf(os.getenv('EMTEST_LACKS_ES6_WORKERS') or current_browser_lacks_feature(Feature.WORKER_ES6_MODULES), 'This test requires a browser with ES6 Module Workers support') +requires_growable_arraybuffers = unittest.skipIf(os.getenv('EMTEST_LACKS_GROWABLE_ARRAYBUFFERS') or current_browser_lacks_feature(Feature.GROWABLE_ARRAYBUFFERS), 'This test requires a browser that supports growable ArrayBuffers') # N.b. not all SharedArrayBuffer requiring tests are annotated with this decorator, since at this point there are so many of such tests. # As a middle ground, if a test has a name 'thread' or 'wasm_worker' in it, then it does not need decorating. To run all single-threaded tests in # the suite, one can run "EMTEST_LACKS_SHARED_ARRAY_BUFFER=1 test/runner browser skip:browser.test_*thread* skip:browser.test_*wasm_worker* skip:browser.test_*audio_worklet*" -requires_shared_array_buffer = unittest.skipIf(os.getenv('EMTEST_LACKS_SHARED_ARRAY_BUFFER'), 'This test requires a browser with SharedArrayBuffer support') +requires_shared_array_buffer = unittest.skipIf(os.getenv('EMTEST_LACKS_SHARED_ARRAY_BUFFER') or current_browser_lacks_feature(Feature.THREADS), 'This test requires a browser with SharedArrayBuffer support') class browser(BrowserCore): diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index 56a8f30b5c736..fbee9d02153f1 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -43,6 +43,9 @@ class Feature(IntEnum): OFFSCREENCANVAS_SUPPORT = auto() WASM_LEGACY_EXCEPTIONS = auto() WASM_EXCEPTIONS = auto() + WEBGL2 = auto() + WEBGPU = auto() + GROWABLE_ARRAYBUFFERS = auto() disable_override_features = set() @@ -97,6 +100,18 @@ class Feature(IntEnum): 'safari': UNSUPPORTED, 'node': 230000, }, + Feature.WEBGL2: { + 'chrome': 56, + 'firefox': 51, + 'safari': 150000, + 'node': UNSUPPORTED, + }, + Feature.WEBGPU: { + 'chrome': 113, + 'firefox': 141, + 'safari': 260000, + 'node': UNSUPPORTED, + }, # https://caniuse.com/mdn-api_worker_worker_ecmascript_modules: The ability to # call new Worker(url, { type: 'module' }); Feature.WORKER_ES6_MODULES: { @@ -134,6 +149,14 @@ class Feature(IntEnum): # Node.js 26) 'node': 240000, }, + # Growable SharedArrayBuffers improves memory growth feature in multithreaded + # builds by avoiding need to poll resizes to ArrayBuffer views in Workers. + Feature.GROWABLE_ARRAYBUFFERS: { + 'chrome': 111, + 'firefox': 128, + 'safari': 160400, + 'node': 200000, + } } # Static assertion to check that we actually need each of the above feature flags From 3e76ea0f574c1be800f1da4f4719b09c0929d3fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Oct 2025 03:14:32 +0300 Subject: [PATCH 2/6] ruff --- tools/feature_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index fbee9d02153f1..7ed8199c316b2 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -156,7 +156,7 @@ class Feature(IntEnum): 'firefox': 128, 'safari': 160400, 'node': 200000, - } + }, } # Static assertion to check that we actually need each of the above feature flags From c4c1e01a0a4d267f266962f469c52d70fea81c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 9 Oct 2025 13:26:31 +0300 Subject: [PATCH 3/6] Refactor the test skip auto-detect feature so that env. var. values EMTEST_LACKS_x=0 can be used to force-don't-skip tests. --- test/test_browser.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 201b5c1ab4d87..efe59a7d08759 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -230,17 +230,19 @@ def decorated(self, *args, **kwargs): return decorator -def webgl2_disabled(): - return os.getenv('EMTEST_LACKS_WEBGL2') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') or current_browser_lacks_feature(Feature.WEBGL2) +def test_browser_should_skip_feature(skip_env_var, feature): + if os.getenv(skip_env_var) is None: + min_required = min_browser_versions[feature] + return get_firefox_version() < min_required['firefox'] or get_safari_version() < min_required['safari'] + return int(os.getenv(skip_env_var)) != 0 -def webgpu_disabled(): - return os.getenv('EMTEST_LACKS_WEBGPU') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') or current_browser_lacks_feature(Feature.WEBGPU) +def webgl2_disabled(): + return os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') or test_browser_should_skip_feature('EMTEST_LACKS_WEBGL2', Feature.WEBGL2) -def current_browser_lacks_feature(feature): - min_required = min_browser_versions[feature] - return get_firefox_version() < min_required['firefox'] or get_safari_version() < min_required['safari'] +def webgpu_disabled(): + return os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE') or test_browser_should_skip_feature('EMTEST_LACKS_WEBGPU', Feature.WEBGPU) requires_graphics_hardware = skipExecIf(os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE'), 'This test requires graphics hardware') @@ -248,13 +250,13 @@ def current_browser_lacks_feature(feature): requires_webgpu = unittest.skipIf(webgpu_disabled(), "This test requires WebGPU to be available") requires_sound_hardware = skipExecIf(os.getenv('EMTEST_LACKS_SOUND_HARDWARE'), 'This test requires sound hardware') requires_microphone_access = skipExecIf(os.getenv('EMTEST_LACKS_MICROPHONE_ACCESS'), 'This test accesses microphone, which may need accepting a user prompt to enable it.') -requires_offscreen_canvas = unittest.skipIf(os.getenv('EMTEST_LACKS_OFFSCREEN_CANVAS') or current_browser_lacks_feature(Feature.OFFSCREENCANVAS_SUPPORT), 'This test requires a browser with OffscreenCanvas') -requires_es6_workers = unittest.skipIf(os.getenv('EMTEST_LACKS_ES6_WORKERS') or current_browser_lacks_feature(Feature.WORKER_ES6_MODULES), 'This test requires a browser with ES6 Module Workers support') -requires_growable_arraybuffers = unittest.skipIf(os.getenv('EMTEST_LACKS_GROWABLE_ARRAYBUFFERS') or current_browser_lacks_feature(Feature.GROWABLE_ARRAYBUFFERS), 'This test requires a browser that supports growable ArrayBuffers') +requires_offscreen_canvas = unittest.skipIf(test_browser_should_skip_feature('EMTEST_LACKS_OFFSCREEN_CANVAS', Feature.OFFSCREENCANVAS_SUPPORT), 'This test requires a browser with OffscreenCanvas') +requires_es6_workers = unittest.skipIf(test_browser_should_skip_feature('EMTEST_LACKS_ES6_WORKERS', Feature.WORKER_ES6_MODULES), 'This test requires a browser with ES6 Module Workers support') +requires_growable_arraybuffers = unittest.skipIf(test_browser_should_skip_feature('EMTEST_LACKS_GROWABLE_ARRAYBUFFERS', Feature.GROWABLE_ARRAYBUFFERS), 'This test requires a browser that supports growable ArrayBuffers') # N.b. not all SharedArrayBuffer requiring tests are annotated with this decorator, since at this point there are so many of such tests. # As a middle ground, if a test has a name 'thread' or 'wasm_worker' in it, then it does not need decorating. To run all single-threaded tests in # the suite, one can run "EMTEST_LACKS_SHARED_ARRAY_BUFFER=1 test/runner browser skip:browser.test_*thread* skip:browser.test_*wasm_worker* skip:browser.test_*audio_worklet*" -requires_shared_array_buffer = unittest.skipIf(os.getenv('EMTEST_LACKS_SHARED_ARRAY_BUFFER') or current_browser_lacks_feature(Feature.THREADS), 'This test requires a browser with SharedArrayBuffer support') +requires_shared_array_buffer = unittest.skipIf(test_browser_should_skip_feature('EMTEST_LACKS_SHARED_ARRAY_BUFFER', Feature.THREADS), 'This test requires a browser with SharedArrayBuffer support') class browser(BrowserCore): From 14abd61776fa7b7c1c2038b095ba62e97b24b133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 21:38:52 +0300 Subject: [PATCH 4/6] Add EMTEST_AUTOSKIP --- test/test_browser.py | 17 +++++++++++++---- tools/feature_matrix.py | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index efe59a7d08759..3f1bbe28b5ecb 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -231,10 +231,19 @@ def decorated(self, *args, **kwargs): def test_browser_should_skip_feature(skip_env_var, feature): - if os.getenv(skip_env_var) is None: - min_required = min_browser_versions[feature] - return get_firefox_version() < min_required['firefox'] or get_safari_version() < min_required['safari'] - return int(os.getenv(skip_env_var)) != 0 + if os.getenv(skip_env_var) is not None: + return int(os.getenv(skip_env_var)) != 0 + + min_required = min_browser_versions[feature] + not_supported = get_firefox_version() < min_required['firefox'] or get_safari_version() < min_required['safari'] + + if not_supported and int(os.getenv('EMTEST_AUTOSKIP', '0')): + return True + +# if not_supported and os.getenv('EMTEST_AUTOSKIP') is None: +# TODO: failTest(f'This test requires a browser that supports {feature} but your browser does not support this. Run with {skip_env_var}=1 or EMTEST_AUTOSKIP=1 to skip this test automatically.') + + return False def webgl2_disabled(): diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index 7ed8199c316b2..3c90832ac8dac 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -153,9 +153,9 @@ class Feature(IntEnum): # builds by avoiding need to poll resizes to ArrayBuffer views in Workers. Feature.GROWABLE_ARRAYBUFFERS: { 'chrome': 111, - 'firefox': 128, + 'firefox': 145, 'safari': 160400, - 'node': 200000, + 'node': 260000, }, } From 0afeb4cfe3f1326d079402cf236d34e2bba28386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Thu, 16 Oct 2025 21:40:42 +0300 Subject: [PATCH 5/6] Revise targets --- tools/feature_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index 3c90832ac8dac..ca83c6da2b20c 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -154,8 +154,8 @@ class Feature(IntEnum): Feature.GROWABLE_ARRAYBUFFERS: { 'chrome': 111, 'firefox': 145, - 'safari': 160400, - 'node': 260000, + 'safari': UNSUPPORTED, + 'node': 240000, }, } From 1c5aa1546ef0d2ecbfa3665719da964db123936d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 17 Oct 2025 18:54:29 +0300 Subject: [PATCH 6/6] Bump min chrome version on growable arraybuffers --- tools/feature_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index ca83c6da2b20c..025664b75413f 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -152,7 +152,7 @@ class Feature(IntEnum): # Growable SharedArrayBuffers improves memory growth feature in multithreaded # builds by avoiding need to poll resizes to ArrayBuffer views in Workers. Feature.GROWABLE_ARRAYBUFFERS: { - 'chrome': 111, + 'chrome': 136, 'firefox': 145, 'safari': UNSUPPORTED, 'node': 240000,