Implement automatic test suite skipping facility into the browser test harness (#25533)

Implement automatic test suite skipping facility into the browser test
harness, so that users do not need to manually maintain skip lists of
features. (or in present state, have to maintain fewer of them).

Paves the way toward test harness working for users out of the box,
improving First-Time User Experience, rather than have a large bulk of
tests fail for users by default.

Author: juj

test/test_browser.py

@@ -12,7 +12,6 @@
 import shutil
 import subprocess
 import time
-import unittest
 import zlib
 from functools import wraps
 from http.server import (
@@ -65,7 +64,7 @@
 )
 
 from tools import ports, shared
-from tools.feature_matrix import UNSUPPORTED
+from tools.feature_matrix import UNSUPPORTED, Feature, min_browser_versions
 from tools.shared import DEBUG, EMCC, FILE_PACKAGER, PIPE, WINDOWS
 from tools.utils import delete_dir, memoize
 
@@ -195,6 +194,23 @@ def get_safari_version():
   return parts[0] * 10000 + parts[1] * 100 + parts[2]
 
 
+@memoize
+def get_firefox_version():
+  if not is_firefox():
+    return UNSUPPORTED
+  exe_path = shlex.split(common.EMTEST_BROWSER)[0]
+  ini_path = os.path.join(os.path.dirname(exe_path), "platform.ini")
+  # Extract the first numeric part before any dot (e.g. "Milestone=102.15.1" → 102)
+  m = re.search(r"^Milestone=(.*)$", open(ini_path).read(), re.MULTILINE)
+  milestone = m.group(1).strip()
+  version = int(re.match(r"(\d+)", milestone).group(1))
+  # On Nightly and Beta, e.g. 145.0a1, pretend it to still mean version 144,
+  # since it is a pre-release version
+  if any(c in milestone for c in ('a', 'b')):
+    version -= 1
+  return version
+
+
 no_swiftshader = skip_if_simple('not compatible with swiftshader', is_swiftshader)
 
 no_chrome = skip_if('no_chrome', lambda _: is_chrome(), 'chrome is not supported')
@@ -235,13 +251,50 @@ def decorated(self, threads, *args, **kwargs):
   return decorated
 
 
-def skipExecIf(cond, message):
+def browser_should_skip_feature(skip_env_var, feature):
+  # If an env. var. EMTEST_LACKS_x to skip the given test is set (to either
+  # value 0 or 1), don't bother checking if current browser supports the feature
+  # - just unconditionally run the test, or skip the test.
+  if os.getenv(skip_env_var) is not None:
+    return int(os.getenv(skip_env_var)) != 0
+
+  # If there is no Feature object associated with this capability, then we
+  # should run the test.
+  if feature is None:
+    return False
+
+  # If EMTEST_AUTOSKIP=0, also never skip.
+  if os.getenv('EMTEST_AUTOSKIP') == '0':
+    return False
+
+  # Otherwise EMTEST_AUTOSKIP=1 or EMTEST_AUTOSKIP is not set: check whether
+  # the current browser supports the test or not.
+  min_required = min_browser_versions[feature]
+  not_supported = get_firefox_version() < min_required['firefox'] or get_safari_version() < min_required['safari']
+
+  # Current browser does not support the test, and EMTEST_AUTOSKIP is not set?
+  # Then error out to have end user decide what to do in this situation.
+  if not_supported and os.getenv('EMTEST_AUTOSKIP') is None:
+    return 'error'
+
+  # Report whether to skip the test based on browser support.
+  return not_supported
+
+
+def skipIfFeatureNotAvailable(skip_env_var, feature, message):
+  for env_var in skip_env_var if type(skip_env_var) == list else [skip_env_var]:
+    should_skip = browser_should_skip_feature(env_var, feature)
+    if should_skip:
+      break
+
   def decorator(f):
     assert callable(f)
 
     @wraps(f)
     def decorated(self, *args, **kwargs):
-      if cond:
+      if should_skip == 'error':
+        raise Exception(f'This test requires a browser that supports {feature.name} but your browser {common.EMTEST_BROWSER} does not support this. Run with {skip_env_var}=1 or EMTEST_AUTOSKIP=1 to skip this test automatically.')
+      elif should_skip:
         self.skip_exec = message
       f(self, *args, **kwargs)
 
@@ -251,25 +304,21 @@ def decorated(self, *args, **kwargs):
 
 
 def webgl2_disabled():
-  return os.getenv('EMTEST_LACKS_WEBGL2') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE')
-
-
-def webgpu_disabled():
-  return os.getenv('EMTEST_LACKS_WEBGPU') or os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE')
+  return browser_should_skip_feature('EMTEST_LACKS_WEBGL2', Feature.WEBGL2) or browser_should_skip_feature('EMTEST_LACKS_GRAPHICS_HARDWARE', Feature.WEBGL2)
 
 
-requires_graphics_hardware = skipExecIf(os.getenv('EMTEST_LACKS_GRAPHICS_HARDWARE'), 'This test requires graphics hardware')
-requires_webgl2 = unittest.skipIf(webgl2_disabled(), "This test requires WebGL2 to be available")
-requires_webgpu = unittest.skipIf(webgpu_disabled(), "This test requires WebGPU to be available")
-requires_sound_hardware = skipExecIf(os.getenv('EMTEST_LACKS_SOUND_HARDWARE'), 'This test requires sound hardware')
-requires_microphone_access = skipExecIf(os.getenv('EMTEST_LACKS_MICROPHONE_ACCESS'), 'This test accesses microphone, which may need accepting a user prompt to enable it.')
-requires_offscreen_canvas = unittest.skipIf(os.getenv('EMTEST_LACKS_OFFSCREEN_CANVAS'), 'This test requires a browser with OffscreenCanvas')
-requires_es6_workers = unittest.skipIf(os.getenv('EMTEST_LACKS_ES6_WORKERS'), 'This test requires a browser with ES6 Module Workers support')
-requires_growable_arraybuffers = unittest.skipIf(os.getenv('EMTEST_LACKS_GROWABLE_ARRAYBUFFERS'), 'This test requires a browser that supports growable ArrayBuffers')
+requires_graphics_hardware = skipIfFeatureNotAvailable('EMTEST_LACKS_GRAPHICS_HARDWARE', None, 'This test requires graphics hardware')
+requires_webgl2 = skipIfFeatureNotAvailable(['EMTEST_LACKS_WEBGL2', 'EMTEST_LACKS_GRAPHICS_HARDWARE'], Feature.WEBGL2, 'This test requires WebGL2 to be available')
+requires_webgpu = skipIfFeatureNotAvailable(['EMTEST_LACKS_WEBGPU', 'EMTEST_LACKS_GRAPHICS_HARDWARE'], Feature.WEBGPU, 'This test requires WebGPU to be available')
+requires_sound_hardware = skipIfFeatureNotAvailable('EMTEST_LACKS_SOUND_HARDWARE', None, 'This test requires sound hardware')
+requires_microphone_access = skipIfFeatureNotAvailable('EMTEST_LACKS_MICROPHONE_ACCESS', None, 'This test accesses microphone, which may need accepting a user prompt to enable it.')
+requires_offscreen_canvas = skipIfFeatureNotAvailable('EMTEST_LACKS_OFFSCREEN_CANVAS', Feature.OFFSCREENCANVAS_SUPPORT, 'This test requires a browser with OffscreenCanvas')
+requires_es6_workers = skipIfFeatureNotAvailable('EMTEST_LACKS_ES6_WORKERS', Feature.WORKER_ES6_MODULES, 'This test requires a browser with ES6 Module Workers support')
+requires_growable_arraybuffers = skipIfFeatureNotAvailable('EMTEST_LACKS_GROWABLE_ARRAYBUFFERS', Feature.GROWABLE_ARRAYBUFFERS, 'This test requires a browser that supports growable ArrayBuffers')
 # N.b. not all SharedArrayBuffer requiring tests are annotated with this decorator, since at this point there are so many of such tests.
 # As a middle ground, if a test has a name 'thread' or 'wasm_worker' in it, then it does not need decorating. To run all single-threaded tests in
 # the suite, one can run "EMTEST_LACKS_SHARED_ARRAY_BUFFER=1 test/runner browser skip:browser.test_*thread* skip:browser.test_*wasm_worker* skip:browser.test_*audio_worklet*"
-requires_shared_array_buffer = unittest.skipIf(os.getenv('EMTEST_LACKS_SHARED_ARRAY_BUFFER'), 'This test requires a browser with SharedArrayBuffer support')
+requires_shared_array_buffer = skipIfFeatureNotAvailable('EMTEST_LACKS_SHARED_ARRAY_BUFFER', Feature.THREADS, 'This test requires a browser with SharedArrayBuffer support')
 
 
 class browser(BrowserCore):

tools/feature_matrix.py

@@ -43,6 +43,9 @@ class Feature(IntEnum):
   OFFSCREENCANVAS_SUPPORT = auto()
   WASM_LEGACY_EXCEPTIONS = auto()
   WASM_EXCEPTIONS = auto()
+  WEBGL2 = auto()
+  WEBGPU = auto()
+  GROWABLE_ARRAYBUFFERS = auto()
 
 
 disable_override_features = set()
@@ -97,6 +100,24 @@ class Feature(IntEnum):
     'safari': UNSUPPORTED,
     'node': 230000,
   },
+  # Emscripten itself does not use this feature but we use it in our browser
+  # tests.
+  Feature.WEBGL2: {
+    'chrome': 56,
+    'firefox': 51,
+    'safari': 150000,
+    'node': UNSUPPORTED,
+  },
+  # Emscripten itself does not use this feature but we use it in our browser
+  # tests.
+  Feature.WEBGPU: {
+    'chrome': 113,
+    'firefox': 141,
+    'safari': 260000,
+    'node': UNSUPPORTED,
+  },
+  # Emscripten itself does not use this feature but we use it in our browser
+  # tests.
   # https://caniuse.com/mdn-api_worker_worker_ecmascript_modules: The ability to
   # call new Worker(url, { type: 'module' });
   Feature.WORKER_ES6_MODULES: {
@@ -134,6 +155,16 @@ class Feature(IntEnum):
     # Node.js 26)
     'node': 240000,
   },
+  # Growable SharedArrayBuffers improves memory growth feature in multithreaded
+  # builds by avoiding need to poll resizes to ArrayBuffer views in Workers.
+  # This feature is not used anywhere else except the test harness to detect
+  # browser version.
+  Feature.GROWABLE_ARRAYBUFFERS: {
+    'chrome': 136,
+    'firefox': 145,
+    'safari': UNSUPPORTED,
+    'node': 240000,
+  },
 }
 
 # Static assertion to check that we actually need each of the above feature flags