test(wtr): run integration tests in native and synthetic shadow modes @W-19098130 (#5517)

* test(wtr): run all the flavors and see what happens

* test(wtr): prefer DISABLE_SYNTHETIC over duplicate env var name

* test(wtr): fix hydration tests

they weren't actually running before 😰

* test(wtr): create `process.env` with only string values

* test(wtr): split integration and hydration tests into separate jobs

* Revert "test(wtr): prefer DISABLE_SYNTHETIC over duplicate env var name"

This reverts commit 0c81acb.

* test(wtr): revert to using original env var

* test(wtr): make integration and hydration tests have different default shadow modes

they were supposed to all along, but it was implemented incorrectly

* test(wtr): fix tests running in native shadow mode

* test(wtr): make working tests fail for real in CI if they break

* test(wtr): avoid accidentally not running tests again

* test(wtr): enable FORCE_NATIVE_SHADOW_MODE_FOR_TEST in CI

* test(wtr): enable DISABLE_DETACHED_REHYDRATION=1 in CI

* test(wtr): enable more test scenarios in CI

* test(wtr): split CI job into groups

* test(wtr): remove matrix config from job that doesn't need it

oops

* test(wtr): enable API_VERSION tests in CI and alphabetize list

also make john fix it later

* test(wtr): use vitest spy instead of jasmine spy

* test(wtr): fix broken test assertion

Author: wjhsf

.github/workflows/web-test-runner.yml

@@ -18,14 +18,18 @@ env:
     NODE_VERSION: '20.19.4'
 
 jobs:
-    # Starting with the basics, just get tests running in CI
-    # TODO: add env var combos we use for Karma tests
     # TODO: upload result artifacts
     # TODO: make it saucy 🥫
-    wtr-group-1:
+    integration-tests:
+        name: Integration tests (${{ matrix.shadow_mode }} shadow)
+        strategy:
+            matrix:
+                shadow_mode: [native, synthetic]
+
         runs-on: ubuntu-22.04
         env:
             SAUCE_TUNNEL_ID: github-action-tunnel-wtr-${{github.run_id}}-group-1
+            SHADOW_MODE_OVERRIDE: ${{ matrix.shadow_mode }}
         defaults:
             run:
                 working-directory: ./packages/@lwc/integration-not-karma
@@ -51,4 +55,93 @@ jobs:
             #       region: us
 
             - run: yarn test
+            - run: API_VERSION=58 yarn test
+            - run: API_VERSION=59 yarn test
+            - run: API_VERSION=60 yarn test
+            - run: API_VERSION=61 yarn test
+            - run: API_VERSION=62 yarn test
+            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test || true
+            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
+            - run: ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1 yarn test
+            - run: NODE_ENV_FOR_TEST=production yarn test
+
+    integration-tests-not-both-modes:
+        # Tests that should run in only synthetic or native shadow, not both
+        name: Integration tests (singleton batch)
+        runs-on: ubuntu-22.04
+        env:
+            SAUCE_TUNNEL_ID: github-action-tunnel-wtr-${{github.run_id}}-group-1
+            SHADOW_MODE_OVERRIDE: ${{ matrix.shadow_mode }}
+        defaults:
+            run:
+                working-directory: ./packages/@lwc/integration-not-karma
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Setup Node
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ env.NODE_VERSION }}
+                  cache: 'yarn'
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+              working-directory: ./
+
+            # - uses: saucelabs/sauce-connect-action@v3.0.0
+            #   with:
+            #       username: ${{ secrets.SAUCE_USERNAME }}
+            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+            #       region: us
+
+            # Synthetic shadow only
+            - run: LEGACY_BROWSERS=1 yarn test || true
+            - run: FORCE_NATIVE_SHADOW_MODE_FOR_TEST=1 yarn test
+            - run: DISABLE_DETACHED_REHYDRATION=1 yarn test
+            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 DISABLE_DETACHED_REHYDRATION=1 yarn test || true
+
+            # Native shadow only -- don't forget SHADOW_MODE_OVERRIDE!
+            - run: SHADOW_MODE_OVERRIDE=native DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 yarn test
+            - run: SHADOW_MODE_OVERRIDE=native DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER=1 DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
+
+    hydration-tests:
+        runs-on: ubuntu-22.04
+        env:
+            SAUCE_TUNNEL_ID: github-action-tunnel-wtr-${{github.run_id}}-group-1
+        defaults:
+            run:
+                working-directory: ./packages/@lwc/integration-not-karma
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Setup Node
+              uses: actions/setup-node@v4
+              with:
+                  node-version: ${{ env.NODE_VERSION }}
+                  cache: 'yarn'
+
+            - name: Install dependencies
+              run: yarn install --frozen-lockfile
+              working-directory: ./
+
+            # - uses: saucelabs/sauce-connect-action@v3.0.0
+            #   with:
+            #       username: ${{ secrets.SAUCE_USERNAME }}
+            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+            #       region: us
+            - run: ENGINE_SERVER=1 yarn test:hydration
+            - run: ENGINE_SERVER=1 SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
+            - run: ENGINE_SERVER=1 NODE_ENV_FOR_TEST=production yarn test:hydration
+            - run: ENGINE_SERVER=1 DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
+            - run: ENGINE_SERVER=1 DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration
+            - run: ENGINE_SERVER=1 DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
+            - run: ENGINE_SERVER=1 DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 DISABLE_DETACHED_REHYDRATION=1 yarn test:hydration
             - run: yarn test:hydration
+            - run: SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
+            - run: NODE_ENV_FOR_TEST=production yarn test:hydration
+            - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test:hydration
+            - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test:hydration

packages/@lwc/integration-not-karma/configs/base.js

@@ -1,55 +1,70 @@
 import { join } from 'node:path';
 import { LWC_VERSION } from '@lwc/shared';
-import * as options from '../helpers/options.js';
 import { resolvePathOutsideRoot } from '../helpers/utils.js';
 
+/**
+ * We want to convert from parsed options (true/false) to a `process.env` with only strings.
+ * This drops `false` values and converts everything else to a string.
+ */
+const envify = (obj) => {
+    const clone = {};
+    for (const [key, val] of Object.entries(obj)) {
+        if (val !== false) {
+            clone[key] = String(val);
+        }
+    }
+    return clone;
+};
 const pluck = (obj, keys) => Object.fromEntries(keys.map((k) => [k, obj[k]]));
 const maybeImport = (file, condition) => (condition ? `await import('${file}');` : '');
 
-/** `process.env` to inject into test environment. */
-const env = {
-    ...pluck(options, [
-        'API_VERSION',
-        'DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE',
-        'DISABLE_STATIC_CONTENT_OPTIMIZATION',
-        'DISABLE_SYNTHETIC',
-        'ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL',
-        'ENGINE_SERVER',
-        'FORCE_NATIVE_SHADOW_MODE_FOR_TEST',
-        'NATIVE_SHADOW',
-        'DISABLE_DETACHED_REHYDRATION',
-    ]),
-    LWC_VERSION,
-    NODE_ENV: options.NODE_ENV_FOR_TEST,
-};
+/** @type {() => import("@web/test-runner").TestRunnerConfig} */
+export default (options) => {
+    /** `process.env` to inject into test environment. */
+    const env = envify({
+        ...pluck(options, [
+            'API_VERSION',
+            'DISABLE_DETACHED_REHYDRATION',
+            'DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE',
+            'DISABLE_STATIC_CONTENT_OPTIMIZATION',
+            'ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL',
+            'ENGINE_SERVER',
+            'FORCE_NATIVE_SHADOW_MODE_FOR_TEST',
+            'NATIVE_SHADOW',
+        ]),
+        LWC_VERSION,
+        NODE_ENV: options.NODE_ENV_FOR_TEST,
+    });
 
-/** @type {import("@web/test-runner").TestRunnerConfig} */
-export default {
-    // FIXME: Parallelism breaks tests that rely on focus/requestAnimationFrame, because they often
-    // time out before they receive focus. But it also makes the full suite take 3x longer to run...
-    // Potential workaround: https://github.com/modernweb-dev/web/issues/2588
-    concurrency: 1,
-    browserLogs: false,
-    nodeResolve: true,
-    rootDir: join(import.meta.dirname, '..'),
-    plugins: [
-        {
-            name: 'lwc-base-plugin',
-            resolveImport({ source }) {
-                if (source === 'wire-service') {
-                    return resolvePathOutsideRoot('../wire-service/dist/index.js');
-                }
-            },
-            async transform(ctx) {
-                if (ctx.type === 'application/javascript') {
-                    // FIXME: copy/paste Nolan's spiel about why we do this ugly thing
-                    return ctx.body.replace(/process\.env\.NODE_ENV === 'test-karma-lwc'/g, 'true');
-                }
+    return {
+        // FIXME: Parallelism breaks tests that rely on focus/requestAnimationFrame, because they often
+        // time out before they receive focus. But it also makes the full suite take 3x longer to run...
+        // Potential workaround: https://github.com/modernweb-dev/web/issues/2588
+        concurrency: 1,
+        browserLogs: false,
+        nodeResolve: true,
+        rootDir: join(import.meta.dirname, '..'),
+        plugins: [
+            {
+                name: 'lwc-base-plugin',
+                resolveImport({ source }) {
+                    if (source === 'wire-service') {
+                        return resolvePathOutsideRoot('../wire-service/dist/index.js');
+                    }
+                },
+                async transform(ctx) {
+                    if (ctx.type === 'application/javascript') {
+                        // FIXME: copy/paste Nolan's spiel about why we do this ugly thing
+                        return ctx.body.replace(
+                            /process\.env\.NODE_ENV === 'test-karma-lwc'/g,
+                            'true'
+                        );
+                    }
+                },
             },
-        },
-    ],
-    testRunnerHtml: (testFramework) =>
-        `<!DOCTYPE html>
+        ],
+        testRunnerHtml: (testFramework) =>
+            `<!DOCTYPE html>
         <html>
           <head>
             <script type="module">
@@ -61,11 +76,12 @@ export default {
                 ])
             )};
 
-            ${maybeImport('@lwc/synthetic-shadow', !options.DISABLE_SYNTHETIC)}
+            ${maybeImport('@lwc/synthetic-shadow', !options.NATIVE_SHADOW)}
             ${maybeImport('@lwc/aria-reflection', options.ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL)}
             </script>
             <script type="module" src="./helpers/setup.js"></script>
             <script type="module" src="${testFramework}"></script>
           </head>
         </html>`,
+    };
 };

packages/@lwc/integration-not-karma/configs/hydration.js

@@ -1,8 +1,14 @@
-// Use native shadow by default in hydration tests; MUST be set before imports
-process.env.DISABLE_SYNTHETIC ??= 'true';
-import baseConfig from './base.js';
+import * as options from '../helpers/options.js';
+import createConfig from './base.js';
 import hydrationTestPlugin from './plugins/serve-hydration.js';
 
+const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'native';
+
+const baseConfig = createConfig({
+    ...options,
+    NATIVE_SHADOW: SHADOW_MODE === 'native' || options.FORCE_NATIVE_SHADOW_MODE_FOR_TEST,
+});
+
 /** @type {import("@web/test-runner").TestRunnerConfig} */
 export default {
     ...baseConfig,

packages/@lwc/integration-not-karma/configs/integration.js

@@ -1,11 +1,24 @@
 import { importMapsPlugin } from '@web/dev-server-import-maps';
-import baseConfig from './base.js';
+import * as options from '../helpers/options.js';
+import createConfig from './base.js';
 import testPlugin from './plugins/serve-integration.js';
 
+const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'synthetic';
+
+const baseConfig = createConfig({
+    ...options,
+    NATIVE_SHADOW: SHADOW_MODE === 'native' || options.FORCE_NATIVE_SHADOW_MODE_FOR_TEST,
+});
+
 /** @type {import("@web/test-runner").TestRunnerConfig} */
 export default {
     ...baseConfig,
-    files: ['test/**/*.spec.js'],
+    files: [
+        'test/**/*.spec.js',
+        // Make John fix this after his PR is merged
+        '!test/template-expressions/errors/index.spec.js',
+        '!test/template-expressions/smoke-test/index.spec.js',
+    ],
     plugins: [
         ...baseConfig.plugins,
         importMapsPlugin({ inject: { importMap: { imports: { lwc: './mocks/lwc.js' } } } }),

packages/@lwc/integration-not-karma/configs/plugins/test-hydration.js

@@ -52,8 +52,8 @@ export function runTest(configPath, componentPath, ssrRendered) {
     let Component;
 
     beforeAll(async () => {
-        testConfig = await import(configPath);
-        Component = await import(componentPath);
+        testConfig = (await import(configPath)).default;
+        Component = (await import(componentPath)).default;
         setFeatureFlags(testConfig.requiredFeatureFlags, true);
     });
 
@@ -93,6 +93,8 @@ export function runTest(configPath, componentPath, ssrRendered) {
                 container,
                 selector,
             });
+        } else {
+            throw new Error(`Missing test or advancedTest function in ${configPath}.`);
         }
     });
 }

packages/@lwc/integration-not-karma/helpers/options.js

@@ -8,8 +8,6 @@ import { HIGHEST_API_VERSION } from '@lwc/shared';
 
 export const LEGACY_BROWSERS = Boolean(process.env.LEGACY_BROWSERS);
 
-export const DISABLE_SYNTHETIC = Boolean(process.env.DISABLE_SYNTHETIC);
-
 export const FORCE_NATIVE_SHADOW_MODE_FOR_TEST = Boolean(
     process.env.FORCE_NATIVE_SHADOW_MODE_FOR_TEST
 );
@@ -36,20 +34,26 @@ export const ENGINE_SERVER = Boolean(process.env.ENGINE_SERVER);
 
 // --- Test config --- //
 
+/**
+ * Integration tests default to synthetic shadow mode, while hydration tests default to native.
+ * This should be set to "native" or "synthetic" to override the default mode.
+ * @type {'native'|'synthetic'|undefined}
+ */
+// NOTE: NATIVE_SHADOW is not defined here because integration/hydration have different defaults
+export const SHADOW_MODE_OVERRIDE = process.env.SHADOW_MODE_OVERRIDE;
+
 export const API_VERSION = process.env.API_VERSION
     ? parseInt(process.env.API_VERSION, 10)
     : HIGHEST_API_VERSION;
 
 export const NODE_ENV_FOR_TEST = process.env.NODE_ENV_FOR_TEST || 'development';
 
-export const NATIVE_SHADOW = DISABLE_SYNTHETIC || FORCE_NATIVE_SHADOW_MODE_FOR_TEST;
-
 /** Unique directory name that encodes the flags that the tests were executed with. */
 export const COVERAGE_DIR_FOR_OPTIONS =
     Object.entries({
         API_VERSION,
         DISABLE_STATIC_CONTENT_OPTIMIZATION,
-        DISABLE_SYNTHETIC,
+        SHADOW_MODE_OVERRIDE,
         DISABLE_SYNTHETIC_SHADOW_SUPPORT_IN_COMPILER,
         ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL,
         FORCE_NATIVE_SHADOW_MODE_FOR_TEST,

packages/@lwc/integration-not-karma/test-hydration/synthetic-shadow/index.spec.js

@@ -22,7 +22,7 @@ export default {
         expect(child.shadowRoot.synthetic).toBeUndefined();
 
         // sanity check that the env var is working
-        if (process.env.DISABLE_SYNTHETIC) {
+        if (process.env.NATIVE_SHADOW) {
             expect(document.body.attachShadow.toString()).toContain('[native code');
         } else {
             expect(document.body.attachShadow.toString()).not.toContain('[native code');

packages/@lwc/integration-not-karma/test/component/LightningElement.attachInternals/elementInternals/sanity/index.spec.js

@@ -9,10 +9,6 @@ beforeEach(() => {
     document.body.appendChild(elm);
 });
 
-afterEach(() => {
-    document.body.removeChild(elm);
-});
-
 describe.runIf(
     ENABLE_ELEMENT_INTERNALS_AND_FACE &&
         process.env.NATIVE_SHADOW &&