Improve hls runtime keep async only databse keys downsweep skip non dirties

.github/workflows/bench.yml

@@ -19,7 +19,10 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       should_skip: ${{ steps.skip_check.outputs.should_skip }}
+      bench_ghc_versions: ${{ steps.set_ghc_versions.outputs.versions }}
     steps:
+      - id: set_ghc_versions
+        run: echo 'versions=["9.12", "9.10"]' >> $GITHUB_OUTPUT
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@v5.3.1
         with:
@@ -46,14 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # benching the two latest GHCs we support now
-        # since benchmark are expansive.
-        # choosing the two latest are easier to maintain and more forward looking
-        # see discussion https://github.com/haskell/haskell-language-server/pull/4118
-        # also possible to add more GHCs if we performs better in the future.
-        ghc:
-          - '9.8'
-          - '9.10'
+        ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }}
         os:
           - ubuntu-latest
 
@@ -121,7 +117,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ghc: ['9.8', '9.10']
+        ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }}
         os: [ubuntu-latest]
         cabal: ['3.14']
         example: ['cabal', 'lsp-types']

.github/workflows/flakiness.yml

@@ -0,0 +1,109 @@
+name: flakiness
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+on:
+  # Run on PRs that touch relevant areas and on manual dispatch
+  pull_request:
+    branches:
+      - '**'
+    paths:
+      - 'scripts/flaky-test-loop.sh'
+      - 'scripts/flaky-test-patterns.txt'
+      - 'ghcide/**'
+      - 'ghcide-test/**'
+      - 'hls-test-utils/**'
+      - 'src/**'
+      - 'exe/**'
+      - 'plugins/**'
+      - 'cabal.project'
+      - 'stack.yaml'
+      - 'haskell-language-server.cabal'
+      - '.github/workflows/flakiness.yml'
+  workflow_dispatch:
+    inputs:
+      max_iter:
+        description: 'Maximum iterations to attempt'
+        required: false
+        default: '1000'
+      sleep_secs:
+        description: 'Seconds to sleep between iterations'
+        required: false
+        default: '0'
+      test_patterns:
+        description: 'Comma-separated Tasty patterns to run each iteration (overrides default)'
+        required: false
+        default: ''
+
+jobs:
+  loop:
+    name: Flakiness Test (broken pipe and test failures)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          # - windows-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup GHC and caching
+        uses: ./.github/actions/setup-build
+        with:
+          ghc: '9.12'
+          os: ${{ runner.os }}
+
+      - name: Show cabal and GHC versions
+        run: |
+          cabal --version
+          ghc --version
+      - name: Build
+        env:
+          PATTERN_FILE: 'scripts/flaky-test-patterns.txt'
+          RUN_MODE: 'build'
+        run: HLS_TEST_EXE="$(cabal exec which hls)" bash scripts/flaky-test-loop.sh
+
+      - name: Run flakiness loop
+        id: run-loop
+        # Let this run for a while; build is done once inside the script
+        timeout-minutes: 60
+        env:
+          # Use workflow_dispatch inputs when present, else defaults
+          SLEEP_SECS: ${{ github.event.inputs.sleep_secs || '0' }}
+          LOG_STDERR: '1'
+          TEST_PATTERNS: ${{ github.event.inputs.test_patterns }}
+          PATTERN_FILE: 'scripts/flaky-test-patterns.txt'
+          NO_BUILD_ONCE: '1'
+          RUN_MODE: 'run'
+          # HLS_TEST_EXE: 'hls' # HLS_WRAPPER_TEST_EXE: 'hls-wrapper'
+        run: |
+          # Run with a sensible default of 500 iterations on PRs;
+          max_iter="${{ github.event.inputs.max_iter }}"
+          max_iter="${max_iter:-500}"
+          # copy hls to current dir so the script can find it
+          HLS_TEST_EXE="$(cabal exec which hls)" bash scripts/flaky-test-loop.sh "${max_iter}"
+          ec=$?
+          # Interpret exit codes from flaky-test-loop.sh
+          #   0 => no issues reproduced within MAX_ITER          -> pass job
+          #   1 => issue reproduced (broken pipe or test failure) -> fail job
+          #   2+ => setup/infra error                            -> fail job
+          if [[ $ec -eq 1 ]]; then
+            echo "Issue reproduced (broken pipe or test failure): failing job"
+            exit 1
+          elif [[ $ec -eq 0 ]]; then
+            echo "No issues reproduced within MAX_ITER=${max_iter}: passing"
+            exit 0
+          else
+            echo "Loop script error (exit $ec): failing"
+            exit $ec
+          fi
+

.github/workflows/supported-ghc-versions.json

@@ -1 +1 @@
-["9.12", "9.10", "9.8", "9.6"]
+["9.12", "9.10"]

.github/workflows/test.yml

@@ -113,7 +113,7 @@ jobs:
       - if: needs.pre_job.outputs.should_skip_ghcide != 'true' && matrix.test
         name: Test ghcide
         # run the tests without parallelism to avoid running out of memory
-        run: cabal test ghcide-tests || cabal test ghcide-tests
+        run: cabal test ghcide-tests
 
       - if: matrix.test
         name: Test hls-plugin-api
@@ -135,124 +135,124 @@ jobs:
 
       - if: matrix.test
         name: Test hls-refactor-plugin
-        run: cabal test hls-refactor-plugin-tests || cabal test hls-refactor-plugin-tests
+        run: cabal test hls-refactor-plugin-tests
 
       # TODO enable when it supports 9.10
       - if: matrix.test && matrix.ghc != '9.10' && matrix.ghc != '9.12'
         name: Test hls-floskell-plugin
-        run: cabal test hls-floskell-plugin-tests || cabal test hls-floskell-plugin-tests
+        run: cabal test hls-floskell-plugin-tests
 
       - if: matrix.test
         name: Test hls-class-plugin
-        run: cabal test hls-class-plugin-tests || cabal test hls-class-plugin-tests
+        run: cabal test hls-class-plugin-tests
 
       - if: matrix.test
         name: Test hls-pragmas-plugin
-        run: cabal test hls-pragmas-plugin-tests || cabal test hls-pragmas-plugin-tests
+        run: cabal test hls-pragmas-plugin-tests
 
       - if: matrix.test
         name: Test hls-eval-plugin
-        run: cabal test hls-eval-plugin-tests || cabal test hls-eval-plugin-tests
+        run: cabal test hls-eval-plugin-tests
 
       # TODO enable when it supports 9.10
       - if: matrix.test && matrix.ghc != '9.10' && matrix.ghc != '9.12'
         name: Test hls-splice-plugin
-        run: cabal test hls-splice-plugin-tests || cabal test hls-splice-plugin-tests
+        run: cabal test hls-splice-plugin-tests
 
       - if: matrix.test && matrix.ghc != '9.12'
         name: Test hls-stan-plugin
-        run: cabal test hls-stan-plugin-tests || cabal test hls-stan-plugin-tests
+        run: cabal test hls-stan-plugin-tests
 
       - if: matrix.test
         name: Test hls-stylish-haskell-plugin
-        run: cabal test hls-stylish-haskell-plugin-tests || cabal test hls-stylish-haskell-plugin-tests
+        run: cabal test hls-stylish-haskell-plugin-tests
 
       - if: matrix.test
         name: Test hls-ormolu-plugin
-        run: cabal test hls-ormolu-plugin-tests || cabal test hls-ormolu-plugin-tests
+        run: cabal test hls-ormolu-plugin-tests
 
       - if: matrix.test
         name: Test hls-fourmolu-plugin
-        run: cabal test hls-fourmolu-plugin-tests || cabal test hls-fourmolu-plugin-tests
+        run: cabal test hls-fourmolu-plugin-tests
 
       - if: matrix.test
         name: Test hls-explicit-imports-plugin test suite
-        run: cabal test hls-explicit-imports-plugin-tests || cabal test hls-explicit-imports-plugin-tests
+        run: cabal test hls-explicit-imports-plugin-tests
 
       - if: matrix.test
         name: Test hls-call-hierarchy-plugin test suite
-        run: cabal test hls-call-hierarchy-plugin-tests || cabal test hls-call-hierarchy-plugin-tests
+        run: cabal test hls-call-hierarchy-plugin-tests
 
       - if: matrix.test && matrix.os != 'windows-latest'
         name: Test hls-rename-plugin test suite
-        run: cabal test hls-rename-plugin-tests || cabal test hls-rename-plugin-tests
+        run: cabal test hls-rename-plugin-tests
 
       # TODO enable when it supports 9.10
       - if: matrix.test && matrix.ghc != '9.10'
         name: Test hls-hlint-plugin test suite
-        run: cabal test hls-hlint-plugin-tests || cabal test hls-hlint-plugin-tests
+        run: cabal test hls-hlint-plugin-tests
 
       - if: matrix.test
         name: Test hls-module-name-plugin test suite
-        run: cabal test hls-module-name-plugin-tests || cabal test hls-module-name-plugin-tests
+        run: cabal test hls-module-name-plugin-tests
 
       - if: matrix.test
         name: Test hls-alternate-number-format-plugin test suite
-        run: cabal test hls-alternate-number-format-plugin-tests || cabal test hls-alternate-number-format-plugin-tests
+        run: cabal test hls-alternate-number-format-plugin-tests
 
       - if: matrix.test
         name: Test hls-qualify-imported-names-plugin test suite
-        run: cabal test hls-qualify-imported-names-plugin-tests || cabal test hls-qualify-imported-names-plugin-tests
+        run: cabal test hls-qualify-imported-names-plugin-tests
 
       - if: matrix.test
         name: Test hls-code-range-plugin test suite
-        run: cabal test hls-code-range-plugin-tests || cabal test hls-code-range-plugin-tests
+        run: cabal test hls-code-range-plugin-tests
 
       - if: matrix.test
         name: Test hls-change-type-signature test suite
-        run: cabal test hls-change-type-signature-plugin-tests || cabal test hls-change-type-signature-plugin-tests
+        run: cabal test hls-change-type-signature-plugin-tests
 
       - if: matrix.test
         name: Test hls-gadt-plugin test suit
-        run: cabal test hls-gadt-plugin-tests || cabal test hls-gadt-plugin-tests
+        run: cabal test hls-gadt-plugin-tests
 
       - if: matrix.test
         name: Test hls-explicit-fixity-plugin test suite
-        run: cabal test hls-explicit-fixity-plugin-tests || cabal test hls-explicit-fixity-plugin-tests
+        run: cabal test hls-explicit-fixity-plugin-tests
 
       - if: matrix.test
         name: Test hls-explicit-record-fields-plugin test suite
-        run: cabal test hls-explicit-record-fields-plugin-tests || cabal test hls-explicit-record-fields-plugin-tests
+        run: cabal test hls-explicit-record-fields-plugin-tests
 
       # versions need to be limited since the tests depend on cabal-fmt which only builds with ghc <9.10
       - if: matrix.test && matrix.ghc != '9.10' && matrix.ghc != '9.12'
         name: Test hls-cabal-fmt-plugin test suite
-        run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests || cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests
+        run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests
 
       - if: matrix.test && matrix.ghc != '9.12'
         name: Test hls-cabal-gild-plugin test suite
-        run: cabal test hls-cabal-gild-plugin-tests --flag=isolateCabalGildTests || cabal test hls-cabal-gild-plugin-tests --flag=isolateCabalGildTests
+        run: cabal test hls-cabal-gild-plugin-tests --flag=isolateCabalGildTests
 
       - if: matrix.test
         name: Test hls-cabal-plugin test suite
-        run: cabal test hls-cabal-plugin-tests || cabal test hls-cabal-plugin-tests
+        run: cabal test hls-cabal-plugin-tests
 
       # TODO enable when it supports 9.10
       - if: matrix.test && matrix.ghc != '9.10' && matrix.ghc != '9.12'
         name: Test hls-retrie-plugin test suite
-        run: cabal test hls-retrie-plugin-tests || cabal test hls-retrie-plugin-tests
+        run: cabal test hls-retrie-plugin-tests
 
       - if: matrix.test
         name: Test hls-overloaded-record-dot-plugin test suite
-        run: cabal test hls-overloaded-record-dot-plugin-tests || cabal test hls-overloaded-record-dot-plugin-tests
+        run: cabal test hls-overloaded-record-dot-plugin-tests
 
       - if: matrix.test
         name: Test hls-semantic-tokens-plugin test suite
-        run: cabal test hls-semantic-tokens-plugin-tests || cabal test hls-semantic-tokens-plugin-tests
+        run: cabal test hls-semantic-tokens-plugin-tests
 
       - if: matrix.test
         name: Test hls-notes-plugin test suite
-        run: cabal test hls-notes-plugin-tests || cabal test hls-notes-plugin-tests
+        run: cabal test hls-notes-plugin-tests
 
       # The plugin tutorial is only compatible with 9.6 and 9.8.
       # No particular reason, just to avoid excessive CPP.
@@ -262,7 +262,7 @@ jobs:
 
       - if: matrix.test
         name: Test hls-signature-help-plugin test suite
-        run: cabal test hls-signature-help-plugin-tests || cabal test hls-signature-help-plugin-tests
+        run: cabal test hls-signature-help-plugin-tests
 
   test_post_job:
     if: always()