Seeds of stucking Javascript breaking the browser

[gitreich]

We face Problems with old webpages and not working Javascript on this page.
The Browser Stucks and the ChromeDevolper Protocol is not going any further, and so is Browsertrix

The Crawler could at least try to fire an ESC

Here is one Example, you could have a couple of them more if you need

docker run -v $PWD/crawls:/crawls/ -it webrecorder/browsertrix-crawler:1.6.4 crawl --url https://www.sgoe.at/ --collection sgoe_test
{"timestamp":"2025-10-02T12:30:08.809Z","logLevel":"info","context":"general","message":"Browsertrix-Crawler 1.6.4 (with warcio.js 2.4.4)","details":{}}
{"timestamp":"2025-10-02T12:30:08.810Z","logLevel":"info","context":"general","message":"Seeds","details":[{"url":"https://www.sgoe.at/","scopeType":"prefix","include":["/^https?:\/\/www\.sgoe\.at\//"],"exclude":[],"allowHash":false,"depth":-1,"sitemap":null,"auth":null,"_authEncoded":null,"maxExtraHops":0,"maxDepth":1000000}]}
{"timestamp":"2025-10-02T12:30:08.810Z","logLevel":"info","context":"general","message":"Link Selectors","details":[{"selector":"a[href]","extract":"href","isAttribute":false}]}
{"timestamp":"2025-10-02T12:30:08.810Z","logLevel":"info","context":"general","message":"Behavior Options","details":{"message":"{"autoplay":true,"autofetch":true,"autoscroll":true,"siteSpecific":true,"log":"__bx_log","startEarly":true,"clickSelector":"a"}"}}
{"timestamp":"2025-10-02T12:30:09.196Z","logLevel":"info","context":"worker","message":"Creating 1 workers","details":{}}
{"timestamp":"2025-10-02T12:30:09.197Z","logLevel":"info","context":"worker","message":"Worker starting","details":{"workerid":0}}
{"timestamp":"2025-10-02T12:30:09.357Z","logLevel":"info","context":"worker","message":"Starting page","details":{"workerid":0,"page":"https://www.sgoe.at/"}}
{"timestamp":"2025-10-02T12:30:09.358Z","logLevel":"info","context":"crawlStatus","message":"Crawl statistics","details":{"crawled":0,"total":1,"pending":1,"failed":0,"limit":{"max":0,"hit":false},"pendingPages":["{"seedId":0,"started":"2025-10-02T12:30:09.198Z","extraHops":0,"url":"https:\/\/www.sgoe.at\/","added":"2025-10-02T12:30:08.884Z","depth":0}"]}}
{"timestamp":"2025-10-02T12:30:09.748Z","logLevel":"info","context":"general","message":"Awaiting page load","details":{"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:31:39.829Z","logLevel":"warn","context":"general","message":"Page load timed out, loading but slowly, skipping behaviors","details":{"msg":"Navigation timeout of 90000 ms exceeded","page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:31:39.830Z","logLevel":"info","context":"general","message":"Seed page redirected, adding redirected seed","details":{"origUrl":"https://www.sgoe.at/","newUrl":"https://sgoe.lima-city.de/","seedId":1}}
{"timestamp":"2025-10-02T12:31:44.836Z","logLevel":"error","context":"general","message":"Frame check timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:31:44.840Z","logLevel":"error","context":"general","message":"Frame check timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:31:44.843Z","logLevel":"error","context":"general","message":"Frame check timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:31:49.850Z","logLevel":"warn","context":"general","message":"Cloudflare check timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:31:55.858Z","logLevel":"error","context":"general","message":"Custom page load check timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:32:00.863Z","logLevel":"error","context":"general","message":"Link extraction timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:32:00.863Z","logLevel":"error","context":"general","message":"Link extraction timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:32:00.863Z","logLevel":"error","context":"general","message":"Link extraction timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:32:00.864Z","logLevel":"error","context":"general","message":"Link extraction timed out","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:32:05.869Z","logLevel":"error","context":"general","message":"Timed out getting page title, something is likely wrong","details":{"seconds":5,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:33:19.427Z","logLevel":"warn","context":"worker","message":"Page Worker Timeout","details":{"seconds":190,"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:33:19.590Z","logLevel":"error","context":"browser","message":"Browser disconnected (crashed?), interrupting crawl","details":{}}
{"timestamp":"2025-10-02T12:33:19.592Z","logLevel":"warn","context":"behavior","message":"Skipping behaviors for slow page","details":{"page":"https://www.sgoe.at/","workerid":0}}
{"timestamp":"2025-10-02T12:33:19.592Z","logLevel":"warn","context":"general","message":"Link Extraction failed in frame","details":{"page":"https://www.sgoe.at/","workerid":0,"type":"exception","message":"Protocol error (Runtime.evaluate): Target closed","stack":"TargetCloseError: Protocol error (Runtime.evaluate): Target closed\n at CallbackRegistry.clear (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/common/CallbackRegistry.js:77:36)\n at CdpCDPSession.onClosed (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/CdpSession.js:111:25)\n at #onClose (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/Connection.js:177:21)\n at WebSocket. (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/node/NodeWebSocketTransport.js:42:30)\n at callListener (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:290:14)\n at WebSocket.onClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:220:9)\n at WebSocket.emit (node:events:524:28)\n at WebSocket.emitClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:272:10)\n at Socket.socketOnClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:1341:15)\n at Socket.emit (node:events:524:28)"}}
{"timestamp":"2025-10-02T12:33:19.593Z","logLevel":"warn","context":"general","message":"Link Extraction failed in frame","details":{"page":"https://www.sgoe.at/","workerid":0,"type":"exception","message":"Protocol error (Runtime.evaluate): Target closed","stack":"TargetCloseError: Protocol error (Runtime.evaluate): Target closed\n at CallbackRegistry.clear (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/common/CallbackRegistry.js:77:36)\n at CdpCDPSession.onClosed (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/CdpSession.js:111:25)\n at #onClose (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/Connection.js:177:21)\n at WebSocket. (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/node/NodeWebSocketTransport.js:42:30)\n at callListener (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:290:14)\n at WebSocket.onClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:220:9)\n at WebSocket.emit (node:events:524:28)\n at WebSocket.emitClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:272:10)\n at Socket.socketOnClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:1341:15)\n at Socket.emit (node:events:524:28)"}}
{"timestamp":"2025-10-02T12:33:19.593Z","logLevel":"warn","context":"general","message":"Link Extraction failed in frame","details":{"page":"https://www.sgoe.at/","workerid":0,"type":"exception","message":"Protocol error (Runtime.evaluate): Target closed","stack":"TargetCloseError: Protocol error (Runtime.evaluate): Target closed\n at CallbackRegistry.clear (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/common/CallbackRegistry.js:77:36)\n at CdpCDPSession.onClosed (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/CdpSession.js:111:25)\n at #onClose (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/Connection.js:177:21)\n at WebSocket. (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/node/NodeWebSocketTransport.js:42:30)\n at callListener (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:290:14)\n at WebSocket.onClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:220:9)\n at WebSocket.emit (node:events:524:28)\n at WebSocket.emitClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:272:10)\n at Socket.socketOnClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:1341:15)\n at Socket.emit (node:events:524:28)"}}
{"timestamp":"2025-10-02T12:33:19.593Z","logLevel":"warn","context":"general","message":"Link Extraction failed in frame","details":{"page":"https://www.sgoe.at/","workerid":0,"type":"exception","message":"Protocol error (Runtime.evaluate): Target closed","stack":"TargetCloseError: Protocol error (Runtime.evaluate): Target closed\n at CallbackRegistry.clear (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/common/CallbackRegistry.js:77:36)\n at CdpCDPSession.onClosed (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/CdpSession.js:111:25)\n at #onClose (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/cdp/Connection.js:177:21)\n at WebSocket. (file:///app/node_modules/puppeteer-core/lib/esm/puppeteer/node/NodeWebSocketTransport.js:42:30)\n at callListener (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:290:14)\n at WebSocket.onClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/event-target.js:220:9)\n at WebSocket.emit (node:events:524:28)\n at WebSocket.emitClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:272:10)\n at Socket.socketOnClose (/app/node_modules/puppeteer-core/node_modules/ws/lib/websocket.js:1341:15)\n at Socket.emit (node:events:524:28)"}}
{"timestamp":"2025-10-02T12:33:19.594Z","logLevel":"error","context":"worker","message":"Worker error, exiting","details":{"type":"exception","message":"no page available, shouldn't get here","stack":"Error: no page available, shouldn't get here\n at PageWorker.initPage (file:///app/dist/util/worker.js:163:15)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async PageWorker.runLoop (file:///app/dist/util/worker.js:235:30)\n at async PageWorker.run (file:///app/dist/util/worker.js:208:13)\n at async Promise.allSettled (index 0)\n at async runWorkers (file:///app/dist/util/worker.js:285:5)\n at async Crawler.crawl (file:///app/dist/crawler.js:1103:9)\n at async Crawler.run (file:///app/dist/crawler.js:375:13)\n at async file:///app/dist/main.js:58:1","workerid":0}}
{"timestamp":"2025-10-02T12:33:19.600Z","logLevel":"info","context":"general","message":"Saving crawl state to: /crawls/collections/sgoe_test/crawls/20251002123319596-f065640b3b08-sgoe_test.yaml","details":{}}
{"timestamp":"2025-10-02T12:33:19.601Z","logLevel":"info","context":"crawlStatus","message":"Crawl statistics","details":{"crawled":0,"total":1,"pending":1,"failed":0,"limit":{"max":0,"hit":false},"pendingPages":["{"url":"https:\/\/www.sgoe.at\/","seedId":0,"started":"2025-10-02T12:33:19.431Z","extraHops":0,"depth":0,"added":"2025-10-02T12:30:08.884Z","retry":1}"]}}
{"timestamp":"2025-10-02T12:33:19.602Z","logLevel":"info","context":"general","message":"Crawling done","details":{}}
{"timestamp":"2025-10-02T12:33:19.603Z","logLevel":"info","context":"general","message":"Exiting, Crawl status: interrupted","details":{}}

[ikreymer]

Thanks for the easy repro example! Should be fixed with #895, there is a handler for these JS popups, but was only enabled when autoclick behavior is running - with the fix, it should always be enabled.
With your version, I think it will still work if you add --behaviors autoclick also.