From e3950cf70370a30d6d5c6fdb0594d8e4b37e20ad Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 28 Nov 2025 20:00:20 +0200 Subject: [PATCH 1/7] do not use `prerender` for head and content type requests --- lib/utils.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index c4c839f16..a17975702 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -95,7 +95,7 @@ export function prepareRequestOptions(request_options, options) { var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options); if (proxy) { - if (proxy.prerender && CONFIG.PRERENDER_URL) { + if (proxy.prerender && CONFIG.PRERENDER_URL && !options?.prevent_prerender) { request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); } else if (proxy.proxy && CONFIG.PROXY_URL) { @@ -278,7 +278,10 @@ var getHead = function(url, options, callbacks) { redirect: redirect, follow: follow // No abort controller for head. - }, options); + }, { + ...options, + prevent_prerender: true + }); try { fetchStreamAuthorized(request_options) @@ -626,6 +629,7 @@ export function findMainLdObjectWithVideo(ld) { } getUrlFunctional(uri, { + prevent_prerender: true, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, asBuffer: true }, { @@ -815,7 +819,8 @@ export function getContentType(uriForCache, uriOriginal, options, cb) { var methodCaller = method && method === 'GET' ? getUrlFunctional : getHeadFunctional; methodCaller(uri, { - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + prevent_prerender: true }, { onResponse: function(res, request_options) { @@ -1230,7 +1235,9 @@ var getUriStatusPrivate = function(uri, options, cb) { }, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, follow: CONFIG.MAX_REDIRECTS - }) + }, { + prevent_prerender: true + }); try { fetchStream(request_options) From 4291102739346ccebcf9fd8203bc44f4a253b2e5 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Fri, 28 Nov 2025 20:29:29 +0200 Subject: [PATCH 2/7] `prevent_prerender` for `oembed` and `request` --- lib/plugins/system/oembed/oembedUtils.js | 4 +++- lib/request.js | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js index 6e1ebf861..fcc02dce0 100644 --- a/lib/plugins/system/oembed/oembedUtils.js +++ b/lib/plugins/system/oembed/oembedUtils.js @@ -238,7 +238,9 @@ export function findOembedLinks(uri, meta) { url: uri }); - getUrlFunctional(uri, {}, { + getUrlFunctional(uri, { + prevent_prerender: true + }, { onError: function(error) { options && options.registerFetchError({ diff --git a/lib/request.js b/lib/request.js index 15b4f40c8..54582135c 100644 --- a/lib/request.js +++ b/lib/request.js @@ -162,7 +162,10 @@ export default function(options, iframely_options, callback) { url: options.uri }); - fetchData(utils.prepareRequestOptions(options, iframely_options)) + fetchData(utils.prepareRequestOptions(options, { + ...iframely_options, + prevent_prerender: true + })) .then(result => { finish(null, result); }) From a8a8a9f5c32ec87512a997b968821194e083e1f1 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 2 Dec 2025 15:45:46 +0200 Subject: [PATCH 3/7] send additional params to allow iframes in prerender --- lib/utils.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/utils.js b/lib/utils.js index a17975702..68ef705e2 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -98,6 +98,10 @@ export function prepareRequestOptions(request_options, options) { if (proxy.prerender && CONFIG.PRERENDER_URL && !options?.prevent_prerender) { request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); + if (proxy.prerender === 'iframes') { + request_options.uri += '&allow=iframes'; + } + } else if (proxy.proxy && CONFIG.PROXY_URL) { request_options.uri = /{url}/.test(CONFIG.PROXY_URL) ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri)) From 78f3f63474e01118ae9c9e9bac15ef18b2afffc8 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 2 Dec 2025 15:50:18 +0200 Subject: [PATCH 4/7] better support for PRERENDER_URL --- lib/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index 68ef705e2..80cdd17ad 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -99,7 +99,7 @@ export function prepareRequestOptions(request_options, options) { request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); if (proxy.prerender === 'iframes') { - request_options.uri += '&allow=iframes'; + request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + 'allow=iframes'; } } else if (proxy.proxy && CONFIG.PROXY_URL) { From 95f2588bd934e17ba199a48ca32c93be0a798269 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 2 Dec 2025 18:43:02 +0200 Subject: [PATCH 5/7] use prerender only for htmlparser --- lib/plugins/system/htmlparser/htmlparser.js | 3 ++- lib/plugins/system/oembed/oembedUtils.js | 4 +--- lib/request.js | 3 +-- lib/utils.js | 17 +++++++---------- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/plugins/system/htmlparser/htmlparser.js b/lib/plugins/system/htmlparser/htmlparser.js index dfd79fa9e..fccbf29d2 100644 --- a/lib/plugins/system/htmlparser/htmlparser.js +++ b/lib/plugins/system/htmlparser/htmlparser.js @@ -20,7 +20,8 @@ export default { var options2 = {...options, ...{ followRedirect: !!options.followHTTPRedirect, - reuseCookies: !!options.followHTTPRedirect + reuseCookies: !!options.followHTTPRedirect, + allowPrerender: true }}; options.registerFetch({ diff --git a/lib/plugins/system/oembed/oembedUtils.js b/lib/plugins/system/oembed/oembedUtils.js index fcc02dce0..6e1ebf861 100644 --- a/lib/plugins/system/oembed/oembedUtils.js +++ b/lib/plugins/system/oembed/oembedUtils.js @@ -238,9 +238,7 @@ export function findOembedLinks(uri, meta) { url: uri }); - getUrlFunctional(uri, { - prevent_prerender: true - }, { + getUrlFunctional(uri, {}, { onError: function(error) { options && options.registerFetchError({ diff --git a/lib/request.js b/lib/request.js index 54582135c..bda8ff169 100644 --- a/lib/request.js +++ b/lib/request.js @@ -163,8 +163,7 @@ export default function(options, iframely_options, callback) { }); fetchData(utils.prepareRequestOptions(options, { - ...iframely_options, - prevent_prerender: true + ...iframely_options })) .then(result => { finish(null, result); diff --git a/lib/utils.js b/lib/utils.js index 80cdd17ad..a8b8a8224 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -95,11 +95,12 @@ export function prepareRequestOptions(request_options, options) { var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options); if (proxy) { - if (proxy.prerender && CONFIG.PRERENDER_URL && !options?.prevent_prerender) { + if (proxy.prerender && CONFIG.PRERENDER_URL && options?.allowPrerender) { request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); - if (proxy.prerender === 'iframes') { - request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + 'allow=iframes'; + // Use string `proxy.prerender` as additional prerender param. + if (typeof proxy.prerender === 'string') { + request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${proxy.prerender}`; } } else if (proxy.proxy && CONFIG.PROXY_URL) { @@ -283,8 +284,7 @@ var getHead = function(url, options, callbacks) { follow: follow // No abort controller for head. }, { - ...options, - prevent_prerender: true + ...options }); try { @@ -633,7 +633,7 @@ export function findMainLdObjectWithVideo(ld) { } getUrlFunctional(uri, { - prevent_prerender: true, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, asBuffer: true }, { @@ -823,8 +823,7 @@ export function getContentType(uriForCache, uriOriginal, options, cb) { var methodCaller = method && method === 'GET' ? getUrlFunctional : getHeadFunctional; methodCaller(uri, { - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - prevent_prerender: true + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT }, { onResponse: function(res, request_options) { @@ -1239,8 +1238,6 @@ var getUriStatusPrivate = function(uri, options, cb) { }, timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, follow: CONFIG.MAX_REDIRECTS - }, { - prevent_prerender: true }); try { From 314c5ad4236cdde9dabb41d98b3dcc0454c8d514 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 2 Dec 2025 18:45:18 +0200 Subject: [PATCH 6/7] fix obsolete code --- lib/request.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/request.js b/lib/request.js index bda8ff169..15b4f40c8 100644 --- a/lib/request.js +++ b/lib/request.js @@ -162,9 +162,7 @@ export default function(options, iframely_options, callback) { url: options.uri }); - fetchData(utils.prepareRequestOptions(options, { - ...iframely_options - })) + fetchData(utils.prepareRequestOptions(options, iframely_options)) .then(result => { finish(null, result); }) From c5f021557ac18dc1ea30ccf8ea074c95f0ce1af9 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 2 Dec 2025 18:46:02 +0200 Subject: [PATCH 7/7] fix obsolete code --- lib/utils.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index a8b8a8224..5dbe9a4ab 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -283,9 +283,7 @@ var getHead = function(url, options, callbacks) { redirect: redirect, follow: follow // No abort controller for head. - }, { - ...options - }); + }, options); try { fetchStreamAuthorized(request_options) @@ -633,7 +631,6 @@ export function findMainLdObjectWithVideo(ld) { } getUrlFunctional(uri, { - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, asBuffer: true }, {