From e45ba684207745b06bf1130cc80893e6cafdeaa3 Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Thu, 24 Apr 2025 13:56:35 -0400 Subject: [PATCH 01/12] The types are happy, the tests are not --- README.md | 19 +-- package-lock.json | 181 +++------------------ package.json | 5 +- plugins/redis/redis_cache.ts | 32 ++-- plugins/redis/test_redis_cache.ts | 48 +++--- src/classes/caching/file_system_cache.ts | 13 +- src/classes/caching/memory_cache.ts | 26 ++- src/classes/response.ts | 29 ++-- src/helpers/cache_keys.ts | 13 +- src/helpers/cache_strategies.ts | 5 +- src/helpers/headers.ts | 5 +- src/helpers/shim_response_to_snipe_body.ts | 24 +-- src/index.ts | 22 +-- src/types.ts | 13 +- test/tests.cjs | 3 +- test/tests.ts | 43 +++-- 16 files changed, 146 insertions(+), 335 deletions(-) diff --git a/README.md b/README.md index f7678fd..3949e13 100644 --- a/README.md +++ b/README.md @@ -117,15 +117,15 @@ If none of the existing caching options meet your needs, you can implement your ```ts type INodeFetchCacheCache = { get(key: string): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; } | undefined>; set( key: string, - bodyStream: NodeJS.ReadableStream, + bodyStream: ReadableStream, metaData: NFCResponseMetadata ): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; }>; remove(key: string): Promise; @@ -163,19 +163,6 @@ if (response.isCacheMiss) { ## Advanced API -### Accessing Node-Fetch Exports - -If you need to access `node-fetch` exports (for example you might want to create a Request instance), you can do so by using the `getNodeFetch()` function: - -```js -import fetch, { getNodeFetch } from 'node-fetch-cache'; - -const { Request } = await getNodeFetch(); -const response = await fetch(new Request('https://google.com')); -``` - -You should not import from `node-fetch` directly since it is important that your code is using exports from the same version of `node-fetch` that is being used by `node-fetch-cache` internally. - ### Custom Cache Key Function You can provide custom cache key generation logic to node-fetch-cache by passing a `calculateCacheKey` option to `create()`: diff --git a/package-lock.json b/package-lock.json index 1fb314f..a45ac95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,7 @@ "dependencies": { "cacache": "^18.0.4", "formdata-node": "^6.0.3", - "locko": "^1.1.0", - "node-fetch": "3.3.2" + "locko": "^1.1.0" }, "devDependencies": { "@types/cacache": "^17.0.2", @@ -28,7 +27,7 @@ "rimraf": "^5.0.5", "rollup": "^4.9.1", "tsx": "^4.17.0", - "typescript": "^5.5.4" + "typescript": "^5.8.3" }, "engines": { "node": ">=18.19.0" @@ -972,12 +971,13 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", - "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", + "version": "22.14.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", + "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", "dev": true, + "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "node_modules/@types/semver": { @@ -1693,15 +1693,6 @@ "node": ">= 8" } }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -2062,29 +2053,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -2195,18 +2163,6 @@ "node": ">= 18" } }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -2928,43 +2884,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -3676,9 +3595,9 @@ } }, "node_modules/typescript": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", - "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -3690,10 +3609,11 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" }, "node_modules/unique-filename": { "version": "3.0.0", @@ -3747,15 +3667,6 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -4431,12 +4342,12 @@ "dev": true }, "@types/node": { - "version": "20.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", - "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", + "version": "22.14.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", + "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", "dev": true, "requires": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "@types/semver": { @@ -4916,11 +4827,6 @@ "which": "^2.0.1" } }, - "data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" - }, "debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -5196,15 +5102,6 @@ "reusify": "^1.0.4" } }, - "fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "requires": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - } - }, "file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -5286,14 +5183,6 @@ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==" }, - "formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "requires": { - "fetch-blob": "^3.1.2" - } - }, "fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -5815,21 +5704,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" - }, - "node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "requires": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - } - }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -6303,15 +6177,15 @@ "peer": true }, "typescript": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", - "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true }, "undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true }, "unique-filename": { @@ -6359,11 +6233,6 @@ } } }, - "web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==" - }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 9d9048b..780f401 100644 --- a/package.json +++ b/package.json @@ -52,13 +52,12 @@ "rimraf": "^5.0.5", "rollup": "^4.9.1", "tsx": "^4.17.0", - "typescript": "^5.5.4" + "typescript": "^5.8.3" }, "dependencies": { "cacache": "^18.0.4", "formdata-node": "^6.0.3", - "locko": "^1.1.0", - "node-fetch": "3.3.2" + "locko": "^1.1.0" }, "husky": { "hooks": { diff --git a/plugins/redis/redis_cache.ts b/plugins/redis/redis_cache.ts index bdd07ff..29289d7 100644 --- a/plugins/redis/redis_cache.ts +++ b/plugins/redis/redis_cache.ts @@ -1,5 +1,5 @@ +import { ReadableStream } from "stream/web"; import { Buffer } from 'buffer'; -import { Readable } from 'stream'; import Redis from 'ioredis'; import type { RedisOptions } from 'ioredis'; import type { INodeFetchCacheCache, NFCResponseMetadata } from 'node-fetch-cache'; @@ -35,7 +35,7 @@ export class RedisCache implements INodeFetchCacheCache { return undefined; } - const readableStream = Readable.from(cachedObjectInfo); + const readableStream = new Blob([cachedObjectInfo]).stream(); const storedMetadata = await this.redis.get(`${key}:meta`); if (!storedMetadata) { @@ -46,7 +46,7 @@ export class RedisCache implements INodeFetchCacheCache { const { expiration, ...nfcMetadata } = storedMetadataJson; return { - bodyStream: readableStream, + bodyStream: readableStream as Omit, "closed">, metaData: nfcMetadata, }; } @@ -57,7 +57,7 @@ export class RedisCache implements INodeFetchCacheCache { return true; } - async set(key: string, bodyStream: NodeJS.ReadableStream, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: Omit, metaData: NFCResponseMetadata) { const metaToStore = { ...metaData, expiration: undefined as undefined | number, @@ -67,24 +67,14 @@ export class RedisCache implements INodeFetchCacheCache { metaToStore.expiration = Date.now() + this.ttl; } - const buffer: Buffer = await new Promise((fulfill, reject) => { - const chunks: Buffer[] = []; + const buffer: Buffer = await new Promise(async (fulfill, reject) => { + const chunks = []; - bodyStream.on('data', chunk => { - chunks.push(chunk as Buffer); - }); + for await (const chunk of bodyStream) { + chunks.push(chunk); + } - bodyStream.on('end', async () => { - try { - fulfill(Buffer.concat(chunks)); - } catch (error) { - reject(error); - } - }); - - bodyStream.on('error', error => { - reject(error); - }); + fulfill(Buffer.concat(chunks)); }); await (typeof this.ttl === 'number' ? this.redis.set(key, buffer, 'PX', this.ttl) : this.redis.set(key, buffer)); @@ -94,7 +84,7 @@ export class RedisCache implements INodeFetchCacheCache { } return { - bodyStream: Readable.from(buffer), + bodyStream: new Blob([buffer]).stream() as Omit, "closed">, metaData: metaToStore, }; } diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index f0dd8b1..19e17c9 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -1,3 +1,4 @@ +import type { ReadableStream } from "stream/web"; // eslint-disable-next-line import/no-unassigned-import,import/order import 'dotenv/config.js'; import path, { dirname } from 'path'; @@ -7,7 +8,6 @@ import fs from 'fs'; import assert from 'assert'; import { Agent } from 'http'; import { FormData } from 'formdata-node'; -import standardFetch, { Request as StandardFetchRequest } from 'node-fetch'; import FetchCache, { cacheStrategies, FetchResource, @@ -84,10 +84,10 @@ function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) return arrayOrObject; } -async function dualFetch(...args: Parameters) { +async function dualFetch(...args: Parameters) { const [cachedFetchResponse, standardFetchResponse] = await Promise.all([ defaultCachedFetch(...args), - standardFetch(...args), + fetch(...args), ]); return { cachedFetchResponse, standardFetchResponse }; @@ -156,14 +156,14 @@ describe('REDIS Plugin Tests', function() { it('Gets correct raw headers', async () => { let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); cachedFetchResponse = await defaultCachedFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); }); @@ -367,7 +367,7 @@ describe('REDIS Plugin Tests', function() { it('Can use a client-provided custom cache key', async () => { const cacheFunction = async (resource: FetchResource) => { - if (resource instanceof StandardFetchRequest) { + if (resource instanceof Request) { return resource.url; } @@ -402,12 +402,12 @@ describe('REDIS Plugin Tests', function() { it('Can get PNG buffer body', async () => { defaultCachedFetch = FetchCache.create({ cache: new RedisCache(undefined, redisClient) }); response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); @@ -433,18 +433,18 @@ describe('REDIS Plugin Tests', function() { describe('REDIS Data tests', () => { it('Supports request objects', async () => { - let request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); + let request = new Request('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, false); - request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); + request = new Request('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, true); }); it('Supports request objects with custom headers', async () => { - const request1 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); - const request2 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); + const request1 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); + const request2 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); response = await defaultCachedFetch(request1); assert.strictEqual(response.returnedFromCache, false); @@ -456,7 +456,7 @@ describe('REDIS Plugin Tests', function() { it('Refuses to consume body twice', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); await response.text(); - await assert.rejects(async () => response.text(), /body used already for:/); + await assert.rejects(async () => response.text(), /Body has already been read/); }); it('Can get text body', async () => { @@ -485,12 +485,12 @@ describe('REDIS Plugin Tests', function() { it('Can get PNG buffer body', async () => { response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); @@ -527,7 +527,7 @@ describe('REDIS Plugin Tests', function() { it('Errors if the resource type is not supported', async () => { await assert.rejects( async () => defaultCachedFetch(1 as unknown as string), - /The first argument to fetch must be either a string or a node-fetch Request instance/, + /The first argument to fetch must be either a string or a fetch Request instance/, ); }); @@ -570,14 +570,14 @@ describe('REDIS Plugin Tests', function() { assert(initialResponse.ok); assert(!initialResponse.returnedFromCache); - const initialResponseBuffer = await initialResponse.buffer(); + const initialResponseBuffer = await initialResponse.arrayBuffer(); assert.equal(initialResponseBuffer.length, 100_000); const secondResponse = await defaultCachedFetch(HUNDRED_THOUSAND_BYTES_URL); assert(secondResponse.ok); assert(secondResponse.returnedFromCache); - const secondResponseBuffer = await secondResponse.buffer(); + const secondResponseBuffer = await secondResponse.arrayBuffer(); assert.equal(secondResponseBuffer.length, 100_000); }); }).timeout(10_000); @@ -599,20 +599,20 @@ describe('REDIS Plugin Tests', function() { it('Can use the only-if-cached cache control setting via resource', async () => { response = await defaultCachedFetch( - new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response.status === 504 && response.isCacheMiss); - response = await defaultCachedFetch(new StandardFetchRequest(TWO_HUNDRED_URL)); + response = await defaultCachedFetch(new Request(TWO_HUNDRED_URL)); assert(response && !response.returnedFromCache); response = await defaultCachedFetch( - new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response?.returnedFromCache); }); it('Works with only-if-cached along with other cache-control directives', async () => { response = await defaultCachedFetch( - new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), + new Request(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), ); assert(response.status === 504 && response.isCacheMiss); response = await defaultCachedFetch(TWO_HUNDRED_URL, { diff --git a/src/classes/caching/file_system_cache.ts b/src/classes/caching/file_system_cache.ts index 2cb9b80..52ee515 100644 --- a/src/classes/caching/file_system_cache.ts +++ b/src/classes/caching/file_system_cache.ts @@ -1,8 +1,9 @@ +import { ReadableStream } from "stream/web"; import assert from 'assert'; import { Buffer } from 'buffer'; -import { Readable } from 'stream'; import cacache from 'cacache'; import type { INodeFetchCacheCache, NFCResponseMetadata } from '../../types'; +import { Stream } from 'stream'; type StoredMetadata = { emptyBody?: boolean; @@ -36,13 +37,13 @@ export class FileSystemCache implements INodeFetchCacheCache { if (emptyBody) { return { - bodyStream: Readable.from(emptyBuffer), + bodyStream: ReadableStream.from(emptyBuffer), metaData: storedMetadata, }; } return { - bodyStream: cacache.get.stream.byDigest(this.cacheDirectory, cachedObjectInfo.integrity), + bodyStream: ReadableStream.from(cacache.get.stream.byDigest(this.cacheDirectory, cachedObjectInfo.integrity)), metaData: nfcMetadata, }; } @@ -51,7 +52,7 @@ export class FileSystemCache implements INodeFetchCacheCache { return cacache.rm.entry(this.cacheDirectory, key); } - async set(key: string, bodyStream: NodeJS.ReadableStream, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: Omit, metaData: NFCResponseMetadata) { const metaToStore = { ...metaData, expiration: undefined as (undefined | number), @@ -73,11 +74,11 @@ export class FileSystemCache implements INodeFetchCacheCache { private async writeDataToCache( key: string, storedMetadata: StoredMetadata, - stream: NodeJS.ReadableStream, + stream: ReadableStream, ) { try { await new Promise((fulfill, reject) => { - stream.pipe(cacache.put.stream(this.cacheDirectory, key, { metadata: storedMetadata })) + Stream.Readable.fromWeb(stream).pipe(cacache.put.stream(this.cacheDirectory, key, { metadata: storedMetadata })) .on('integrity', (i: string) => { fulfill(i); }) diff --git a/src/classes/caching/memory_cache.ts b/src/classes/caching/memory_cache.ts index d5c2a73..ace61bf 100644 --- a/src/classes/caching/memory_cache.ts +++ b/src/classes/caching/memory_cache.ts @@ -1,20 +1,16 @@ +import { ReadableStream } from "stream/web"; import assert from 'assert'; -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; import type { INodeFetchCacheCache, NFCResponseMetadata } from '../../types.js'; import { KeyTimeout } from './key_timeout.js'; -async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { - const chunks: Buffer[] = []; - return new Promise((resolve, reject) => { - stream.on('data', chunk => - chunks.push(chunk as Buffer), - ).on('error', error => { - reject(error); - }).on('end', () => { - resolve(Buffer.concat(chunks)); - }); - }); +async function streamToBuffer(stream: ReadableStream): Promise { + const chunks = []; + + for await (const chunk of stream) { + chunks.push(chunk); + } + + return Buffer.concat(chunks); } export class MemoryCache implements INodeFetchCacheCache { @@ -30,7 +26,7 @@ export class MemoryCache implements INodeFetchCacheCache { const cachedValue = this.cache.get(key); if (cachedValue) { return { - bodyStream: Readable.from(cachedValue.bodyBuffer), + bodyStream: new Blob([cachedValue.bodyBuffer]).stream() as Omit, "closed">, metaData: cachedValue.metaData, }; } @@ -43,7 +39,7 @@ export class MemoryCache implements INodeFetchCacheCache { this.cache.delete(key); } - async set(key: string, bodyStream: NodeJS.ReadableStream, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: Omit, metaData: NFCResponseMetadata) { const bodyBuffer = await streamToBuffer(bodyStream); this.cache.set(key, { bodyBuffer, metaData }); diff --git a/src/classes/response.ts b/src/classes/response.ts index d38b5a1..0bdb4f7 100644 --- a/src/classes/response.ts +++ b/src/classes/response.ts @@ -1,24 +1,19 @@ +import type { ReadableStream } from "stream/web"; import assert from 'assert'; -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; -import type { Response as NodeFetchResponseType, ResponseInit as NodeFetchResponseInit } from 'node-fetch'; import { NFCResponseMetadata } from '../types.js'; -import { getNodeFetch } from '../helpers/node_fetch_imports.js'; async function createNFCResponseClass() { - const { NodeFetchResponse } = await getNodeFetch(); - - const responseInternalSymbol = Object.getOwnPropertySymbols(new NodeFetchResponse())[1]; + const responseInternalSymbol = Object.getOwnPropertySymbols(new Response())[1]; assert(responseInternalSymbol, 'Failed to get node-fetch responseInternalSymbol'); - return class NFCResponse extends NodeFetchResponse { - static serializeMetaFromNodeFetchResponse(response: NodeFetchResponseType): NFCResponseMetadata { + return class NFCResponse extends Response { + static serializeMetaFromNodeFetchResponse(response: Response): NFCResponseMetadata { const metaData = { url: response.url, status: response.status, statusText: response.statusText, - headers: response.headers.raw(), - size: response.size, + headers: Array.from(response.headers.entries()).reduce>((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {}), + //size: response.size, counter: (response as any)[responseInternalSymbol!].counter as number, }; @@ -29,13 +24,13 @@ async function createNFCResponseClass() { url: string, ) { return new NFCResponse( - Readable.from(Buffer.alloc(0)), + new Blob().stream() as Omit, "closed">, { url, status: 504, statusText: 'Gateway Timeout', headers: {}, - size: 0, + //size: 0, counter: 0, }, async () => undefined, @@ -45,10 +40,10 @@ async function createNFCResponseClass() { } constructor( - bodyStream: NodeJS.ReadableStream, - metaData: Omit & { + bodyStream: Omit, + metaData: Omit & { url: string; - size: number; + //size: number; counter: number; headers: Record; }, @@ -57,7 +52,7 @@ async function createNFCResponseClass() { public readonly isCacheMiss = false, ) { super( - Readable.from(bodyStream), + bodyStream, metaData as any, // eslint-disable-line @typescript-eslint/no-unsafe-argument ); } diff --git a/src/helpers/cache_keys.ts b/src/helpers/cache_keys.ts index 287198d..a63d0cf 100644 --- a/src/helpers/cache_keys.ts +++ b/src/helpers/cache_keys.ts @@ -2,10 +2,8 @@ import fs from 'fs'; import crypto from 'crypto'; import assert from 'assert'; import { Buffer } from 'buffer'; -import type { Request as NodeFetchRequestType } from 'node-fetch'; import type { FetchInit, FetchResource } from '../types.js'; import { FormData } from '../types.js'; -import { getNodeFetch } from './node_fetch_imports.js'; export const CACHE_VERSION = 6; @@ -57,9 +55,8 @@ function getBodyCacheKeyJson(body: unknown): string | object | undefined { throw new Error('Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, fs.ReadStream, FormData'); } -async function getRequestCacheKeyJson(request: NodeFetchRequestType) { - const { NodeFetchRequest } = await getNodeFetch(); - const bodyInternalsSymbol = Object.getOwnPropertySymbols(new NodeFetchRequest('http://url.com'))[0]; +async function getRequestCacheKeyJson(request: Request) { + const bodyInternalsSymbol = Object.getOwnPropertySymbols(new Request('http://url.com'))[0]; assert(bodyInternalsSymbol, 'Failed to get node-fetch bodyInternalsSymbol'); return { @@ -73,13 +70,12 @@ async function getRequestCacheKeyJson(request: NodeFetchRequestType) { follow: (request as any).follow, // eslint-disable-line @typescript-eslint/no-unsafe-assignment // Confirmed that this property exists, but it's not in the types compress: (request as any).compress, // eslint-disable-line @typescript-eslint/no-unsafe-assignment - size: request.size, + //size: request.size, }; } export async function calculateCacheKey(resource: FetchResource, init?: FetchInit) { - const { NodeFetchRequest } = await getNodeFetch(); - const resourceCacheKeyJson = resource instanceof NodeFetchRequest + const resourceCacheKeyJson = resource instanceof Request ? await getRequestCacheKeyJson(resource) : { url: resource, body: undefined }; @@ -92,6 +88,7 @@ export async function calculateCacheKey(resource: FetchResource, init?: FetchIni resourceCacheKeyJson.body = getBodyCacheKeyJson(resourceCacheKeyJson.body); initCacheKeyJson.body = getBodyCacheKeyJson(initCacheKeyJson.body); + // @ts-expect-error delete initCacheKeyJson.agent; return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION])); diff --git a/src/helpers/cache_strategies.ts b/src/helpers/cache_strategies.ts index 4b9683b..9073a72 100644 --- a/src/helpers/cache_strategies.ts +++ b/src/helpers/cache_strategies.ts @@ -1,5 +1,4 @@ -import type { Response } from 'node-fetch'; import type { CacheStrategy } from '../types.js'; -export const cacheOkayOnly: CacheStrategy = (response: Response) => response.ok; -export const cacheNon5xxOnly: CacheStrategy = (response: Response) => response.status < 500; +export const cacheOkayOnly: CacheStrategy = async (response: Response) => response.ok; +export const cacheNon5xxOnly: CacheStrategy = async (response: Response) => response.status < 500; diff --git a/src/helpers/headers.ts b/src/helpers/headers.ts index 0a17dba..729cd4c 100644 --- a/src/helpers/headers.ts +++ b/src/helpers/headers.ts @@ -1,5 +1,4 @@ import { FetchInit, FetchResource } from '../types.js'; -import { getNodeFetch } from './node_fetch_imports.js'; function headerKeyIsCacheControl(key: string) { return key.trim().toLowerCase() === 'cache-control'; @@ -26,10 +25,8 @@ export async function hasOnlyIfCachedOption(resource: FetchResource, init: Fetch return true; } - const { NodeFetchRequest } = await getNodeFetch(); - if ( - resource instanceof NodeFetchRequest + resource instanceof Request && headerValueContainsOnlyIfCached(resource.headers.get('Cache-Control') ?? undefined) ) { return true; diff --git a/src/helpers/shim_response_to_snipe_body.ts b/src/helpers/shim_response_to_snipe_body.ts index 42be9b0..7558530 100644 --- a/src/helpers/shim_response_to_snipe_body.ts +++ b/src/helpers/shim_response_to_snipe_body.ts @@ -1,7 +1,3 @@ -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; -import type { Response as NodeFetchResponse } from 'node-fetch'; - /* This is a bit of a hack to deal with the case when the user * consumes the response body in their `shouldCacheResponse` delegate. * The response body can only be consumed once, so if the user consumes @@ -12,41 +8,35 @@ import type { Response as NodeFetchResponse } from 'node-fetch'; * My initial inclination was to use Response.prototype.clone() for this, * but the problems with backpressure seem significant. */ export function shimResponseToSnipeBody( - response: NodeFetchResponse, - replaceBodyStream: (strean: NodeJS.ReadableStream) => void, + response: Response, + replaceBodyStream: (stream: Omit) => void, ) { const origArrayBuffer = response.arrayBuffer; response.arrayBuffer = async function () { const arrayBuffer = await origArrayBuffer.call(this); - replaceBodyStream(Readable.from(Buffer.from(arrayBuffer))); + replaceBodyStream(new Blob([arrayBuffer]).stream()); return arrayBuffer; }; - const origBuffer = response.buffer; - response.buffer = async function () { - const buffer = await origBuffer.call(this); - replaceBodyStream(Readable.from(buffer)); - return buffer; - }; - const origJson = response.json; response.json = async function () { const json = await origJson.call(this); - replaceBodyStream(Readable.from(Buffer.from(JSON.stringify(json)))); + replaceBodyStream(new Blob([JSON.stringify(json)]).stream()); return json; }; const origText = response.text; response.text = async function () { const text = await origText.call(this); - replaceBodyStream(Readable.from(Buffer.from(text))); + replaceBodyStream(new Blob([text]).stream()); return text; }; const origBlob = response.blob; response.blob = async function () { const blob = await origBlob.call(this); - replaceBodyStream(Readable.from(Buffer.from(await blob.text()))); + const text = await blob.text(); + replaceBodyStream(new Blob([text]).stream()); return blob; }; } diff --git a/src/index.ts b/src/index.ts index 02fbcb8..5683e7a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import type { Request as NodeFetchRequestType } from 'node-fetch'; +import { ReadableStream } from "stream/web"; import assert from 'assert'; import { FormData } from 'formdata-node'; import { getNFCResponseClass as getNFCResponseClass } from './classes/response.js'; @@ -15,7 +15,6 @@ import type { INodeFetchCacheCache, ISynchronizationStrategy, } from './types.js'; -import { getNodeFetch } from './helpers/node_fetch_imports.js'; type CacheKeyCalculator = typeof calculateCacheKey; @@ -28,10 +27,8 @@ type NFCCustomizations = { type NFCOptions = Partial; -async function getUrlFromRequestArguments(resource: NodeFetchRequestType | string) { - const { NodeFetchRequest } = await getNodeFetch(); - - if (resource instanceof NodeFetchRequest) { +async function getUrlFromRequestArguments(resource: Request | string) { + if (resource instanceof Request) { return resource.url; } @@ -43,12 +40,11 @@ async function getResponse( resource: FetchResource, init: FetchInit, ) { - const { NodeFetchRequest, fetch } = await getNodeFetch(); const NFCResponse = await getNFCResponseClass(); - if (typeof resource !== 'string' && !(resource instanceof NodeFetchRequest)) { + if (typeof resource !== 'string' && !(resource instanceof Request)) { throw new TypeError( - 'The first argument to fetch must be either a string or a node-fetch Request instance', + 'The first argument to fetch must be either a string or a fetch Request instance', ); } @@ -96,7 +92,7 @@ async function getResponse( if (shouldCache) { const cacheSetResult = await fetchCustomization.cache.set( cacheKey, - bodyStream, + bodyStream as Omit, "closed">, serializedMeta, ); @@ -104,7 +100,7 @@ async function getResponse( } return new NFCResponse( - bodyStream, + bodyStream as Omit, "closed">, serializedMeta, ejectSelfFromCache, false, @@ -118,7 +114,7 @@ function create(creationOptions: NFCOptions) { const fetchOptions: NFCCustomizations = { cache: creationOptions.cache ?? globalMemoryCache, synchronizationStrategy: creationOptions.synchronizationStrategy ?? new LockoSynchronizationStrategy(), - shouldCacheResponse: creationOptions.shouldCacheResponse ?? (() => true), + shouldCacheResponse: creationOptions.shouldCacheResponse ?? (() => Promise.resolve(true)), calculateCacheKey: creationOptions.calculateCacheKey ?? calculateCacheKey, }; @@ -148,7 +144,6 @@ export default defaultFetch; export { MemoryCache } from './classes/caching/memory_cache.js'; export { FileSystemCache } from './classes/caching/file_system_cache.js'; export { CACHE_VERSION } from './helpers/cache_keys.js'; -export { getNodeFetch }; export type { NFCResponse } from './classes/response.js'; export type { NFCResponseMetadata } from './types.js'; export { @@ -157,7 +152,6 @@ export { calculateCacheKey as getCacheKey, calculateCacheKey, FormData, - type NodeFetchRequestType as NodeFetchRequest, type NFCOptions, type CacheKeyCalculator, type INodeFetchCacheCache, diff --git a/src/types.ts b/src/types.ts index 68d6209..5e03f40 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,10 +1,9 @@ -import type { Response as NodeFetchResponse } from 'node-fetch'; -import type fetch from 'node-fetch'; +import type { ReadableStream } from "stream/web"; import { FormData } from 'formdata-node'; export type FetchResource = Parameters[0]; export type FetchInit = Parameters[1]; -export type CacheStrategy = (response: NodeFetchResponse) => Promise | boolean; +export type CacheStrategy = (response: Response) => Promise; export { FormData }; @@ -13,21 +12,21 @@ export type NFCResponseMetadata = { status: number; statusText: string; headers: Record; - size: number; + //size: number; counter: number; }; export type INodeFetchCacheCache = { get(key: string): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: Omit; metaData: NFCResponseMetadata; } | undefined>; set( key: string, - bodyStream: NodeJS.ReadableStream, + bodyStream: Omit, metaData: NFCResponseMetadata ): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: Omit; metaData: NFCResponseMetadata; }>; remove(key: string): Promise; diff --git a/test/tests.cjs b/test/tests.cjs index 58dac42..9ee6d32 100644 --- a/test/tests.cjs +++ b/test/tests.cjs @@ -12,8 +12,7 @@ describe('Commonjs module tests', () => { }); it('Can make a request via Request object', async () => { - const nodeFetch = await fetch.getNodeFetch(); - const res = await fetch(new nodeFetch.Request(TWO_HUNDRED_URL)); + const res = await fetch(new Request(TWO_HUNDRED_URL)); assert.strictEqual(res.status, 200); }); diff --git a/test/tests.ts b/test/tests.ts index b07a1f0..660abc8 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -8,7 +8,6 @@ import assert from 'assert'; import { Agent } from 'http'; import { rimraf } from 'rimraf'; import { FormData } from 'formdata-node'; -import standardFetch, { Request as StandardFetchRequest } from 'node-fetch'; import FetchCache, { MemoryCache, FileSystemCache, @@ -84,10 +83,10 @@ function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) return arrayOrObject; } -async function dualFetch(...args: Parameters) { +async function dualFetch(...args: Parameters) { const [cachedFetchResponse, standardFetchResponse] = await Promise.all([ defaultCachedFetch(...args), - standardFetch(...args), + fetch(...args), ]); return { cachedFetchResponse, standardFetchResponse }; @@ -149,14 +148,14 @@ describe('Header tests', () => { it('Gets correct raw headers', async () => { let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); cachedFetchResponse = await defaultCachedFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); }); @@ -380,7 +379,7 @@ describe('Cache tests', () => { it('Can use a client-provided custom cache key', async () => { const cacheFunction = async (resource: FetchResource) => { - if (resource instanceof StandardFetchRequest) { + if (resource instanceof Request) { return resource.url; } @@ -401,18 +400,18 @@ describe('Cache tests', () => { describe('Data tests', () => { it('Supports request objects', async () => { - let request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); + let request = new Request('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, false); - request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); + request = new Request('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, true); }); it('Supports request objects with custom headers', async () => { - const request1 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); - const request2 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); + const request1 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); + const request2 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); response = await defaultCachedFetch(request1); assert.strictEqual(response.returnedFromCache, false); @@ -424,7 +423,7 @@ describe('Data tests', () => { it('Refuses to consume body twice', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); await response.text(); - await assert.rejects(async () => response.text(), /body used already for:/); + await assert.rejects(async () => response.text(), /Body has already been read/); }); it('Can get text body', async () => { @@ -453,12 +452,12 @@ describe('Data tests', () => { it('Can get PNG buffer body', async () => { response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); @@ -495,7 +494,7 @@ describe('Data tests', () => { it('Errors if the resource type is not supported', async () => { await assert.rejects( async () => defaultCachedFetch(1 as unknown as string), - /The first argument to fetch must be either a string or a node-fetch Request instance/, + /The first argument to fetch must be either a string or a fetch Request instance/, ); }); @@ -582,12 +581,12 @@ describe('File system cache tests', () => { it('Can get PNG buffer body', async () => { defaultCachedFetch = FetchCache.create({ cache: new FileSystemCache() }); response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); @@ -628,20 +627,20 @@ describe('Cache mode tests', () => { it('Can use the only-if-cached cache control setting via resource', async () => { response = await defaultCachedFetch( - new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response.status === 504 && response.isCacheMiss); - response = await defaultCachedFetch(new StandardFetchRequest(TWO_HUNDRED_URL)); + response = await defaultCachedFetch(new Request(TWO_HUNDRED_URL)); assert(response && !response.returnedFromCache); response = await defaultCachedFetch( - new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response?.returnedFromCache); }); it('Works with only-if-cached along with other cache-control directives', async () => { response = await defaultCachedFetch( - new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), + new Request(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), ); assert(response.status === 504 && response.isCacheMiss); response = await defaultCachedFetch(TWO_HUNDRED_URL, { From 672d5b6bd6d64c9d74ccd49989df7d2471e96a30 Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Thu, 24 Apr 2025 17:29:46 -0400 Subject: [PATCH 02/12] 94 passing, 5 failing --- plugins/redis/redis_cache.ts | 1 - plugins/redis/test_redis_cache.ts | 23 +++++++++---------- src/classes/caching/file_system_cache.ts | 3 +-- src/helpers/cache_keys.ts | 28 ++++++++++-------------- src/index.ts | 10 +++++++++ test/tests.ts | 19 ++++++++-------- 6 files changed, 44 insertions(+), 40 deletions(-) diff --git a/plugins/redis/redis_cache.ts b/plugins/redis/redis_cache.ts index 29289d7..626bf2c 100644 --- a/plugins/redis/redis_cache.ts +++ b/plugins/redis/redis_cache.ts @@ -1,5 +1,4 @@ import { ReadableStream } from "stream/web"; -import { Buffer } from 'buffer'; import Redis from 'ioredis'; import type { RedisOptions } from 'ioredis'; import type { INodeFetchCacheCache, NFCResponseMetadata } from 'node-fetch-cache'; diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index 19e17c9..d96a662 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -1,4 +1,3 @@ -import type { ReadableStream } from "stream/web"; // eslint-disable-next-line import/no-unassigned-import,import/order import 'dotenv/config.js'; import path, { dirname } from 'path'; @@ -308,7 +307,8 @@ describe('REDIS Plugin Tests', function() { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a'))); assert.strictEqual(response.returnedFromCache, true); }); - + + /* it('Gives different read streams different cache keys', async () => { const s1 = fs.createReadStream(path.join(__dirname, '..', '..', 'test', 'expected_png.png')); const s2 = fs.createReadStream(path.join(__dirname, '..', '..', 'test', '..', 'src', 'index.ts')); @@ -329,7 +329,8 @@ describe('REDIS Plugin Tests', function() { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(s1)); assert.strictEqual(response.returnedFromCache, true); }); - + */ + it('Gives different form data different cache keys', async () => { const data1 = new FormData(); data1.append('a', 'a'); @@ -497,23 +498,23 @@ describe('REDIS Plugin Tests', function() { it('Can stream a body', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); - let body = ''; + let chunks = []; for await (const chunk of response.body ?? []) { - body += chunk.toString(); + chunks.push(chunk); } - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(TEXT_BODY_URL); - body = ''; + chunks = []; for await (const chunk of response.body ?? []) { - body += chunk.toString(); + chunks.push(chunk); } - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); assert.strictEqual(response.returnedFromCache, true); }); @@ -715,7 +716,7 @@ describe('REDIS Plugin Tests', function() { }); it('Can use a custom cache strategy that uses the response for all response types', async () => { - const functionsThatUseResponse = ['arrayBuffer', 'blob', 'buffer', 'json', 'text'] as const; + const functionsThatUseResponse = ['arrayBuffer', 'blob', 'json', 'text'] as const; for (const functionName of functionsThatUseResponse) { await redisClient.flushall(); @@ -745,7 +746,7 @@ describe('REDIS Plugin Tests', function() { describe('REDIS Network error tests', () => { it('Bubbles up network errors', async () => { - await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /^FetchError:/); + await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /TypeError: fetch failed/); }); }); }); diff --git a/src/classes/caching/file_system_cache.ts b/src/classes/caching/file_system_cache.ts index 52ee515..3f78d53 100644 --- a/src/classes/caching/file_system_cache.ts +++ b/src/classes/caching/file_system_cache.ts @@ -1,6 +1,5 @@ import { ReadableStream } from "stream/web"; import assert from 'assert'; -import { Buffer } from 'buffer'; import cacache from 'cacache'; import type { INodeFetchCacheCache, NFCResponseMetadata } from '../../types'; import { Stream } from 'stream'; @@ -10,7 +9,7 @@ type StoredMetadata = { expiration?: number | undefined; } & NFCResponseMetadata; -const emptyBuffer = Buffer.alloc(0); +const emptyBuffer = Buffer.from([]) export class FileSystemCache implements INodeFetchCacheCache { private readonly ttl?: number | undefined; diff --git a/src/helpers/cache_keys.ts b/src/helpers/cache_keys.ts index a63d0cf..e6ab78a 100644 --- a/src/helpers/cache_keys.ts +++ b/src/helpers/cache_keys.ts @@ -1,14 +1,13 @@ -import fs from 'fs'; -import crypto from 'crypto'; -import assert from 'assert'; -import { Buffer } from 'buffer'; import type { FetchInit, FetchResource } from '../types.js'; import { FormData } from '../types.js'; export const CACHE_VERSION = 6; -function md5(string_: string) { - return crypto.createHash('md5').update(string_).digest('hex'); +async function sha1(string_: string) { + return Array.from( + new Uint8Array(await crypto.subtle.digest("SHA-1", new TextEncoder().encode(string_))), + (byte) => byte.toString(16).padStart(2, "0") + ).join(""); } function getFormDataCacheKeyJson(formData: FormData) { @@ -40,32 +39,27 @@ function getBodyCacheKeyJson(body: unknown): string | object | undefined { return body.toString(); } - if (body instanceof fs.ReadStream) { - return body.path.toString(); - } - if (body instanceof FormData) { return getFormDataCacheKeyJson(body); } - if (body instanceof Buffer) { + if (body instanceof ArrayBuffer) { return body.toString(); } - throw new Error('Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, fs.ReadStream, FormData'); + throw new Error("Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, FormData"); } async function getRequestCacheKeyJson(request: Request) { - const bodyInternalsSymbol = Object.getOwnPropertySymbols(new Request('http://url.com'))[0]; - assert(bodyInternalsSymbol, 'Failed to get node-fetch bodyInternalsSymbol'); + const body = await request.arrayBuffer(); return { - headers: getHeadersCacheKeyJson([...request.headers.entries()]), + headers: getHeadersCacheKeyJson(Array.from(request.headers.entries())), method: request.method, redirect: request.redirect, referrer: request.referrer, url: request.url, - body: getBodyCacheKeyJson((request as any)[bodyInternalsSymbol!].body), + body: getBodyCacheKeyJson(body), // Confirmed that this property exists, but it's not in the types follow: (request as any).follow, // eslint-disable-line @typescript-eslint/no-unsafe-assignment // Confirmed that this property exists, but it's not in the types @@ -91,5 +85,5 @@ export async function calculateCacheKey(resource: FetchResource, init?: FetchIni // @ts-expect-error delete initCacheKeyJson.agent; - return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION])); + return sha1(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION])); } diff --git a/src/index.ts b/src/index.ts index 5683e7a..b132077 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,6 +40,8 @@ async function getResponse( resource: FetchResource, init: FetchInit, ) { + const originalResource = resource; + const NFCResponse = await getNFCResponseClass(); if (typeof resource !== 'string' && !(resource instanceof Request)) { @@ -48,6 +50,10 @@ async function getResponse( ); } + if (originalResource instanceof Request) { + resource = originalResource.clone() + } + const cacheKey = await fetchCustomization.calculateCacheKey(resource, init); const ejectSelfFromCache = async () => fetchCustomization.cache.remove(cacheKey); @@ -78,6 +84,10 @@ async function getResponse( ); } + if (originalResource instanceof Request) { + resource = originalResource.clone() + } + const fetchResponse = await fetch(resource, init); const serializedMeta = NFCResponse.serializeMetaFromNodeFetchResponse(fetchResponse); let bodyStream = fetchResponse.body; diff --git a/test/tests.ts b/test/tests.ts index 660abc8..a4cbd10 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -301,6 +301,7 @@ describe('Cache tests', () => { assert.strictEqual(response.returnedFromCache, true); }); + /* it('Gives different read streams different cache keys', async () => { const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png')); const s2 = fs.createReadStream(path.join(__dirname, '..', 'src', 'index.ts')); @@ -321,7 +322,8 @@ describe('Cache tests', () => { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(s1)); assert.strictEqual(response.returnedFromCache, true); }); - + */ + it('Gives different form data different cache keys', async () => { const data1 = new FormData(); data1.append('a', 'a'); @@ -464,23 +466,23 @@ describe('Data tests', () => { it('Can stream a body', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); - let body = ''; + let chunks = []; for await (const chunk of response.body!) { - body += chunk.toString(); + chunks.push(chunk) } - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(TEXT_BODY_URL); - body = ''; + chunks = []; for await (const chunk of response.body!) { - body += chunk.toString(); + chunks.push(chunk) } - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); assert.strictEqual(response.returnedFromCache, true); }); @@ -746,7 +748,6 @@ describe('Cache strategy tests', () => { const functionsThatUseResponse = [ 'arrayBuffer', 'blob', - 'buffer', 'json', 'text', ] as const; @@ -783,6 +784,6 @@ describe('Cache strategy tests', () => { describe('Network error tests', () => { it('Bubbles up network errors', async () => { - await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /^FetchError:/); + await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /TypeError: fetch failed/); }); }); From 310b978aba23494d4d9ad5420d4bd566d15d9d63 Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Thu, 24 Apr 2025 22:20:55 -0400 Subject: [PATCH 03/12] Update test_redis_cache.ts --- plugins/redis/test_redis_cache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index d96a662..1903654 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -572,14 +572,14 @@ describe('REDIS Plugin Tests', function() { assert(!initialResponse.returnedFromCache); const initialResponseBuffer = await initialResponse.arrayBuffer(); - assert.equal(initialResponseBuffer.length, 100_000); + assert.equal(initialResponseBuffer.byteLength, 100_000); const secondResponse = await defaultCachedFetch(HUNDRED_THOUSAND_BYTES_URL); assert(secondResponse.ok); assert(secondResponse.returnedFromCache); const secondResponseBuffer = await secondResponse.arrayBuffer(); - assert.equal(secondResponseBuffer.length, 100_000); + assert.equal(secondResponseBuffer.byteLength, 100_000); }); }).timeout(10_000); From e19ceb95ec960dab6f8d0565380540118d8babdd Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Fri, 25 Apr 2025 10:36:13 -0400 Subject: [PATCH 04/12] `TypeError: RequestInit: duplex option is required when sending a body.` --- plugins/redis/redis_cache.ts | 6 +++--- plugins/redis/test_redis_cache.ts | 22 +++++++++++----------- src/classes/caching/file_system_cache.ts | 2 +- src/classes/caching/memory_cache.ts | 4 ++-- src/classes/response.ts | 15 +++++++++------ src/helpers/cache_keys.ts | 10 +++++++--- src/helpers/node_fetch_imports.ts | 5 ----- src/helpers/shim_response_to_snipe_body.ts | 2 +- src/index.ts | 4 ++-- src/types.ts | 7 +++---- test.ts | 9 +++++++++ test/tests.ts | 2 -- 12 files changed, 48 insertions(+), 40 deletions(-) delete mode 100644 src/helpers/node_fetch_imports.ts create mode 100644 test.ts diff --git a/plugins/redis/redis_cache.ts b/plugins/redis/redis_cache.ts index 626bf2c..e1f0178 100644 --- a/plugins/redis/redis_cache.ts +++ b/plugins/redis/redis_cache.ts @@ -45,7 +45,7 @@ export class RedisCache implements INodeFetchCacheCache { const { expiration, ...nfcMetadata } = storedMetadataJson; return { - bodyStream: readableStream as Omit, "closed">, + bodyStream: readableStream as ReadableStream, metaData: nfcMetadata, }; } @@ -56,7 +56,7 @@ export class RedisCache implements INodeFetchCacheCache { return true; } - async set(key: string, bodyStream: Omit, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { const metaToStore = { ...metaData, expiration: undefined as undefined | number, @@ -83,7 +83,7 @@ export class RedisCache implements INodeFetchCacheCache { } return { - bodyStream: new Blob([buffer]).stream() as Omit, "closed">, + bodyStream: new Blob([buffer]).stream() as ReadableStream, metaData: metaToStore, }; } diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index 1903654..752fea7 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -17,6 +17,8 @@ import FetchCache, { import { RedisCache } from './redis_cache.js'; import { Redis } from 'ioredis'; +const StandardFetchRequest = Request; + const MIN_NODE_VERSION = 16; const httpBinBaseUrl = 'http://localhost:3000'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -308,7 +310,6 @@ describe('REDIS Plugin Tests', function() { assert.strictEqual(response.returnedFromCache, true); }); - /* it('Gives different read streams different cache keys', async () => { const s1 = fs.createReadStream(path.join(__dirname, '..', '..', 'test', 'expected_png.png')); const s2 = fs.createReadStream(path.join(__dirname, '..', '..', 'test', '..', 'src', 'index.ts')); @@ -329,7 +330,6 @@ describe('REDIS Plugin Tests', function() { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(s1)); assert.strictEqual(response.returnedFromCache, true); }); - */ it('Gives different form data different cache keys', async () => { const data1 = new FormData(); @@ -368,7 +368,7 @@ describe('REDIS Plugin Tests', function() { it('Can use a client-provided custom cache key', async () => { const cacheFunction = async (resource: FetchResource) => { - if (resource instanceof Request) { + if (resource instanceof StandardFetchRequest) { return resource.url; } @@ -434,18 +434,18 @@ describe('REDIS Plugin Tests', function() { describe('REDIS Data tests', () => { it('Supports request objects', async () => { - let request = new Request('https://google.com', { body: 'test', method: 'POST' }); + let request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, false); - request = new Request('https://google.com', { body: 'test', method: 'POST' }); + request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, true); }); it('Supports request objects with custom headers', async () => { - const request1 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); - const request2 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); + const request1 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); + const request2 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); response = await defaultCachedFetch(request1); assert.strictEqual(response.returnedFromCache, false); @@ -600,20 +600,20 @@ describe('REDIS Plugin Tests', function() { it('Can use the only-if-cached cache control setting via resource', async () => { response = await defaultCachedFetch( - new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response.status === 504 && response.isCacheMiss); - response = await defaultCachedFetch(new Request(TWO_HUNDRED_URL)); + response = await defaultCachedFetch(new StandardFetchRequest(TWO_HUNDRED_URL)); assert(response && !response.returnedFromCache); response = await defaultCachedFetch( - new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response?.returnedFromCache); }); it('Works with only-if-cached along with other cache-control directives', async () => { response = await defaultCachedFetch( - new Request(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), + new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), ); assert(response.status === 504 && response.isCacheMiss); response = await defaultCachedFetch(TWO_HUNDRED_URL, { diff --git a/src/classes/caching/file_system_cache.ts b/src/classes/caching/file_system_cache.ts index 3f78d53..c90e45f 100644 --- a/src/classes/caching/file_system_cache.ts +++ b/src/classes/caching/file_system_cache.ts @@ -51,7 +51,7 @@ export class FileSystemCache implements INodeFetchCacheCache { return cacache.rm.entry(this.cacheDirectory, key); } - async set(key: string, bodyStream: Omit, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { const metaToStore = { ...metaData, expiration: undefined as (undefined | number), diff --git a/src/classes/caching/memory_cache.ts b/src/classes/caching/memory_cache.ts index ace61bf..240e84e 100644 --- a/src/classes/caching/memory_cache.ts +++ b/src/classes/caching/memory_cache.ts @@ -26,7 +26,7 @@ export class MemoryCache implements INodeFetchCacheCache { const cachedValue = this.cache.get(key); if (cachedValue) { return { - bodyStream: new Blob([cachedValue.bodyBuffer]).stream() as Omit, "closed">, + bodyStream: new Blob([cachedValue.bodyBuffer]).stream() as ReadableStream, metaData: cachedValue.metaData, }; } @@ -39,7 +39,7 @@ export class MemoryCache implements INodeFetchCacheCache { this.cache.delete(key); } - async set(key: string, bodyStream: Omit, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { const bodyBuffer = await streamToBuffer(bodyStream); this.cache.set(key, { bodyBuffer, metaData }); diff --git a/src/classes/response.ts b/src/classes/response.ts index 0bdb4f7..4ccf80c 100644 --- a/src/classes/response.ts +++ b/src/classes/response.ts @@ -8,12 +8,17 @@ async function createNFCResponseClass() { return class NFCResponse extends Response { static serializeMetaFromNodeFetchResponse(response: Response): NFCResponseMetadata { + const headers = Array.from(response.headers.entries()).reduce>(function(headers, [key, value]) { + headers[key] = [...(headers[key] ?? []), value]; + + return headers; + }, {}) + const metaData = { url: response.url, status: response.status, statusText: response.statusText, - headers: Array.from(response.headers.entries()).reduce>((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {}), - //size: response.size, + headers: headers, counter: (response as any)[responseInternalSymbol!].counter as number, }; @@ -24,13 +29,12 @@ async function createNFCResponseClass() { url: string, ) { return new NFCResponse( - new Blob().stream() as Omit, "closed">, + new Blob().stream() as ReadableStream, { url, status: 504, statusText: 'Gateway Timeout', headers: {}, - //size: 0, counter: 0, }, async () => undefined, @@ -40,10 +44,9 @@ async function createNFCResponseClass() { } constructor( - bodyStream: Omit, + bodyStream: ReadableStream, metaData: Omit & { url: string; - //size: number; counter: number; headers: Record; }, diff --git a/src/helpers/cache_keys.ts b/src/helpers/cache_keys.ts index e6ab78a..283b5b3 100644 --- a/src/helpers/cache_keys.ts +++ b/src/helpers/cache_keys.ts @@ -1,5 +1,6 @@ import type { FetchInit, FetchResource } from '../types.js'; import { FormData } from '../types.js'; +import * as fs from "fs"; export const CACHE_VERSION = 6; @@ -39,6 +40,10 @@ function getBodyCacheKeyJson(body: unknown): string | object | undefined { return body.toString(); } + if (body instanceof fs.ReadStream) { + return body.path.toString(); + } + if (body instanceof FormData) { return getFormDataCacheKeyJson(body); } @@ -47,7 +52,7 @@ function getBodyCacheKeyJson(body: unknown): string | object | undefined { return body.toString(); } - throw new Error("Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, FormData"); + throw new Error('Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, fs.ReadStream, FormData'); } async function getRequestCacheKeyJson(request: Request) { @@ -63,8 +68,7 @@ async function getRequestCacheKeyJson(request: Request) { // Confirmed that this property exists, but it's not in the types follow: (request as any).follow, // eslint-disable-line @typescript-eslint/no-unsafe-assignment // Confirmed that this property exists, but it's not in the types - compress: (request as any).compress, // eslint-disable-line @typescript-eslint/no-unsafe-assignment - //size: request.size, + compress: (request as any).compress // eslint-disable-line @typescript-eslint/no-unsafe-assignment }; } diff --git a/src/helpers/node_fetch_imports.ts b/src/helpers/node_fetch_imports.ts deleted file mode 100644 index d0ea182..0000000 --- a/src/helpers/node_fetch_imports.ts +++ /dev/null @@ -1,5 +0,0 @@ -export async function getNodeFetch() { - const nodeFetchModule = await import('node-fetch'); - const { default: fetch, Request: NodeFetchRequest, Response: NodeFetchResponse } = nodeFetchModule; - return { ...nodeFetchModule, fetch, NodeFetchRequest, NodeFetchResponse }; -} diff --git a/src/helpers/shim_response_to_snipe_body.ts b/src/helpers/shim_response_to_snipe_body.ts index 7558530..9c8145a 100644 --- a/src/helpers/shim_response_to_snipe_body.ts +++ b/src/helpers/shim_response_to_snipe_body.ts @@ -9,7 +9,7 @@ * but the problems with backpressure seem significant. */ export function shimResponseToSnipeBody( response: Response, - replaceBodyStream: (stream: Omit) => void, + replaceBodyStream: (stream: ReadableStream) => void, ) { const origArrayBuffer = response.arrayBuffer; response.arrayBuffer = async function () { diff --git a/src/index.ts b/src/index.ts index b132077..3403483 100644 --- a/src/index.ts +++ b/src/index.ts @@ -102,7 +102,7 @@ async function getResponse( if (shouldCache) { const cacheSetResult = await fetchCustomization.cache.set( cacheKey, - bodyStream as Omit, "closed">, + bodyStream as ReadableStream, serializedMeta, ); @@ -110,7 +110,7 @@ async function getResponse( } return new NFCResponse( - bodyStream as Omit, "closed">, + bodyStream as ReadableStream, serializedMeta, ejectSelfFromCache, false, diff --git a/src/types.ts b/src/types.ts index 5e03f40..6f45ac1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,21 +12,20 @@ export type NFCResponseMetadata = { status: number; statusText: string; headers: Record; - //size: number; counter: number; }; export type INodeFetchCacheCache = { get(key: string): Promise<{ - bodyStream: Omit; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; } | undefined>; set( key: string, - bodyStream: Omit, + bodyStream: ReadableStream, metaData: NFCResponseMetadata ): Promise<{ - bodyStream: Omit; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; }>; remove(key: string): Promise; diff --git a/test.ts b/test.ts new file mode 100644 index 0000000..9d86bb3 --- /dev/null +++ b/test.ts @@ -0,0 +1,9 @@ +import fetch from './src/index'; + +let request = new Request(`http://httpbin.org/post`, { body: 'test', method: 'POST' }); + +const response1 = await fetch(request); +console.log(await response1.text()); + +const response2 = await fetch(request); +console.log(await response2.text()); diff --git a/test/tests.ts b/test/tests.ts index a4cbd10..e06c527 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -301,7 +301,6 @@ describe('Cache tests', () => { assert.strictEqual(response.returnedFromCache, true); }); - /* it('Gives different read streams different cache keys', async () => { const s1 = fs.createReadStream(path.join(__dirname, 'expected_png.png')); const s2 = fs.createReadStream(path.join(__dirname, '..', 'src', 'index.ts')); @@ -322,7 +321,6 @@ describe('Cache tests', () => { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(s1)); assert.strictEqual(response.returnedFromCache, true); }); - */ it('Gives different form data different cache keys', async () => { const data1 = new FormData(); From a2f222c9768bb1a95aa32a9c340783c5d8ddf74a Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Fri, 25 Apr 2025 10:54:08 -0400 Subject: [PATCH 05/12] Remove `node-fetch` `responseInternalSymbol` --- src/classes/response.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/classes/response.ts b/src/classes/response.ts index 4ccf80c..bf466ae 100644 --- a/src/classes/response.ts +++ b/src/classes/response.ts @@ -1,11 +1,7 @@ import type { ReadableStream } from "stream/web"; -import assert from 'assert'; import { NFCResponseMetadata } from '../types.js'; async function createNFCResponseClass() { - const responseInternalSymbol = Object.getOwnPropertySymbols(new Response())[1]; - assert(responseInternalSymbol, 'Failed to get node-fetch responseInternalSymbol'); - return class NFCResponse extends Response { static serializeMetaFromNodeFetchResponse(response: Response): NFCResponseMetadata { const headers = Array.from(response.headers.entries()).reduce>(function(headers, [key, value]) { @@ -19,7 +15,7 @@ async function createNFCResponseClass() { status: response.status, statusText: response.statusText, headers: headers, - counter: (response as any)[responseInternalSymbol!].counter as number, + counter: (response as any).counter as number, }; return metaData; From 920f9d75a8e40269af4c1379a063a51c5b43ad18 Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Fri, 25 Apr 2025 11:02:27 -0400 Subject: [PATCH 06/12] s/Request/StandardFetchRequest/ --- test.ts | 9 --------- test/tests.ts | 20 +++++++++++--------- 2 files changed, 11 insertions(+), 18 deletions(-) delete mode 100644 test.ts diff --git a/test.ts b/test.ts deleted file mode 100644 index 9d86bb3..0000000 --- a/test.ts +++ /dev/null @@ -1,9 +0,0 @@ -import fetch from './src/index'; - -let request = new Request(`http://httpbin.org/post`, { body: 'test', method: 'POST' }); - -const response1 = await fetch(request); -console.log(await response1.text()); - -const response2 = await fetch(request); -console.log(await response2.text()); diff --git a/test/tests.ts b/test/tests.ts index e06c527..f3a66ff 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -18,6 +18,8 @@ import FetchCache, { ISynchronizationStrategy, } from '../src/index.js'; +const StandardFetchRequest = Request; + const httpBinBaseUrl = 'http://localhost:3000'; const __dirname = dirname(fileURLToPath(import.meta.url)); const wait = util.promisify(setTimeout); @@ -379,7 +381,7 @@ describe('Cache tests', () => { it('Can use a client-provided custom cache key', async () => { const cacheFunction = async (resource: FetchResource) => { - if (resource instanceof Request) { + if (resource instanceof StandardFetchRequest) { return resource.url; } @@ -400,18 +402,18 @@ describe('Cache tests', () => { describe('Data tests', () => { it('Supports request objects', async () => { - let request = new Request('https://google.com', { body: 'test', method: 'POST' }); + let request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, false); - request = new Request('https://google.com', { body: 'test', method: 'POST' }); + request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); assert.strictEqual(response.returnedFromCache, true); }); it('Supports request objects with custom headers', async () => { - const request1 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); - const request2 = new Request(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); + const request1 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'YYY' } }); + const request2 = new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { XXX: 'ZZZ' } }); response = await defaultCachedFetch(request1); assert.strictEqual(response.returnedFromCache, false); @@ -627,20 +629,20 @@ describe('Cache mode tests', () => { it('Can use the only-if-cached cache control setting via resource', async () => { response = await defaultCachedFetch( - new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response.status === 504 && response.isCacheMiss); - response = await defaultCachedFetch(new Request(TWO_HUNDRED_URL)); + response = await defaultCachedFetch(new StandardFetchRequest(TWO_HUNDRED_URL)); assert(response && !response.returnedFromCache); response = await defaultCachedFetch( - new Request(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), + new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'Cache-Control': 'only-if-cached' } }), ); assert(response?.returnedFromCache); }); it('Works with only-if-cached along with other cache-control directives', async () => { response = await defaultCachedFetch( - new Request(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), + new StandardFetchRequest(TWO_HUNDRED_URL, { headers: { 'cAcHe-cOnTrOl': ' only-if-cached , no-store ' } }), ); assert(response.status === 504 && response.isCacheMiss); response = await defaultCachedFetch(TWO_HUNDRED_URL, { From 5301af1b98b6cf0d6ef4291b547d1bbf9252cf6e Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Fri, 25 Apr 2025 11:04:39 -0400 Subject: [PATCH 07/12] All tests passing except "Has a url property" and "Has a redirected property" --- plugins/redis/test_redis_cache.ts | 2 +- test/tests.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index 752fea7..8272212 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -64,7 +64,7 @@ let defaultCachedFetch: typeof FetchCache; let defaultCache: RedisCache; function post(body: string | URLSearchParams | FormData | fs.ReadStream) { - return { method: 'POST', body }; + return { method: 'POST', body, duplex: "half" }; } function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) { diff --git a/test/tests.ts b/test/tests.ts index f3a66ff..51b58ee 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -64,7 +64,7 @@ let defaultCachedFetch: typeof FetchCache; let defaultCache: MemoryCache; function post(body: string | URLSearchParams | FormData | fs.ReadStream) { - return { method: 'POST', body }; + return { method: 'POST', body, duplex: "half" }; } function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) { From 5d6ceb2d05bcfce932a29a614bfbf79dcc072b17 Mon Sep 17 00:00:00 2001 From: mistval Date: Fri, 25 Apr 2025 12:24:47 -0400 Subject: [PATCH 08/12] define url and redirected when constructing NFCResponse --- src/classes/response.ts | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/classes/response.ts b/src/classes/response.ts index bf466ae..e277900 100644 --- a/src/classes/response.ts +++ b/src/classes/response.ts @@ -11,6 +11,7 @@ async function createNFCResponseClass() { }, {}) const metaData = { + redirected: response.redirected, url: response.url, status: response.status, statusText: response.statusText, @@ -32,6 +33,7 @@ async function createNFCResponseClass() { statusText: 'Gateway Timeout', headers: {}, counter: 0, + redirected: false, }, async () => undefined, false, @@ -39,10 +41,25 @@ async function createNFCResponseClass() { ); } + static setUrlAndRedirected(target: Response, url: string, redirected: boolean) { + Object.defineProperty(target, 'url', { + get: () => url, + enumerable: true, + configurable: true, + }); + + Object.defineProperty(target, 'redirected', { + get: () => redirected, + enumerable: true, + configurable: true, + }); + } + constructor( bodyStream: ReadableStream, metaData: Omit & { url: string; + redirected: boolean; counter: number; headers: Record; }, @@ -52,8 +69,16 @@ async function createNFCResponseClass() { ) { super( bodyStream, - metaData as any, // eslint-disable-line @typescript-eslint/no-unsafe-argument + metaData as any, ); + + NFCResponse.setUrlAndRedirected(this, metaData.url, metaData.redirected); + } + + override clone(): Response { + const superClone = super.clone(); + NFCResponse.setUrlAndRedirected(superClone, this.url, this.redirected); + return superClone; } } } From 76a3cc5cec71327002305502e9b8473ad76f79ba Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Fri, 25 Apr 2025 13:45:02 -0400 Subject: [PATCH 09/12] Unneeded `async` --- src/helpers/cache_strategies.ts | 4 ++-- src/types.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/cache_strategies.ts b/src/helpers/cache_strategies.ts index 9073a72..883af49 100644 --- a/src/helpers/cache_strategies.ts +++ b/src/helpers/cache_strategies.ts @@ -1,4 +1,4 @@ import type { CacheStrategy } from '../types.js'; -export const cacheOkayOnly: CacheStrategy = async (response: Response) => response.ok; -export const cacheNon5xxOnly: CacheStrategy = async (response: Response) => response.status < 500; +export const cacheOkayOnly: CacheStrategy = (response: Response) => response.ok; +export const cacheNon5xxOnly: CacheStrategy = (response: Response) => response.status < 500; diff --git a/src/types.ts b/src/types.ts index 6f45ac1..b864a04 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,7 +3,7 @@ import { FormData } from 'formdata-node'; export type FetchResource = Parameters[0]; export type FetchInit = Parameters[1]; -export type CacheStrategy = (response: Response) => Promise; +export type CacheStrategy = (response: Response) => Promise | boolean; export { FormData }; From 54a372872e3f2accbff894ed0d360beb87f982ac Mon Sep 17 00:00:00 2001 From: Brian Jenkins Date: Fri, 25 Apr 2025 16:38:33 -0400 Subject: [PATCH 10/12] Different approach --- plugins/redis/redis_cache.ts | 10 +----- plugins/redis/test_redis_cache.ts | 14 ++------ plugins/redis/tsconfig.json | 51 +++++++++++++++-------------- src/classes/caching/memory_cache.ts | 12 +------ src/classes/response.ts | 32 ++++++------------ src/types.ts | 1 + test/tests.ts | 16 ++------- tsconfig.json | 51 +++++++++++++++-------------- 8 files changed, 70 insertions(+), 117 deletions(-) diff --git a/plugins/redis/redis_cache.ts b/plugins/redis/redis_cache.ts index e1f0178..3cf3095 100644 --- a/plugins/redis/redis_cache.ts +++ b/plugins/redis/redis_cache.ts @@ -66,15 +66,7 @@ export class RedisCache implements INodeFetchCacheCache { metaToStore.expiration = Date.now() + this.ttl; } - const buffer: Buffer = await new Promise(async (fulfill, reject) => { - const chunks = []; - - for await (const chunk of bodyStream) { - chunks.push(chunk); - } - - fulfill(Buffer.concat(chunks)); - }); + const buffer: Buffer = Buffer.concat(await Array.fromAsync(bodyStream)); await (typeof this.ttl === 'number' ? this.redis.set(key, buffer, 'PX', this.ttl) : this.redis.set(key, buffer)); diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index 8272212..7bfca75 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -498,23 +498,13 @@ describe('REDIS Plugin Tests', function() { it('Can stream a body', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); - let chunks = []; - for await (const chunk of response.body ?? []) { - chunks.push(chunk); - } - - assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(TEXT_BODY_URL); - chunks = []; - - for await (const chunk of response.body ?? []) { - chunks.push(chunk); - } - assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, true); }); diff --git a/plugins/redis/tsconfig.json b/plugins/redis/tsconfig.json index 2ad5e7e..0cc7761 100644 --- a/plugins/redis/tsconfig.json +++ b/plugins/redis/tsconfig.json @@ -12,7 +12,7 @@ /* Language and Environment */ "target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + "lib": ["DOM", "DOM.Iterable", "ESNext"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ @@ -25,9 +25,9 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "ES2022", /* Specify what module code is generated. */ + "module": "ES2022", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ @@ -49,13 +49,13 @@ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ - "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./dist", /* Specify an output folder for all emitted files. */ + "outDir": "./dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ @@ -83,24 +83,24 @@ /* Type Checking */ "strict": true, /* Enable all strict type-checking options. */ - "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ - "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ - "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ - "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ - "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ - "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ - "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ - "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ @@ -109,5 +109,6 @@ "ts-node": { "esm": true, }, - "exclude": ["test_redis_cache.ts", "dist"] + "include": ["**/[!test]*.ts"], + "exclude": ["dist", "node_modules"] } diff --git a/src/classes/caching/memory_cache.ts b/src/classes/caching/memory_cache.ts index 240e84e..82d5446 100644 --- a/src/classes/caching/memory_cache.ts +++ b/src/classes/caching/memory_cache.ts @@ -3,16 +3,6 @@ import assert from 'assert'; import type { INodeFetchCacheCache, NFCResponseMetadata } from '../../types.js'; import { KeyTimeout } from './key_timeout.js'; -async function streamToBuffer(stream: ReadableStream): Promise { - const chunks = []; - - for await (const chunk of stream) { - chunks.push(chunk); - } - - return Buffer.concat(chunks); -} - export class MemoryCache implements INodeFetchCacheCache { private readonly ttl?: number | undefined; private readonly keyTimeout = new KeyTimeout(); @@ -40,7 +30,7 @@ export class MemoryCache implements INodeFetchCacheCache { } async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { - const bodyBuffer = await streamToBuffer(bodyStream); + const bodyBuffer = Buffer.concat(await Array.fromAsync(bodyStream)); this.cache.set(key, { bodyBuffer, metaData }); if (typeof this.ttl === 'number') { diff --git a/src/classes/response.ts b/src/classes/response.ts index e277900..eb567a6 100644 --- a/src/classes/response.ts +++ b/src/classes/response.ts @@ -11,8 +11,8 @@ async function createNFCResponseClass() { }, {}) const metaData = { - redirected: response.redirected, url: response.url, + redirected: response.redirected, status: response.status, statusText: response.statusText, headers: headers, @@ -29,11 +29,11 @@ async function createNFCResponseClass() { new Blob().stream() as ReadableStream, { url, + redirected: false, status: 504, statusText: 'Gateway Timeout', headers: {}, counter: 0, - redirected: false, }, async () => undefined, false, @@ -41,23 +41,12 @@ async function createNFCResponseClass() { ); } - static setUrlAndRedirected(target: Response, url: string, redirected: boolean) { - Object.defineProperty(target, 'url', { - get: () => url, - enumerable: true, - configurable: true, - }); - - Object.defineProperty(target, 'redirected', { - get: () => redirected, - enumerable: true, - configurable: true, - }); - } + public override url; + public override redirected; constructor( bodyStream: ReadableStream, - metaData: Omit & { + public metaData: Omit & { url: string; redirected: boolean; counter: number; @@ -69,16 +58,15 @@ async function createNFCResponseClass() { ) { super( bodyStream, - metaData as any, + metaData as any ); - NFCResponse.setUrlAndRedirected(this, metaData.url, metaData.redirected); + this.url = metaData.url; + this.redirected = metaData.redirected } - override clone(): Response { - const superClone = super.clone(); - NFCResponse.setUrlAndRedirected(superClone, this.url, this.redirected); - return superClone; + public override clone(): Response { + return new NFCResponse(super.body as ReadableStream, this.metaData, this.ejectFromCache, this.returnedFromCache, this.isCacheMiss) } } } diff --git a/src/types.ts b/src/types.ts index b864a04..e6c5b31 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,6 +9,7 @@ export { FormData }; export type NFCResponseMetadata = { url: string; + redirect: boolean; status: number; statusText: string; headers: Record; diff --git a/test/tests.ts b/test/tests.ts index 51b58ee..79ecdaa 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -466,23 +466,13 @@ describe('Data tests', () => { it('Can stream a body', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); - let chunks = []; - for await (const chunk of response.body!) { - chunks.push(chunk) - } - - assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(TEXT_BODY_URL); - chunks = []; - - for await (const chunk of response.body!) { - chunks.push(chunk) - } - - assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(chunks).toString()); + + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, true); }); diff --git a/tsconfig.json b/tsconfig.json index d74dbf7..03d1d61 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,7 @@ /* Language and Environment */ "target": "ES2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + "lib": ["DOM", "DOM.Iterable", "ESNext"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ @@ -25,9 +25,9 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "ES2022", /* Specify what module code is generated. */ + "module": "ES2022", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ @@ -49,13 +49,13 @@ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ - "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./dist", /* Specify an output folder for all emitted files. */ + "outDir": "./dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ @@ -83,24 +83,24 @@ /* Type Checking */ "strict": true, /* Enable all strict type-checking options. */ - "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ - "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ - "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ - "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ - "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ - "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ - "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ - "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ @@ -109,5 +109,6 @@ "ts-node": { "esm": true, }, - "exclude": ["test", "dist", "plugins"] + "include": ["src/**/*.ts"], + "exclude": ["dist", "plugins", "node_modules"] } From c4ce61ebee00d68023cc1575e983a8e2a42c30b6 Mon Sep 17 00:00:00 2001 From: mistval Date: Mon, 28 Apr 2025 11:31:53 -0400 Subject: [PATCH 11/12] fix redirected type --- src/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.ts b/src/types.ts index e6c5b31..63460b9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,7 +9,7 @@ export { FormData }; export type NFCResponseMetadata = { url: string; - redirect: boolean; + redirected: boolean; status: number; statusText: string; headers: Record; From b4df92ca73e779f73b8a2d0d3e66475dde2728f8 Mon Sep 17 00:00:00 2001 From: mistval Date: Mon, 28 Apr 2025 11:58:01 -0400 Subject: [PATCH 12/12] add cloning body read test --- test/tests.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/tests.ts b/test/tests.ts index 79ecdaa..3d5975d 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -401,6 +401,15 @@ describe('Cache tests', () => { }).timeout(10_000); describe('Data tests', () => { + it('Can clone a response and read both', async () => { + const response = await defaultCachedFetch(TEXT_BODY_URL); + const clonedResponse = response.clone(); + const body1 = await response.text(); + const body2 = await clonedResponse.text(); + assert.strictEqual(body1, TEXT_BODY_EXPECTED); + assert.strictEqual(body2, TEXT_BODY_EXPECTED); + }); + it('Supports request objects', async () => { let request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request);