diff --git a/README.md b/README.md index f7678fd..3949e13 100644 --- a/README.md +++ b/README.md @@ -117,15 +117,15 @@ If none of the existing caching options meet your needs, you can implement your ```ts type INodeFetchCacheCache = { get(key: string): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; } | undefined>; set( key: string, - bodyStream: NodeJS.ReadableStream, + bodyStream: ReadableStream, metaData: NFCResponseMetadata ): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; }>; remove(key: string): Promise; @@ -163,19 +163,6 @@ if (response.isCacheMiss) { ## Advanced API -### Accessing Node-Fetch Exports - -If you need to access `node-fetch` exports (for example you might want to create a Request instance), you can do so by using the `getNodeFetch()` function: - -```js -import fetch, { getNodeFetch } from 'node-fetch-cache'; - -const { Request } = await getNodeFetch(); -const response = await fetch(new Request('https://google.com')); -``` - -You should not import from `node-fetch` directly since it is important that your code is using exports from the same version of `node-fetch` that is being used by `node-fetch-cache` internally. - ### Custom Cache Key Function You can provide custom cache key generation logic to node-fetch-cache by passing a `calculateCacheKey` option to `create()`: diff --git a/package-lock.json b/package-lock.json index 1fb314f..a45ac95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,7 @@ "dependencies": { "cacache": "^18.0.4", "formdata-node": "^6.0.3", - "locko": "^1.1.0", - "node-fetch": "3.3.2" + "locko": "^1.1.0" }, "devDependencies": { "@types/cacache": "^17.0.2", @@ -28,7 +27,7 @@ "rimraf": "^5.0.5", "rollup": "^4.9.1", "tsx": "^4.17.0", - "typescript": "^5.5.4" + "typescript": "^5.8.3" }, "engines": { "node": ">=18.19.0" @@ -972,12 +971,13 @@ "dev": true }, "node_modules/@types/node": { - "version": "20.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", - "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", + "version": "22.14.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", + "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", "dev": true, + "license": "MIT", "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "node_modules/@types/semver": { @@ -1693,15 +1693,6 @@ "node": ">= 8" } }, - "node_modules/data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, "node_modules/debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -2062,29 +2053,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "paypal", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - }, - "engines": { - "node": "^12.20 || >= 14.13" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -2195,18 +2163,6 @@ "node": ">= 18" } }, - "node_modules/formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "license": "MIT", - "dependencies": { - "fetch-blob": "^3.1.2" - }, - "engines": { - "node": ">=12.20.0" - } - }, "node_modules/fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -2928,43 +2884,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "license": "MIT", - "dependencies": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/node-fetch" - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -3676,9 +3595,9 @@ } }, "node_modules/typescript": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", - "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", "bin": { @@ -3690,10 +3609,11 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" }, "node_modules/unique-filename": { "version": "3.0.0", @@ -3747,15 +3667,6 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true }, - "node_modules/web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -4431,12 +4342,12 @@ "dev": true }, "@types/node": { - "version": "20.10.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", - "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", + "version": "22.14.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", + "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", "dev": true, "requires": { - "undici-types": "~5.26.4" + "undici-types": "~6.21.0" } }, "@types/semver": { @@ -4916,11 +4827,6 @@ "which": "^2.0.1" } }, - "data-uri-to-buffer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", - "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" - }, "debug": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", @@ -5196,15 +5102,6 @@ "reusify": "^1.0.4" } }, - "fetch-blob": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", - "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", - "requires": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" - } - }, "file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -5286,14 +5183,6 @@ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz", "integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==" }, - "formdata-polyfill": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", - "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", - "requires": { - "fetch-blob": "^3.1.2" - } - }, "fs-minipass": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", @@ -5815,21 +5704,6 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, - "node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" - }, - "node-fetch": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", - "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", - "requires": { - "data-uri-to-buffer": "^4.0.0", - "fetch-blob": "^3.1.4", - "formdata-polyfill": "^4.0.10" - } - }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -6303,15 +6177,15 @@ "peer": true }, "typescript": { - "version": "5.5.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", - "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", + "version": "5.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true }, "undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", "dev": true }, "unique-filename": { @@ -6359,11 +6233,6 @@ } } }, - "web-streams-polyfill": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", - "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==" - }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 9d9048b..780f401 100644 --- a/package.json +++ b/package.json @@ -52,13 +52,12 @@ "rimraf": "^5.0.5", "rollup": "^4.9.1", "tsx": "^4.17.0", - "typescript": "^5.5.4" + "typescript": "^5.8.3" }, "dependencies": { "cacache": "^18.0.4", "formdata-node": "^6.0.3", - "locko": "^1.1.0", - "node-fetch": "3.3.2" + "locko": "^1.1.0" }, "husky": { "hooks": { diff --git a/plugins/redis/redis_cache.ts b/plugins/redis/redis_cache.ts index bdd07ff..3cf3095 100644 --- a/plugins/redis/redis_cache.ts +++ b/plugins/redis/redis_cache.ts @@ -1,5 +1,4 @@ -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; +import { ReadableStream } from "stream/web"; import Redis from 'ioredis'; import type { RedisOptions } from 'ioredis'; import type { INodeFetchCacheCache, NFCResponseMetadata } from 'node-fetch-cache'; @@ -35,7 +34,7 @@ export class RedisCache implements INodeFetchCacheCache { return undefined; } - const readableStream = Readable.from(cachedObjectInfo); + const readableStream = new Blob([cachedObjectInfo]).stream(); const storedMetadata = await this.redis.get(`${key}:meta`); if (!storedMetadata) { @@ -46,7 +45,7 @@ export class RedisCache implements INodeFetchCacheCache { const { expiration, ...nfcMetadata } = storedMetadataJson; return { - bodyStream: readableStream, + bodyStream: readableStream as ReadableStream, metaData: nfcMetadata, }; } @@ -57,7 +56,7 @@ export class RedisCache implements INodeFetchCacheCache { return true; } - async set(key: string, bodyStream: NodeJS.ReadableStream, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { const metaToStore = { ...metaData, expiration: undefined as undefined | number, @@ -67,25 +66,7 @@ export class RedisCache implements INodeFetchCacheCache { metaToStore.expiration = Date.now() + this.ttl; } - const buffer: Buffer = await new Promise((fulfill, reject) => { - const chunks: Buffer[] = []; - - bodyStream.on('data', chunk => { - chunks.push(chunk as Buffer); - }); - - bodyStream.on('end', async () => { - try { - fulfill(Buffer.concat(chunks)); - } catch (error) { - reject(error); - } - }); - - bodyStream.on('error', error => { - reject(error); - }); - }); + const buffer: Buffer = Buffer.concat(await Array.fromAsync(bodyStream)); await (typeof this.ttl === 'number' ? this.redis.set(key, buffer, 'PX', this.ttl) : this.redis.set(key, buffer)); @@ -94,7 +75,7 @@ export class RedisCache implements INodeFetchCacheCache { } return { - bodyStream: Readable.from(buffer), + bodyStream: new Blob([buffer]).stream() as ReadableStream, metaData: metaToStore, }; } diff --git a/plugins/redis/test_redis_cache.ts b/plugins/redis/test_redis_cache.ts index f0dd8b1..7bfca75 100644 --- a/plugins/redis/test_redis_cache.ts +++ b/plugins/redis/test_redis_cache.ts @@ -7,7 +7,6 @@ import fs from 'fs'; import assert from 'assert'; import { Agent } from 'http'; import { FormData } from 'formdata-node'; -import standardFetch, { Request as StandardFetchRequest } from 'node-fetch'; import FetchCache, { cacheStrategies, FetchResource, @@ -18,6 +17,8 @@ import FetchCache, { import { RedisCache } from './redis_cache.js'; import { Redis } from 'ioredis'; +const StandardFetchRequest = Request; + const MIN_NODE_VERSION = 16; const httpBinBaseUrl = 'http://localhost:3000'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -63,7 +64,7 @@ let defaultCachedFetch: typeof FetchCache; let defaultCache: RedisCache; function post(body: string | URLSearchParams | FormData | fs.ReadStream) { - return { method: 'POST', body }; + return { method: 'POST', body, duplex: "half" }; } function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) { @@ -84,10 +85,10 @@ function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) return arrayOrObject; } -async function dualFetch(...args: Parameters) { +async function dualFetch(...args: Parameters) { const [cachedFetchResponse, standardFetchResponse] = await Promise.all([ defaultCachedFetch(...args), - standardFetch(...args), + fetch(...args), ]); return { cachedFetchResponse, standardFetchResponse }; @@ -156,14 +157,14 @@ describe('REDIS Plugin Tests', function() { it('Gets correct raw headers', async () => { let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); cachedFetchResponse = await defaultCachedFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); }); @@ -308,7 +309,7 @@ describe('REDIS Plugin Tests', function() { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(new URLSearchParams('a=a'))); assert.strictEqual(response.returnedFromCache, true); }); - + it('Gives different read streams different cache keys', async () => { const s1 = fs.createReadStream(path.join(__dirname, '..', '..', 'test', 'expected_png.png')); const s2 = fs.createReadStream(path.join(__dirname, '..', '..', 'test', '..', 'src', 'index.ts')); @@ -329,7 +330,7 @@ describe('REDIS Plugin Tests', function() { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(s1)); assert.strictEqual(response.returnedFromCache, true); }); - + it('Gives different form data different cache keys', async () => { const data1 = new FormData(); data1.append('a', 'a'); @@ -402,12 +403,12 @@ describe('REDIS Plugin Tests', function() { it('Can get PNG buffer body', async () => { defaultCachedFetch = FetchCache.create({ cache: new RedisCache(undefined, redisClient) }); response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); @@ -456,7 +457,7 @@ describe('REDIS Plugin Tests', function() { it('Refuses to consume body twice', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); await response.text(); - await assert.rejects(async () => response.text(), /body used already for:/); + await assert.rejects(async () => response.text(), /Body has already been read/); }); it('Can get text body', async () => { @@ -485,35 +486,25 @@ describe('REDIS Plugin Tests', function() { it('Can get PNG buffer body', async () => { response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); it('Can stream a body', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); - let body = ''; - for await (const chunk of response.body ?? []) { - body += chunk.toString(); - } - - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(TEXT_BODY_URL); - body = ''; - - for await (const chunk of response.body ?? []) { - body += chunk.toString(); - } - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, true); }); @@ -527,7 +518,7 @@ describe('REDIS Plugin Tests', function() { it('Errors if the resource type is not supported', async () => { await assert.rejects( async () => defaultCachedFetch(1 as unknown as string), - /The first argument to fetch must be either a string or a node-fetch Request instance/, + /The first argument to fetch must be either a string or a fetch Request instance/, ); }); @@ -570,15 +561,15 @@ describe('REDIS Plugin Tests', function() { assert(initialResponse.ok); assert(!initialResponse.returnedFromCache); - const initialResponseBuffer = await initialResponse.buffer(); - assert.equal(initialResponseBuffer.length, 100_000); + const initialResponseBuffer = await initialResponse.arrayBuffer(); + assert.equal(initialResponseBuffer.byteLength, 100_000); const secondResponse = await defaultCachedFetch(HUNDRED_THOUSAND_BYTES_URL); assert(secondResponse.ok); assert(secondResponse.returnedFromCache); - const secondResponseBuffer = await secondResponse.buffer(); - assert.equal(secondResponseBuffer.length, 100_000); + const secondResponseBuffer = await secondResponse.arrayBuffer(); + assert.equal(secondResponseBuffer.byteLength, 100_000); }); }).timeout(10_000); @@ -715,7 +706,7 @@ describe('REDIS Plugin Tests', function() { }); it('Can use a custom cache strategy that uses the response for all response types', async () => { - const functionsThatUseResponse = ['arrayBuffer', 'blob', 'buffer', 'json', 'text'] as const; + const functionsThatUseResponse = ['arrayBuffer', 'blob', 'json', 'text'] as const; for (const functionName of functionsThatUseResponse) { await redisClient.flushall(); @@ -745,7 +736,7 @@ describe('REDIS Plugin Tests', function() { describe('REDIS Network error tests', () => { it('Bubbles up network errors', async () => { - await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /^FetchError:/); + await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /TypeError: fetch failed/); }); }); }); diff --git a/plugins/redis/tsconfig.json b/plugins/redis/tsconfig.json index 2ad5e7e..0cc7761 100644 --- a/plugins/redis/tsconfig.json +++ b/plugins/redis/tsconfig.json @@ -12,7 +12,7 @@ /* Language and Environment */ "target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + "lib": ["DOM", "DOM.Iterable", "ESNext"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ @@ -25,9 +25,9 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "ES2022", /* Specify what module code is generated. */ + "module": "ES2022", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ @@ -49,13 +49,13 @@ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ - "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./dist", /* Specify an output folder for all emitted files. */ + "outDir": "./dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ @@ -83,24 +83,24 @@ /* Type Checking */ "strict": true, /* Enable all strict type-checking options. */ - "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ - "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ - "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ - "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ - "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ - "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ - "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ - "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ @@ -109,5 +109,6 @@ "ts-node": { "esm": true, }, - "exclude": ["test_redis_cache.ts", "dist"] + "include": ["**/[!test]*.ts"], + "exclude": ["dist", "node_modules"] } diff --git a/src/classes/caching/file_system_cache.ts b/src/classes/caching/file_system_cache.ts index 2cb9b80..c90e45f 100644 --- a/src/classes/caching/file_system_cache.ts +++ b/src/classes/caching/file_system_cache.ts @@ -1,15 +1,15 @@ +import { ReadableStream } from "stream/web"; import assert from 'assert'; -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; import cacache from 'cacache'; import type { INodeFetchCacheCache, NFCResponseMetadata } from '../../types'; +import { Stream } from 'stream'; type StoredMetadata = { emptyBody?: boolean; expiration?: number | undefined; } & NFCResponseMetadata; -const emptyBuffer = Buffer.alloc(0); +const emptyBuffer = Buffer.from([]) export class FileSystemCache implements INodeFetchCacheCache { private readonly ttl?: number | undefined; @@ -36,13 +36,13 @@ export class FileSystemCache implements INodeFetchCacheCache { if (emptyBody) { return { - bodyStream: Readable.from(emptyBuffer), + bodyStream: ReadableStream.from(emptyBuffer), metaData: storedMetadata, }; } return { - bodyStream: cacache.get.stream.byDigest(this.cacheDirectory, cachedObjectInfo.integrity), + bodyStream: ReadableStream.from(cacache.get.stream.byDigest(this.cacheDirectory, cachedObjectInfo.integrity)), metaData: nfcMetadata, }; } @@ -51,7 +51,7 @@ export class FileSystemCache implements INodeFetchCacheCache { return cacache.rm.entry(this.cacheDirectory, key); } - async set(key: string, bodyStream: NodeJS.ReadableStream, metaData: NFCResponseMetadata) { + async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { const metaToStore = { ...metaData, expiration: undefined as (undefined | number), @@ -73,11 +73,11 @@ export class FileSystemCache implements INodeFetchCacheCache { private async writeDataToCache( key: string, storedMetadata: StoredMetadata, - stream: NodeJS.ReadableStream, + stream: ReadableStream, ) { try { await new Promise((fulfill, reject) => { - stream.pipe(cacache.put.stream(this.cacheDirectory, key, { metadata: storedMetadata })) + Stream.Readable.fromWeb(stream).pipe(cacache.put.stream(this.cacheDirectory, key, { metadata: storedMetadata })) .on('integrity', (i: string) => { fulfill(i); }) diff --git a/src/classes/caching/memory_cache.ts b/src/classes/caching/memory_cache.ts index d5c2a73..82d5446 100644 --- a/src/classes/caching/memory_cache.ts +++ b/src/classes/caching/memory_cache.ts @@ -1,22 +1,8 @@ +import { ReadableStream } from "stream/web"; import assert from 'assert'; -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; import type { INodeFetchCacheCache, NFCResponseMetadata } from '../../types.js'; import { KeyTimeout } from './key_timeout.js'; -async function streamToBuffer(stream: NodeJS.ReadableStream): Promise { - const chunks: Buffer[] = []; - return new Promise((resolve, reject) => { - stream.on('data', chunk => - chunks.push(chunk as Buffer), - ).on('error', error => { - reject(error); - }).on('end', () => { - resolve(Buffer.concat(chunks)); - }); - }); -} - export class MemoryCache implements INodeFetchCacheCache { private readonly ttl?: number | undefined; private readonly keyTimeout = new KeyTimeout(); @@ -30,7 +16,7 @@ export class MemoryCache implements INodeFetchCacheCache { const cachedValue = this.cache.get(key); if (cachedValue) { return { - bodyStream: Readable.from(cachedValue.bodyBuffer), + bodyStream: new Blob([cachedValue.bodyBuffer]).stream() as ReadableStream, metaData: cachedValue.metaData, }; } @@ -43,8 +29,8 @@ export class MemoryCache implements INodeFetchCacheCache { this.cache.delete(key); } - async set(key: string, bodyStream: NodeJS.ReadableStream, metaData: NFCResponseMetadata) { - const bodyBuffer = await streamToBuffer(bodyStream); + async set(key: string, bodyStream: ReadableStream, metaData: NFCResponseMetadata) { + const bodyBuffer = Buffer.concat(await Array.fromAsync(bodyStream)); this.cache.set(key, { bodyBuffer, metaData }); if (typeof this.ttl === 'number') { diff --git a/src/classes/response.ts b/src/classes/response.ts index d38b5a1..eb567a6 100644 --- a/src/classes/response.ts +++ b/src/classes/response.ts @@ -1,25 +1,22 @@ -import assert from 'assert'; -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; -import type { Response as NodeFetchResponseType, ResponseInit as NodeFetchResponseInit } from 'node-fetch'; +import type { ReadableStream } from "stream/web"; import { NFCResponseMetadata } from '../types.js'; -import { getNodeFetch } from '../helpers/node_fetch_imports.js'; async function createNFCResponseClass() { - const { NodeFetchResponse } = await getNodeFetch(); + return class NFCResponse extends Response { + static serializeMetaFromNodeFetchResponse(response: Response): NFCResponseMetadata { + const headers = Array.from(response.headers.entries()).reduce>(function(headers, [key, value]) { + headers[key] = [...(headers[key] ?? []), value]; - const responseInternalSymbol = Object.getOwnPropertySymbols(new NodeFetchResponse())[1]; - assert(responseInternalSymbol, 'Failed to get node-fetch responseInternalSymbol'); + return headers; + }, {}) - return class NFCResponse extends NodeFetchResponse { - static serializeMetaFromNodeFetchResponse(response: NodeFetchResponseType): NFCResponseMetadata { const metaData = { url: response.url, + redirected: response.redirected, status: response.status, statusText: response.statusText, - headers: response.headers.raw(), - size: response.size, - counter: (response as any)[responseInternalSymbol!].counter as number, + headers: headers, + counter: (response as any).counter as number, }; return metaData; @@ -29,13 +26,13 @@ async function createNFCResponseClass() { url: string, ) { return new NFCResponse( - Readable.from(Buffer.alloc(0)), + new Blob().stream() as ReadableStream, { url, + redirected: false, status: 504, statusText: 'Gateway Timeout', headers: {}, - size: 0, counter: 0, }, async () => undefined, @@ -44,11 +41,14 @@ async function createNFCResponseClass() { ); } + public override url; + public override redirected; + constructor( - bodyStream: NodeJS.ReadableStream, - metaData: Omit & { + bodyStream: ReadableStream, + public metaData: Omit & { url: string; - size: number; + redirected: boolean; counter: number; headers: Record; }, @@ -57,9 +57,16 @@ async function createNFCResponseClass() { public readonly isCacheMiss = false, ) { super( - Readable.from(bodyStream), - metaData as any, // eslint-disable-line @typescript-eslint/no-unsafe-argument + bodyStream, + metaData as any ); + + this.url = metaData.url; + this.redirected = metaData.redirected + } + + public override clone(): Response { + return new NFCResponse(super.body as ReadableStream, this.metaData, this.ejectFromCache, this.returnedFromCache, this.isCacheMiss) } } } diff --git a/src/helpers/cache_keys.ts b/src/helpers/cache_keys.ts index 287198d..283b5b3 100644 --- a/src/helpers/cache_keys.ts +++ b/src/helpers/cache_keys.ts @@ -1,16 +1,14 @@ -import fs from 'fs'; -import crypto from 'crypto'; -import assert from 'assert'; -import { Buffer } from 'buffer'; -import type { Request as NodeFetchRequestType } from 'node-fetch'; import type { FetchInit, FetchResource } from '../types.js'; import { FormData } from '../types.js'; -import { getNodeFetch } from './node_fetch_imports.js'; +import * as fs from "fs"; export const CACHE_VERSION = 6; -function md5(string_: string) { - return crypto.createHash('md5').update(string_).digest('hex'); +async function sha1(string_: string) { + return Array.from( + new Uint8Array(await crypto.subtle.digest("SHA-1", new TextEncoder().encode(string_))), + (byte) => byte.toString(16).padStart(2, "0") + ).join(""); } function getFormDataCacheKeyJson(formData: FormData) { @@ -50,36 +48,32 @@ function getBodyCacheKeyJson(body: unknown): string | object | undefined { return getFormDataCacheKeyJson(body); } - if (body instanceof Buffer) { + if (body instanceof ArrayBuffer) { return body.toString(); } throw new Error('Unsupported body type. Supported body types are: string, number, undefined, null, url.URLSearchParams, fs.ReadStream, FormData'); } -async function getRequestCacheKeyJson(request: NodeFetchRequestType) { - const { NodeFetchRequest } = await getNodeFetch(); - const bodyInternalsSymbol = Object.getOwnPropertySymbols(new NodeFetchRequest('http://url.com'))[0]; - assert(bodyInternalsSymbol, 'Failed to get node-fetch bodyInternalsSymbol'); +async function getRequestCacheKeyJson(request: Request) { + const body = await request.arrayBuffer(); return { - headers: getHeadersCacheKeyJson([...request.headers.entries()]), + headers: getHeadersCacheKeyJson(Array.from(request.headers.entries())), method: request.method, redirect: request.redirect, referrer: request.referrer, url: request.url, - body: getBodyCacheKeyJson((request as any)[bodyInternalsSymbol!].body), + body: getBodyCacheKeyJson(body), // Confirmed that this property exists, but it's not in the types follow: (request as any).follow, // eslint-disable-line @typescript-eslint/no-unsafe-assignment // Confirmed that this property exists, but it's not in the types - compress: (request as any).compress, // eslint-disable-line @typescript-eslint/no-unsafe-assignment - size: request.size, + compress: (request as any).compress // eslint-disable-line @typescript-eslint/no-unsafe-assignment }; } export async function calculateCacheKey(resource: FetchResource, init?: FetchInit) { - const { NodeFetchRequest } = await getNodeFetch(); - const resourceCacheKeyJson = resource instanceof NodeFetchRequest + const resourceCacheKeyJson = resource instanceof Request ? await getRequestCacheKeyJson(resource) : { url: resource, body: undefined }; @@ -92,7 +86,8 @@ export async function calculateCacheKey(resource: FetchResource, init?: FetchIni resourceCacheKeyJson.body = getBodyCacheKeyJson(resourceCacheKeyJson.body); initCacheKeyJson.body = getBodyCacheKeyJson(initCacheKeyJson.body); + // @ts-expect-error delete initCacheKeyJson.agent; - return md5(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION])); + return sha1(JSON.stringify([resourceCacheKeyJson, initCacheKeyJson, CACHE_VERSION])); } diff --git a/src/helpers/cache_strategies.ts b/src/helpers/cache_strategies.ts index 4b9683b..883af49 100644 --- a/src/helpers/cache_strategies.ts +++ b/src/helpers/cache_strategies.ts @@ -1,4 +1,3 @@ -import type { Response } from 'node-fetch'; import type { CacheStrategy } from '../types.js'; export const cacheOkayOnly: CacheStrategy = (response: Response) => response.ok; diff --git a/src/helpers/headers.ts b/src/helpers/headers.ts index 0a17dba..729cd4c 100644 --- a/src/helpers/headers.ts +++ b/src/helpers/headers.ts @@ -1,5 +1,4 @@ import { FetchInit, FetchResource } from '../types.js'; -import { getNodeFetch } from './node_fetch_imports.js'; function headerKeyIsCacheControl(key: string) { return key.trim().toLowerCase() === 'cache-control'; @@ -26,10 +25,8 @@ export async function hasOnlyIfCachedOption(resource: FetchResource, init: Fetch return true; } - const { NodeFetchRequest } = await getNodeFetch(); - if ( - resource instanceof NodeFetchRequest + resource instanceof Request && headerValueContainsOnlyIfCached(resource.headers.get('Cache-Control') ?? undefined) ) { return true; diff --git a/src/helpers/node_fetch_imports.ts b/src/helpers/node_fetch_imports.ts deleted file mode 100644 index d0ea182..0000000 --- a/src/helpers/node_fetch_imports.ts +++ /dev/null @@ -1,5 +0,0 @@ -export async function getNodeFetch() { - const nodeFetchModule = await import('node-fetch'); - const { default: fetch, Request: NodeFetchRequest, Response: NodeFetchResponse } = nodeFetchModule; - return { ...nodeFetchModule, fetch, NodeFetchRequest, NodeFetchResponse }; -} diff --git a/src/helpers/shim_response_to_snipe_body.ts b/src/helpers/shim_response_to_snipe_body.ts index 42be9b0..9c8145a 100644 --- a/src/helpers/shim_response_to_snipe_body.ts +++ b/src/helpers/shim_response_to_snipe_body.ts @@ -1,7 +1,3 @@ -import { Buffer } from 'buffer'; -import { Readable } from 'stream'; -import type { Response as NodeFetchResponse } from 'node-fetch'; - /* This is a bit of a hack to deal with the case when the user * consumes the response body in their `shouldCacheResponse` delegate. * The response body can only be consumed once, so if the user consumes @@ -12,41 +8,35 @@ import type { Response as NodeFetchResponse } from 'node-fetch'; * My initial inclination was to use Response.prototype.clone() for this, * but the problems with backpressure seem significant. */ export function shimResponseToSnipeBody( - response: NodeFetchResponse, - replaceBodyStream: (strean: NodeJS.ReadableStream) => void, + response: Response, + replaceBodyStream: (stream: ReadableStream) => void, ) { const origArrayBuffer = response.arrayBuffer; response.arrayBuffer = async function () { const arrayBuffer = await origArrayBuffer.call(this); - replaceBodyStream(Readable.from(Buffer.from(arrayBuffer))); + replaceBodyStream(new Blob([arrayBuffer]).stream()); return arrayBuffer; }; - const origBuffer = response.buffer; - response.buffer = async function () { - const buffer = await origBuffer.call(this); - replaceBodyStream(Readable.from(buffer)); - return buffer; - }; - const origJson = response.json; response.json = async function () { const json = await origJson.call(this); - replaceBodyStream(Readable.from(Buffer.from(JSON.stringify(json)))); + replaceBodyStream(new Blob([JSON.stringify(json)]).stream()); return json; }; const origText = response.text; response.text = async function () { const text = await origText.call(this); - replaceBodyStream(Readable.from(Buffer.from(text))); + replaceBodyStream(new Blob([text]).stream()); return text; }; const origBlob = response.blob; response.blob = async function () { const blob = await origBlob.call(this); - replaceBodyStream(Readable.from(Buffer.from(await blob.text()))); + const text = await blob.text(); + replaceBodyStream(new Blob([text]).stream()); return blob; }; } diff --git a/src/index.ts b/src/index.ts index 02fbcb8..3403483 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -import type { Request as NodeFetchRequestType } from 'node-fetch'; +import { ReadableStream } from "stream/web"; import assert from 'assert'; import { FormData } from 'formdata-node'; import { getNFCResponseClass as getNFCResponseClass } from './classes/response.js'; @@ -15,7 +15,6 @@ import type { INodeFetchCacheCache, ISynchronizationStrategy, } from './types.js'; -import { getNodeFetch } from './helpers/node_fetch_imports.js'; type CacheKeyCalculator = typeof calculateCacheKey; @@ -28,10 +27,8 @@ type NFCCustomizations = { type NFCOptions = Partial; -async function getUrlFromRequestArguments(resource: NodeFetchRequestType | string) { - const { NodeFetchRequest } = await getNodeFetch(); - - if (resource instanceof NodeFetchRequest) { +async function getUrlFromRequestArguments(resource: Request | string) { + if (resource instanceof Request) { return resource.url; } @@ -43,15 +40,20 @@ async function getResponse( resource: FetchResource, init: FetchInit, ) { - const { NodeFetchRequest, fetch } = await getNodeFetch(); + const originalResource = resource; + const NFCResponse = await getNFCResponseClass(); - if (typeof resource !== 'string' && !(resource instanceof NodeFetchRequest)) { + if (typeof resource !== 'string' && !(resource instanceof Request)) { throw new TypeError( - 'The first argument to fetch must be either a string or a node-fetch Request instance', + 'The first argument to fetch must be either a string or a fetch Request instance', ); } + if (originalResource instanceof Request) { + resource = originalResource.clone() + } + const cacheKey = await fetchCustomization.calculateCacheKey(resource, init); const ejectSelfFromCache = async () => fetchCustomization.cache.remove(cacheKey); @@ -82,6 +84,10 @@ async function getResponse( ); } + if (originalResource instanceof Request) { + resource = originalResource.clone() + } + const fetchResponse = await fetch(resource, init); const serializedMeta = NFCResponse.serializeMetaFromNodeFetchResponse(fetchResponse); let bodyStream = fetchResponse.body; @@ -96,7 +102,7 @@ async function getResponse( if (shouldCache) { const cacheSetResult = await fetchCustomization.cache.set( cacheKey, - bodyStream, + bodyStream as ReadableStream, serializedMeta, ); @@ -104,7 +110,7 @@ async function getResponse( } return new NFCResponse( - bodyStream, + bodyStream as ReadableStream, serializedMeta, ejectSelfFromCache, false, @@ -118,7 +124,7 @@ function create(creationOptions: NFCOptions) { const fetchOptions: NFCCustomizations = { cache: creationOptions.cache ?? globalMemoryCache, synchronizationStrategy: creationOptions.synchronizationStrategy ?? new LockoSynchronizationStrategy(), - shouldCacheResponse: creationOptions.shouldCacheResponse ?? (() => true), + shouldCacheResponse: creationOptions.shouldCacheResponse ?? (() => Promise.resolve(true)), calculateCacheKey: creationOptions.calculateCacheKey ?? calculateCacheKey, }; @@ -148,7 +154,6 @@ export default defaultFetch; export { MemoryCache } from './classes/caching/memory_cache.js'; export { FileSystemCache } from './classes/caching/file_system_cache.js'; export { CACHE_VERSION } from './helpers/cache_keys.js'; -export { getNodeFetch }; export type { NFCResponse } from './classes/response.js'; export type { NFCResponseMetadata } from './types.js'; export { @@ -157,7 +162,6 @@ export { calculateCacheKey as getCacheKey, calculateCacheKey, FormData, - type NodeFetchRequestType as NodeFetchRequest, type NFCOptions, type CacheKeyCalculator, type INodeFetchCacheCache, diff --git a/src/types.ts b/src/types.ts index 68d6209..63460b9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,33 +1,32 @@ -import type { Response as NodeFetchResponse } from 'node-fetch'; -import type fetch from 'node-fetch'; +import type { ReadableStream } from "stream/web"; import { FormData } from 'formdata-node'; export type FetchResource = Parameters[0]; export type FetchInit = Parameters[1]; -export type CacheStrategy = (response: NodeFetchResponse) => Promise | boolean; +export type CacheStrategy = (response: Response) => Promise | boolean; export { FormData }; export type NFCResponseMetadata = { url: string; + redirected: boolean; status: number; statusText: string; headers: Record; - size: number; counter: number; }; export type INodeFetchCacheCache = { get(key: string): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; } | undefined>; set( key: string, - bodyStream: NodeJS.ReadableStream, + bodyStream: ReadableStream, metaData: NFCResponseMetadata ): Promise<{ - bodyStream: NodeJS.ReadableStream; + bodyStream: ReadableStream; metaData: NFCResponseMetadata; }>; remove(key: string): Promise; diff --git a/test/tests.cjs b/test/tests.cjs index 58dac42..9ee6d32 100644 --- a/test/tests.cjs +++ b/test/tests.cjs @@ -12,8 +12,7 @@ describe('Commonjs module tests', () => { }); it('Can make a request via Request object', async () => { - const nodeFetch = await fetch.getNodeFetch(); - const res = await fetch(new nodeFetch.Request(TWO_HUNDRED_URL)); + const res = await fetch(new Request(TWO_HUNDRED_URL)); assert.strictEqual(res.status, 200); }); diff --git a/test/tests.ts b/test/tests.ts index b07a1f0..3d5975d 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -8,7 +8,6 @@ import assert from 'assert'; import { Agent } from 'http'; import { rimraf } from 'rimraf'; import { FormData } from 'formdata-node'; -import standardFetch, { Request as StandardFetchRequest } from 'node-fetch'; import FetchCache, { MemoryCache, FileSystemCache, @@ -19,6 +18,8 @@ import FetchCache, { ISynchronizationStrategy, } from '../src/index.js'; +const StandardFetchRequest = Request; + const httpBinBaseUrl = 'http://localhost:3000'; const __dirname = dirname(fileURLToPath(import.meta.url)); const wait = util.promisify(setTimeout); @@ -63,7 +64,7 @@ let defaultCachedFetch: typeof FetchCache; let defaultCache: MemoryCache; function post(body: string | URLSearchParams | FormData | fs.ReadStream) { - return { method: 'POST', body }; + return { method: 'POST', body, duplex: "half" }; } function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) { @@ -84,10 +85,10 @@ function removeDates(arrayOrObject: { date?: unknown } | string[] | string[][]) return arrayOrObject; } -async function dualFetch(...args: Parameters) { +async function dualFetch(...args: Parameters) { const [cachedFetchResponse, standardFetchResponse] = await Promise.all([ defaultCachedFetch(...args), - standardFetch(...args), + fetch(...args), ]); return { cachedFetchResponse, standardFetchResponse }; @@ -149,14 +150,14 @@ describe('Header tests', () => { it('Gets correct raw headers', async () => { let { cachedFetchResponse, standardFetchResponse } = await dualFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); cachedFetchResponse = await defaultCachedFetch(TWO_HUNDRED_URL); assert.deepStrictEqual( - removeDates(cachedFetchResponse.headers.raw()), - removeDates(standardFetchResponse.headers.raw()), + removeDates(Array.from(cachedFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), + removeDates(Array.from(standardFetchResponse.headers.entries()).reduce((headers, [key, value]) => { headers[key] = [...(headers[key] ?? []), value]; return headers; }, {})), ); }); @@ -322,7 +323,7 @@ describe('Cache tests', () => { response = await defaultCachedFetch(TWO_HUNDRED_URL, post(s1)); assert.strictEqual(response.returnedFromCache, true); }); - + it('Gives different form data different cache keys', async () => { const data1 = new FormData(); data1.append('a', 'a'); @@ -400,6 +401,15 @@ describe('Cache tests', () => { }).timeout(10_000); describe('Data tests', () => { + it('Can clone a response and read both', async () => { + const response = await defaultCachedFetch(TEXT_BODY_URL); + const clonedResponse = response.clone(); + const body1 = await response.text(); + const body2 = await clonedResponse.text(); + assert.strictEqual(body1, TEXT_BODY_EXPECTED); + assert.strictEqual(body2, TEXT_BODY_EXPECTED); + }); + it('Supports request objects', async () => { let request = new StandardFetchRequest('https://google.com', { body: 'test', method: 'POST' }); response = await defaultCachedFetch(request); @@ -424,7 +434,7 @@ describe('Data tests', () => { it('Refuses to consume body twice', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); await response.text(); - await assert.rejects(async () => response.text(), /body used already for:/); + await assert.rejects(async () => response.text(), /Body has already been read/); }); it('Can get text body', async () => { @@ -453,35 +463,25 @@ describe('Data tests', () => { it('Can get PNG buffer body', async () => { response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); it('Can stream a body', async () => { response = await defaultCachedFetch(TEXT_BODY_URL); - let body = ''; - - for await (const chunk of response.body!) { - body += chunk.toString(); - } - assert.strictEqual(TEXT_BODY_EXPECTED, body); + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(TEXT_BODY_URL); - body = ''; - - for await (const chunk of response.body!) { - body += chunk.toString(); - } - - assert.strictEqual(TEXT_BODY_EXPECTED, body); + + assert.strictEqual(TEXT_BODY_EXPECTED, Buffer.concat(await Array.fromAsync(response.body)).toString()); assert.strictEqual(response.returnedFromCache, true); }); @@ -495,7 +495,7 @@ describe('Data tests', () => { it('Errors if the resource type is not supported', async () => { await assert.rejects( async () => defaultCachedFetch(1 as unknown as string), - /The first argument to fetch must be either a string or a node-fetch Request instance/, + /The first argument to fetch must be either a string or a fetch Request instance/, ); }); @@ -582,12 +582,12 @@ describe('File system cache tests', () => { it('Can get PNG buffer body', async () => { defaultCachedFetch = FetchCache.create({ cache: new FileSystemCache() }); response = await defaultCachedFetch(PNG_BODY_URL); - const body1 = await response.buffer(); + const body1 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body1), true); assert.strictEqual(response.returnedFromCache, false); response = await defaultCachedFetch(PNG_BODY_URL); - const body2 = await response.buffer(); + const body2 = new Uint8Array(await response.arrayBuffer()); assert.strictEqual(expectedPngBuffer.equals(body2), true); assert.strictEqual(response.returnedFromCache, true); }); @@ -747,7 +747,6 @@ describe('Cache strategy tests', () => { const functionsThatUseResponse = [ 'arrayBuffer', 'blob', - 'buffer', 'json', 'text', ] as const; @@ -784,6 +783,6 @@ describe('Cache strategy tests', () => { describe('Network error tests', () => { it('Bubbles up network errors', async () => { - await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /^FetchError:/); + await assert.rejects(async () => defaultCachedFetch('http://localhost:1'), /TypeError: fetch failed/); }); }); diff --git a/tsconfig.json b/tsconfig.json index d74dbf7..03d1d61 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,7 +12,7 @@ /* Language and Environment */ "target": "ES2022", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ - // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + "lib": ["DOM", "DOM.Iterable", "ESNext"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ @@ -25,9 +25,9 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "ES2022", /* Specify what module code is generated. */ + "module": "ES2022", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "Node", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ @@ -49,13 +49,13 @@ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ - "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./dist", /* Specify an output folder for all emitted files. */ + "outDir": "./dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ @@ -83,24 +83,24 @@ /* Type Checking */ "strict": true, /* Enable all strict type-checking options. */ - "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ - "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ - "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ - "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ - "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ - "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ - "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ - "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ - "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ @@ -109,5 +109,6 @@ "ts-node": { "esm": true, }, - "exclude": ["test", "dist", "plugins"] + "include": ["src/**/*.ts"], + "exclude": ["dist", "plugins", "node_modules"] }