Support custom fetch function (#46)

Author: jeswr

lib/RdfDereferencerBase.ts

@@ -2,6 +2,7 @@ import {IActorDereferenceRdfOutput} from "@comunica/bus-dereference-rdf";
 import { ActionContext, Actor } from "@comunica/core";
 import * as RDF from "@rdfjs/types";
 import { MediatorDereferenceRdf } from '@comunica/bus-dereference-rdf';
+import { KeysHttp } from '@comunica/context-entries';
 
 /**
  * An RdfDerefencer can dereference URLs to RDF streams, using any RDF serialization.
@@ -21,9 +22,10 @@ export class RdfDereferencerBase<Q extends RDF.BaseQuad = RDF.Quad> {
    * @return {IActorRdfDereferenceOutput} The dereference output.
    */
   public dereference(url: string, options: IDereferenceOptions = {}): Promise<IActorDereferenceRdfOutput> {
+    const context = new ActionContext(options);
     // Delegate dereferencing to the mediator
     return this.mediatorDereferenceRdf.mediate({
-      context: new ActionContext(options),
+      context: typeof options.fetch === 'function' ? context.setDefault(KeysHttp.fetch, options.fetch) : context,
       headers: new Headers(options.headers),
       method: options.method,
       url,
@@ -47,6 +49,10 @@ export interface IDereferenceOptions {
    * This is not possible in browser environments.
    */
   localFiles?: boolean;
+  /**
+   * The fetch function to use.
+   */
+  fetch?: typeof fetch;
 }
 
 export interface IRdfDerefencerArgs {

package.json

@@ -43,6 +43,7 @@
   ],
   "devDependencies": {
     "@comunica/runner": "^2.0.3",
+    "@comunica/types": "^2.8.1",
     "@types/jest": "^27.0.0",
     "@types/n3": "^1.10.3",
     "arrayify-stream": "^1.0.0",
@@ -116,6 +117,7 @@
     "@comunica/bus-rdf-parse": "^2.0.1",
     "@comunica/bus-rdf-parse-html": "^2.0.1",
     "@comunica/config-query-sparql": "^2.0.1",
+    "@comunica/context-entries": "^2.8.1",
     "@comunica/core": "^2.0.1",
     "@comunica/mediator-combine-pipeline": "^2.0.1",
     "@comunica/mediator-combine-union": "^2.0.1",

test/RdfDereferencer-test.ts

@@ -15,18 +15,23 @@ let fetchOut: any;
   return fetchOut;
 });
 
-const mockSetup = (options: { statusCode?: number, error?: boolean, body?: Readable, headers?: any, url?: string }) => {
+const getMock = (options: { statusCode?: number, error?: boolean, body?: Readable, headers?: any, url?: string }) => {
   if (options.error) {
-    fetchOut = Promise.reject(new Error('fetch error'));
+    return Promise.reject(new Error('fetch error'));
   } else {
-    fetchOut = Promise.resolve({
+    return Promise.resolve({
       url: options.url || 'http://example.org/',
       status: options.statusCode || 200,
       body: options.body,
       headers: new Headers(options.headers || {}),
     });
   }
 };
+
+const mockSetup = (...args: Parameters<typeof getMock>) => {
+  fetchOut = getMock(...args);
+};
+
 mockSetup({ statusCode: 200 });
 
 describe('dereferencer', () => {
@@ -66,6 +71,24 @@ describe('dereferencer', () => {
     ]);
   });
 
+  it('should use custom fetch function when provided', async () => {
+    const body = new Readable();
+    body.push(`
+<http://ex.org/s> <http://ex.org/p> <http://ex.org/o1>, <http://ex.org/o2>.
+`);
+    body.push(null);
+
+    const myFetch: typeof fetch = async (url) => getMock({ statusCode: 200, body, headers: { 'content-type': 'text/turtle' } }) as any;
+
+    const out = await dereferencer.dereference('http://example.org/', { fetch: myFetch });
+    expect(out.metadata.triples).toBeTruthy();
+    expect(out.url).toEqual('http://example.org/');
+    return expect(arrayifyStream(out.data)).resolves.toBeRdfIsomorphic([
+      quad('http://ex.org/s', 'http://ex.org/p', 'http://ex.org/o1'),
+      quad('http://ex.org/s', 'http://ex.org/p', 'http://ex.org/o2'),
+    ]);
+  });
+
   it('should handle text/turtle responses by extension', async () => {
     const body = new Readable();
     body.push(`

yarn.lock

@@ -595,6 +595,17 @@
     jsonld-context-parser "^2.2.2"
     sparqlalgebrajs "^4.0.5"
 
+"@comunica/context-entries@^2.8.1":
+  version "2.8.1"
+  resolved "https://registry.yarnpkg.com/@comunica/context-entries/-/context-entries-2.8.1.tgz#7f5af0e7079d93a2bc609e82bc2daec480b96338"
+  integrity sha512-RjRgAehp9Ujaaqvm/5PJd11l4VHUNqrApV/6ZpjdPbWsJKdMo2mpjyjelEn48n2kid+TXK4a1iKF3lwH+mjRvw==
+  dependencies:
+    "@comunica/core" "^2.8.1"
+    "@comunica/types" "^2.8.1"
+    "@rdfjs/types" "*"
+    jsonld-context-parser "^2.2.2"
+    sparqlalgebrajs "^4.2.0"
+
 "@comunica/core@^2.0.1":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@comunica/core/-/core-2.0.1.tgz#69307bddb182f3402a948d605919c79d2bdbd641"
@@ -611,6 +622,14 @@
     "@comunica/types" "^2.6.0"
     immutable "^4.1.0"
 
+"@comunica/core@^2.8.1":
+  version "2.8.1"
+  resolved "https://registry.yarnpkg.com/@comunica/core/-/core-2.8.1.tgz#80207da1bf03a49ed088522160f6d11d13663954"
+  integrity sha512-/rcRdtM0chk6L1ZaZgz84HUbGkFGehR1uAM8Mj0UDfcSZjpV1qUedfF5ktmOA2Er3lz9DZ05Xotgxw6zivILWw==
+  dependencies:
+    "@comunica/types" "^2.8.1"
+    immutable "^4.1.0"
+
 "@comunica/mediator-combine-pipeline@^2.0.1":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@comunica/mediator-combine-pipeline/-/mediator-combine-pipeline-2.0.1.tgz#afbd88b31405b39193807264ff1bf3b341e688ac"
@@ -675,6 +694,16 @@
     asynciterator "^3.8.0"
     sparqlalgebrajs "^4.0.5"
 
+"@comunica/types@^2.8.1":
+  version "2.8.1"
+  resolved "https://registry.yarnpkg.com/@comunica/types/-/types-2.8.1.tgz#f0e81b78ac8767eddc9f03b2342389ff21ec959b"
+  integrity sha512-uRBKBYpG1RVuQw8hfHHE1jdiCt6FTDeM2r4YqFY2OSOfxjrIDBYkcqsgod1m5sFs3l/P6CVVZG9w6uZ7876Y8w==
+  dependencies:
+    "@rdfjs/types" "*"
+    "@types/yargs" "^17.0.13"
+    asynciterator "^3.8.1"
+    sparqlalgebrajs "^4.2.0"
+
 "@dabh/diagnostics@^2.0.2":
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.3.tgz#7f7e97ee9a725dffc7808d93668cc984e1dc477a"
@@ -1458,6 +1487,11 @@ asynciterator@^3.8.0:
   resolved "https://registry.yarnpkg.com/asynciterator/-/asynciterator-3.8.0.tgz#d9246a3d39ce2a9e4b020834d7f352adf35b49d8"
   integrity sha512-bD34LqKHJnkB77MHjL3hOAUOcy9dbB+3lHvL+EiJpD3k2Nyq3i1dCk5adMisB2rwlrHVu/+XRhOdPZL9hzpsfw==
 
+asynciterator@^3.8.1:
+  version "3.8.1"
+  resolved "https://registry.yarnpkg.com/asynciterator/-/asynciterator-3.8.1.tgz#80be735b252332494e186ee733544e5b21dd2123"
+  integrity sha512-SmdG0FUY3pYGOZZGdYq8Qb/DCRDXBFZUk08V1/4lbBXdAQvcC3Kxzz9FUDPBTik7VAVltt4cZirAPtJv3gOpEw==
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -3851,6 +3885,13 @@ rdf-data-factory@^1.1.1:
   dependencies:
     "@rdfjs/types" "*"
 
+rdf-data-factory@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/rdf-data-factory/-/rdf-data-factory-1.1.2.tgz#d47550d2649d0d64f8cae3fcc9efae7a8a895d9a"
+  integrity sha512-TfQD63Lokabd09ES1jAtKK8AA6rkr9rwyUBGo6olOt1CE0Um36CUQIqytyf0am2ouBPR0l7SaHxCiMcPGHkt1A==
+  dependencies:
+    "@rdfjs/types" "*"
+
 rdf-isomorphic@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/rdf-isomorphic/-/rdf-isomorphic-1.3.0.tgz#6a1ad38b27c5ebed8a5f752b15f81988f48f8e22"
@@ -3932,6 +3973,15 @@ rdf-string@^1.5.0, rdf-string@^1.6.0:
     "@rdfjs/types" "*"
     rdf-data-factory "^1.1.0"
 
+rdf-terms@^1.10.0:
+  version "1.11.0"
+  resolved "https://registry.yarnpkg.com/rdf-terms/-/rdf-terms-1.11.0.tgz#0c2e3a2b43f1042959c9263af27dab08dc4b084d"
+  integrity sha512-iKlVgnMopRKl9pHVNrQrax7PtZKRCT/uJIgYqvuw1VVQb88zDvurtDr1xp0rt7N9JtKtFwUXoIQoEsjyRo20qQ==
+  dependencies:
+    "@rdfjs/types" "*"
+    rdf-data-factory "^1.1.0"
+    rdf-string "^1.6.0"
+
 rdf-terms@^1.7.0:
   version "1.7.1"
   resolved "https://registry.yarnpkg.com/rdf-terms/-/rdf-terms-1.7.1.tgz#d53891b208b357ed4b7738134decd537130098e6"
@@ -4301,6 +4351,21 @@ sparqlalgebrajs@^4.0.5:
     rdf-string "^1.6.0"
     sparqljs "^3.6.1"
 
+sparqlalgebrajs@^4.2.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/sparqlalgebrajs/-/sparqlalgebrajs-4.2.0.tgz#b70945bdd2ab2897a0bc22adb804dd1a2b0ec98c"
+  integrity sha512-tdlJdrvgQqgx9zubcl9iiyCxMOp4qRT2fs1Sne8X35QTm1Pj2ulB+gGEHunJJnw5FW7Uhtmw7J3px0sCmgJSbw==
+  dependencies:
+    "@rdfjs/types" "*"
+    "@types/sparqljs" "^3.1.3"
+    fast-deep-equal "^3.1.3"
+    minimist "^1.2.6"
+    rdf-data-factory "^1.1.0"
+    rdf-isomorphic "^1.3.0"
+    rdf-string "^1.6.0"
+    rdf-terms "^1.10.0"
+    sparqljs "^3.7.1"
+
 sparqljs@^3.5.1:
   version "3.5.1"
   resolved "https://registry.yarnpkg.com/sparqljs/-/sparqljs-3.5.1.tgz#631cd4b5bcc3d9ce23c5313b3a158eccf858e28b"
@@ -4315,6 +4380,13 @@ sparqljs@^3.6.1:
   dependencies:
     rdf-data-factory "^1.1.1"
 
+sparqljs@^3.7.1:
+  version "3.7.1"
+  resolved "https://registry.yarnpkg.com/sparqljs/-/sparqljs-3.7.1.tgz#5d121895d491d50214f2e38f2885a3a935b6c093"
+  integrity sha512-I1jYMtcwDkgCEqQ4eQuQIhB8hFAlRAJ6YDXDcV54XztaJaYRFqJlidHt77S3j8Mfh6kY6GK04dXPEIopxbEeuQ==
+  dependencies:
+    rdf-data-factory "^1.1.2"
+
 spawn-sync@^1.0.15:
   version "1.0.15"
   resolved "https://registry.yarnpkg.com/spawn-sync/-/spawn-sync-1.0.15.tgz#b00799557eb7fb0c8376c29d44e8a1ea67e57476"