From 6e3cbbdd4b1fe377ed031ae9f67decf45e2580de Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 11:36:28 -0500 Subject: [PATCH 01/27] Added support for external parsers to bodyParser.json() --- README.md | 9 +++++++-- lib/types/json.js | 9 +++++---- package.json | 2 +- test/json.js | 11 +++++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1eebdffd..76292825 100644 --- a/README.md +++ b/README.md @@ -88,16 +88,21 @@ specifies the number of bytes; if it is a string, the value is passed to the [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults to `'100kb'`. +##### parser + +The `parser` option is the function called against the request body to convert +it to a Javascript object. Defaults to `JSON.parse`. + ##### reviver -The `reviver` option is passed directly to `JSON.parse` as the second +The `reviver` option is passed directly to the `parser` as the second argument. You can find more information on this argument [in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter). ##### strict When set to `true`, will only accept arrays and objects; when `false` will -accept anything `JSON.parse` accepts. Defaults to `true`. +accept anything the `parser` accepts. Defaults to `true`. ##### type diff --git a/lib/types/json.js b/lib/types/json.js index 30bf8cab..11756231 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -62,6 +62,7 @@ function json (options) { var strict = opts.strict !== false var type = opts.type || 'application/json' var verify = opts.verify || false + var parser = opts.parser || JSON.parse if (verify !== false && typeof verify !== 'function') { throw new TypeError('option verify must be function') @@ -84,13 +85,13 @@ function json (options) { if (first !== '{' && first !== '[') { debug('strict violation') - throw createStrictSyntaxError(body, first) + throw createStrictSyntaxError(parser, body, first) } } try { debug('parse json') - return JSON.parse(body, reviver) + return parser(body, reviver) } catch (e) { throw normalizeJsonSyntaxError(e, { message: e.message, @@ -156,7 +157,7 @@ function json (options) { * @private */ -function createStrictSyntaxError (str, char) { +function createStrictSyntaxError (parser, str, char) { var index = str.indexOf(char) var partial = '' @@ -169,7 +170,7 @@ function createStrictSyntaxError (str, char) { } try { - JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation') + parser(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation') } catch (e) { return normalizeJsonSyntaxError(e, { message: e.message.replace(JSON_SYNTAX_REGEXP, function (placeholder) { diff --git a/package.json b/package.json index 5b35fbf3..a4f29891 100644 --- a/package.json +++ b/package.json @@ -48,4 +48,4 @@ "test-ci": "nyc --reporter=lcov --reporter=text npm test", "test-cov": "nyc --reporter=html --reporter=text npm test" } -} +} \ No newline at end of file diff --git a/test/json.js b/test/json.js index 3b5cc653..dbb0c47d 100644 --- a/test/json.js +++ b/test/json.js @@ -4,6 +4,7 @@ var assert = require('assert') var AsyncLocalStorage = require('async_hooks').AsyncLocalStorage var http = require('http') var request = require('supertest') +var JSONbig = require('json-bigint') var bodyParser = require('..') @@ -94,6 +95,16 @@ describe('bodyParser.json()', function () { .expect(200, '{"user":"tobi"}', done) }) + it('should use external parsers', function (done) { + request(createServer({ + parser: JSONbig({ storeAsString: true }).parse + })) + .post('/') + .set('Content-Type', 'application/json') + .send('{"user_id":1234567890123456789012345678901234567890123456789012345678901234567890}') + .expect(200, '{"user_id":"1234567890123456789012345678901234567890123456789012345678901234567890"}', done) + }) + describe('when JSON is invalid', function () { before(function () { this.server = createServer() From 81bcdb8be70cf5db7a1e98848ce19161703cd7c9 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 11:54:20 -0500 Subject: [PATCH 02/27] removed test dependency on json-bigint --- package.json | 2 +- test/json.js | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index a4f29891..3d9b7852 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,4 @@ -{ +j{ "name": "body-parser", "description": "Node.js body parsing middleware", "version": "2.0.2", diff --git a/test/json.js b/test/json.js index dbb0c47d..88c6df83 100644 --- a/test/json.js +++ b/test/json.js @@ -4,7 +4,6 @@ var assert = require('assert') var AsyncLocalStorage = require('async_hooks').AsyncLocalStorage var http = require('http') var request = require('supertest') -var JSONbig = require('json-bigint') var bodyParser = require('..') @@ -97,12 +96,14 @@ describe('bodyParser.json()', function () { it('should use external parsers', function (done) { request(createServer({ - parser: JSONbig({ storeAsString: true }).parse + parser: function (body) { + return { foo: "bar" } + } })) .post('/') .set('Content-Type', 'application/json') - .send('{"user_id":1234567890123456789012345678901234567890123456789012345678901234567890}') - .expect(200, '{"user_id":"1234567890123456789012345678901234567890123456789012345678901234567890"}', done) + .send('{"str":') + .expect(200, '{"foo":"bar"}', done) }) describe('when JSON is invalid', function () { From c1ed1bc849636445b6f65906065e4ccf413e92ae Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 15:43:42 -0500 Subject: [PATCH 03/27] reworked doc to describe json parser() func better --- README.md | 12 +++++++++--- lib/types/json.js | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 76292825..8faf21ab 100644 --- a/README.md +++ b/README.md @@ -91,12 +91,18 @@ to `'100kb'`. ##### parser The `parser` option is the function called against the request body to convert -it to a Javascript object. Defaults to `JSON.parse`. +it to a Javascript object. If a `reviver` is supplied, it is supplied as the +second argument to this function. + +```javascript +parser(body, reviver) -> req.body +``` + +Defaults to `JSON.parse`. ##### reviver -The `reviver` option is passed directly to the `parser` as the second -argument. You can find more information on this argument +You can find more information on this argument [in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter). ##### strict diff --git a/lib/types/json.js b/lib/types/json.js index 11756231..9332fcc8 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -85,7 +85,7 @@ function json (options) { if (first !== '{' && first !== '[') { debug('strict violation') - throw createStrictSyntaxError(parser, body, first) + throw createStrictSyntaxError(parser, reviver, body, first) } } @@ -157,7 +157,7 @@ function json (options) { * @private */ -function createStrictSyntaxError (parser, str, char) { +function createStrictSyntaxError (parser, reviver, str, char) { var index = str.indexOf(char) var partial = '' @@ -170,7 +170,7 @@ function createStrictSyntaxError (parser, str, char) { } try { - parser(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation') + parser(partial, reviver); /* istanbul ignore next */ throw new SyntaxError('strict violation') } catch (e) { return normalizeJsonSyntaxError(e, { message: e.message.replace(JSON_SYNTAX_REGEXP, function (placeholder) { From 36fdeee84d8629ed55e2c9f6b91085d9cbbde485 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 15:45:41 -0500 Subject: [PATCH 04/27] added parser() option and doc for .text() --- README.md | 9 +++++++++ lib/types/text.js | 7 ++++++- test/text.js | 10 ++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8faf21ab..d4609e8e 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,15 @@ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)` where `buf` is a `Buffer` of the raw request body and `encoding` is the encoding of the request. The parsing can be aborted by throwing an error. +##### parser + +The `parser` option, if supplied, is used to transform the body of a request +before being set to `req.body`. + +```javascript +parser(body) -> req.body +``` + ### bodyParser.urlencoded([options]) Returns middleware that only parses `urlencoded` bodies and only looks at diff --git a/lib/types/text.js b/lib/types/text.js index b153931b..b2f57423 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -41,6 +41,7 @@ function text (options) { : opts.limit var type = opts.type || 'text/plain' var verify = opts.verify || false + var parser = opts.parser || defaultParser if (verify !== false && typeof verify !== 'function') { throw new TypeError('option verify must be function') @@ -52,7 +53,7 @@ function text (options) { : type function parse (buf) { - return buf + return parser(buf) } return function textParser (req, res, next) { @@ -95,6 +96,10 @@ function text (options) { } } +function defaultParser (buf) { + return buf; +} + /** * Get the charset of a request. * diff --git a/test/text.js b/test/text.js index 021612aa..3dbce3ef 100644 --- a/test/text.js +++ b/test/text.js @@ -20,6 +20,16 @@ describe('bodyParser.text()', function () { .expect(200, '"user is tobi"', done) }) + it('should parse text/plain with a custom parser', function (done) { + request(createServer({ + parser: function (input) { return input.toUpperCase() } + })) + .post('/') + .set('Content-Type', 'text/plain') + .send('user is tobi') + .expect(200, '"USER IS TOBI"', done) + }) + it('should 400 when invalid content-length', function (done) { var textParser = bodyParser.text() var server = createServer(function (req, res, next) { From e0c6d1afea2fac68014fb7b52cdef1cd758f5608 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 15:51:22 -0500 Subject: [PATCH 05/27] added parser() option and doc for .raw() --- README.md | 9 +++++++++ lib/types/raw.js | 7 ++++++- test/raw.js | 10 ++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d4609e8e..cbff0513 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,15 @@ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)` where `buf` is a `Buffer` of the raw request body and `encoding` is the encoding of the request. The parsing can be aborted by throwing an error. +##### parser + +The `parser` option, if supplied, is used to transform the body of a request +before being set to `req.body`. + +```javascript +parser(body) -> req.body +``` + ### bodyParser.text([options]) Returns middleware that parses all bodies as a string and only looks at diff --git a/lib/types/raw.js b/lib/types/raw.js index bfe274cf..c1951a5f 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -39,6 +39,7 @@ function raw (options) { : opts.limit var type = opts.type || 'application/octet-stream' var verify = opts.verify || false + var parser = opts.parser || defaultParser if (verify !== false && typeof verify !== 'function') { throw new TypeError('option verify must be function') @@ -50,7 +51,7 @@ function raw (options) { : type function parse (buf) { - return buf + return parser(buf) } return function rawParser (req, res, next) { @@ -90,6 +91,10 @@ function raw (options) { } } +function defaultParser (buf) { + return buf; +} + /** * Get the simple type checker. * diff --git a/test/raw.js b/test/raw.js index 3f9a7284..94efb46c 100644 --- a/test/raw.js +++ b/test/raw.js @@ -20,6 +20,16 @@ describe('bodyParser.raw()', function () { .expect(200, 'buf:746865207573657220697320746f6269', done) }) + it('should parse application/octet-stream with a custom parser', function (done) { + request(createServer({ + parser: function (body) { return body.toString("utf8") } + })) + .post('/') + .set('Content-Type', 'application/octet-stream') + .send('the user is tobi') + .expect(200, '"the user is tobi"', done) + }) + it('should 400 when invalid content-length', function (done) { var rawParser = bodyParser.raw() var server = createServer(function (req, res, next) { From 00078076c8d26c7e967f94a7b85d43319bb380be Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 16:05:23 -0500 Subject: [PATCH 06/27] added parser() option and doc for .urlencoded() --- README.md | 10 ++++++++++ lib/types/urlencoded.js | 8 ++++++-- test/urlencoded.js | 10 ++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cbff0513..179f0ada 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,16 @@ form. Defaults to `false`. The `depth` option is used to configure the maximum depth of the `qs` library when `extended` is `true`. This allows you to limit the amount of keys that are parsed and can be useful to prevent certain types of abuse. Defaults to `32`. It is recommended to keep this value as low as possible. +##### parser + +The `parser` option, if supplied, is used to in place of the default parser to +convert the request body into a Javascript object. If this option is supplied, +both the `extended` and `parameterLimit` options are ignored. + +```javascript +parser(body) -> req.body +``` + ## Errors The middlewares provided by this module create errors using the diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 687745f8..93556d9e 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -58,7 +58,11 @@ function urlencoded (options) { } // create the appropriate query parser - var queryparse = createQueryParser(opts, extended) + var parser = opts.parser || ( + extended + ? extendedparser(opts) + : simpleparser(opts) + ) // create the appropriate type checking function var shouldParse = typeof type !== 'function' @@ -67,7 +71,7 @@ function urlencoded (options) { function parse (body, encoding) { return body.length - ? queryparse(body, encoding) + ? parser(body) : {} } diff --git a/test/urlencoded.js b/test/urlencoded.js index dfe0eb9f..88d29ce5 100644 --- a/test/urlencoded.js +++ b/test/urlencoded.js @@ -20,6 +20,16 @@ describe('bodyParser.urlencoded()', function () { .expect(200, '{"user":"tobi"}', done) }) + it('should parse x-www-form-urlencoded with custom parser', function (done) { + request(createServer({ + parser: function (input) { return input.toUpperCase() } + })) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '"USER=TOBI"', done) + }) + it('should 400 when invalid content-length', function (done) { var urlencodedParser = bodyParser.urlencoded() var server = createServer(function (req, res, next) { From 38c1c3fab59ad84cfa0e398a340833e15da918db Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 16:10:37 -0500 Subject: [PATCH 07/27] cleanup to satisfy linter --- README.md | 8 ++++---- lib/types/raw.js | 2 +- lib/types/text.js | 2 +- test/json.js | 2 +- test/raw.js | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 179f0ada..8a71c5a0 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ The `parser` option is the function called against the request body to convert it to a Javascript object. If a `reviver` is supplied, it is supplied as the second argument to this function. -```javascript +``` parser(body, reviver) -> req.body ``` @@ -178,7 +178,7 @@ encoding of the request. The parsing can be aborted by throwing an error. The `parser` option, if supplied, is used to transform the body of a request before being set to `req.body`. -```javascript +``` parser(body) -> req.body ``` @@ -237,7 +237,7 @@ encoding of the request. The parsing can be aborted by throwing an error. The `parser` option, if supplied, is used to transform the body of a request before being set to `req.body`. -```javascript +``` parser(body) -> req.body ``` @@ -330,7 +330,7 @@ The `parser` option, if supplied, is used to in place of the default parser to convert the request body into a Javascript object. If this option is supplied, both the `extended` and `parameterLimit` options are ignored. -```javascript +``` parser(body) -> req.body ``` diff --git a/lib/types/raw.js b/lib/types/raw.js index c1951a5f..5710705e 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -92,7 +92,7 @@ function raw (options) { } function defaultParser (buf) { - return buf; + return buf } /** diff --git a/lib/types/text.js b/lib/types/text.js index b2f57423..e34d05b1 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -97,7 +97,7 @@ function text (options) { } function defaultParser (buf) { - return buf; + return buf } /** diff --git a/test/json.js b/test/json.js index 88c6df83..208e5a5e 100644 --- a/test/json.js +++ b/test/json.js @@ -97,7 +97,7 @@ describe('bodyParser.json()', function () { it('should use external parsers', function (done) { request(createServer({ parser: function (body) { - return { foo: "bar" } + return { foo: 'bar' } } })) .post('/') diff --git a/test/raw.js b/test/raw.js index 94efb46c..2381f908 100644 --- a/test/raw.js +++ b/test/raw.js @@ -22,7 +22,7 @@ describe('bodyParser.raw()', function () { it('should parse application/octet-stream with a custom parser', function (done) { request(createServer({ - parser: function (body) { return body.toString("utf8") } + parser: function (body) { return body.toString('utf8') } })) .post('/') .set('Content-Type', 'application/octet-stream') From 30574441e7a036b31e1bf350b11fb649e9fc78a2 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:07:58 -0500 Subject: [PATCH 08/27] added generic parser --- lib/generic-parser.js | 156 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 lib/generic-parser.js diff --git a/lib/generic-parser.js b/lib/generic-parser.js new file mode 100644 index 00000000..858303d8 --- /dev/null +++ b/lib/generic-parser.js @@ -0,0 +1,156 @@ +/*! + * body-parser + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var createError = require('http-errors') +var debug = require('debug')('body-parser:generic') +var read = require('./read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = generic + + +/** + * Create a middleware to parse JSON bodies. + * + * @param {object} [options] + * @return {function} + * @public + */ + +function generic (options) { + var opts = options || {} + + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var charset = opts.charset + var inflate = opts.inflate !== false + var verify = opts.verify || false + var parse = opts.parse || defaultParse + var defaultReqCharset = opts.defaultCharset || 'utf-8' + var type = opts.type + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + // create the appropriate charset validating function + var validCharset = typeof charset !== "function" + ? charsetValidator(charset) + : charset + + return function genericParser (req, res, next) { + if (req._body) { + debug('body already parsed') + next() + return + } + + req.body = req.body || {} + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // assert charset per RFC 7159 sec 8.1 + var reqCharset = null + if (charset !== undefined) { + reqCharset = getCharset(req) || defaultReqCharset + if (!validCharset(reqCharset)) { + debug('invalid charset') + next(createError(415, 'unsupported charset "' + reqCharset.toUpperCase() + '"', { + charset: reqCharset, + type: 'charset.unsupported' + })) + return + } + } + + // read + read(req, res, next, parse, debug, { + encoding: reqCharset, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +function defaultParse (buf) { + return buf +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return (contentType.parse(req).parameters.charset || '').toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} + +/** + * Get the simple charset validator. + * + * @param {string} type + * @return {function} + */ + +function charsetValidator (charset) { + return function validateCharset (reqCharset) { + return charset === reqCharset + } +} From 825a78bc02d7dd183838f11dba743c0de31177bd Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:08:56 -0500 Subject: [PATCH 09/27] converted json parser to use generic parser --- lib/types/json.js | 146 +++++++++------------------------------------- 1 file changed, 26 insertions(+), 120 deletions(-) diff --git a/lib/types/json.js b/lib/types/json.js index 9332fcc8..2220524e 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -12,13 +12,9 @@ * @private */ -var bytes = require('bytes') -var contentType = require('content-type') -var createError = require('http-errors') +var assign = require('object-assign') +var genericParser = require('../generic-parser') var debug = require('debug')('body-parser:json') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') /** * Module exports. @@ -51,101 +47,39 @@ var JSON_SYNTAX_REGEXP = /#+/g * @public */ -function json (options) { +function json(options) { var opts = options || {} - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit - var inflate = opts.inflate !== false var reviver = opts.reviver var strict = opts.strict !== false - var type = opts.type || 'application/json' - var verify = opts.verify || false var parser = opts.parser || JSON.parse + var type = opts.type || 'application/json' - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (body) { - if (body.length === 0) { - // special-case empty json body, as it's a common client-side mistake - // TODO: maybe make this configurable or part of "strict" option - return {} - } + return genericParser(assign({}, opts, { + type: type, - if (strict) { - var first = firstchar(body) + charset: function validateCharset(charset) { + return charset.slice(0, 4) === 'utf-' + }, - if (first !== '{' && first !== '[') { - debug('strict violation') - throw createStrictSyntaxError(parser, reviver, body, first) + parse: function parse(buf) { + if (buf.length === 0) { + // special-case empty json body, as it's a common client-side mistake + // TODO: maybe make this configurable or part of "strict" option + return {} } - } - - try { - debug('parse json') - return parser(body, reviver) - } catch (e) { - throw normalizeJsonSyntaxError(e, { - message: e.message, - stack: e.stack - }) - } - } - return function jsonParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // assert charset per RFC 7159 sec 8.1 - var charset = getCharset(req) || 'utf-8' - if (charset.slice(0, 4) !== 'utf-') { - debug('invalid charset') - next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { - charset: charset, - type: 'charset.unsupported' - })) - return + try { + debug('parse json') + return parser(body, reviver) + } catch (e) { + throw normalizeJsonSyntaxError(e, { + message: e.message, + stack: e.stack + }) + } } - - // read - read(req, res, next, parse, debug, { - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } + })) } /** @@ -157,7 +91,7 @@ function json (options) { * @private */ -function createStrictSyntaxError (parser, reviver, str, char) { +function createStrictSyntaxError(parser, reviver, str, char) { var index = str.indexOf(char) var partial = '' @@ -197,21 +131,6 @@ function firstchar (str) { : undefined } -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - /** * Normalize a SyntaxError for JSON.parse. * @@ -220,7 +139,7 @@ function getCharset (req) { * @return {SyntaxError} */ -function normalizeJsonSyntaxError (error, obj) { +function normalizeJsonSyntaxError(error, obj) { var keys = Object.getOwnPropertyNames(error) for (var i = 0; i < keys.length; i++) { @@ -236,16 +155,3 @@ function normalizeJsonSyntaxError (error, obj) { return error } - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } -} From bd0601b8a0bc273d0f5de11b9406aa543060fe73 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:10:01 -0500 Subject: [PATCH 10/27] converted raw parser to use generic parser --- lib/generic-parser.js | 7 ++-- lib/types/raw.js | 81 +++---------------------------------------- test/raw.js | 10 ------ 3 files changed, 10 insertions(+), 88 deletions(-) diff --git a/lib/generic-parser.js b/lib/generic-parser.js index 858303d8..92dab429 100644 --- a/lib/generic-parser.js +++ b/lib/generic-parser.js @@ -16,6 +16,7 @@ var bytes = require('bytes') var contentType = require('content-type') var createError = require('http-errors') var debug = require('debug')('body-parser:generic') +var isFinished = require('on-finished').isFinished var read = require('./read') var typeis = require('type-is') @@ -62,13 +63,15 @@ function generic (options) { : charset return function genericParser (req, res, next) { - if (req._body) { + if (isFinished(req)) { debug('body already parsed') next() return } - req.body = req.body || {} + if (!('body' in req)) { + req.body = undefined + } // skip requests without bodies if (!typeis.hasBody(req)) { diff --git a/lib/types/raw.js b/lib/types/raw.js index 5710705e..0e8b359c 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -10,11 +10,8 @@ * Module dependencies. */ -var bytes = require('bytes') -var debug = require('debug')('body-parser:raw') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var assign = require('object-assign') +var genericParser = require('../generic-parser') /** * Module exports. @@ -33,77 +30,9 @@ module.exports = raw function raw (options) { var opts = options || {} - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'application/octet-stream' - var verify = opts.verify || false - var parser = opts.parser || defaultParser - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (buf) { - return parser(buf) - } - - return function rawParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // read - read(req, res, next, parse, debug, { - encoding: null, - inflate: inflate, - limit: limit, - verify: verify - }) - } -} - -function defaultParser (buf) { - return buf -} - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } + return genericParser(assign({}, opts, { + type: type + })) } diff --git a/test/raw.js b/test/raw.js index 2381f908..3f9a7284 100644 --- a/test/raw.js +++ b/test/raw.js @@ -20,16 +20,6 @@ describe('bodyParser.raw()', function () { .expect(200, 'buf:746865207573657220697320746f6269', done) }) - it('should parse application/octet-stream with a custom parser', function (done) { - request(createServer({ - parser: function (body) { return body.toString('utf8') } - })) - .post('/') - .set('Content-Type', 'application/octet-stream') - .send('the user is tobi') - .expect(200, '"the user is tobi"', done) - }) - it('should 400 when invalid content-length', function (done) { var rawParser = bodyParser.raw() var server = createServer(function (req, res, next) { From 149966b589b632876440758bc223565e1b60e535 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:10:46 -0500 Subject: [PATCH 11/27] converted text parser to use generic parser --- lib/types/text.js | 102 ++++------------------------------------------ test/text.js | 10 ----- 2 files changed, 7 insertions(+), 105 deletions(-) diff --git a/lib/types/text.js b/lib/types/text.js index e34d05b1..2956684e 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -10,12 +10,8 @@ * Module dependencies. */ -var bytes = require('bytes') -var contentType = require('content-type') -var debug = require('debug')('body-parser:text') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var assign = require('object-assign') +var genericParser = require('../generic-parser') /** * Module exports. @@ -35,95 +31,11 @@ function text (options) { var opts = options || {} var defaultCharset = opts.defaultCharset || 'utf-8' - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'text/plain' - var verify = opts.verify || false - var parser = opts.parser || defaultParser - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (buf) { - return parser(buf) - } - - return function textParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // get charset - var charset = getCharset(req) || defaultCharset - - // read - read(req, res, next, parse, debug, { - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } -} - -function defaultParser (buf) { - return buf -} - -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } + return genericParser(assign({}, opts, { + type: type, + charset: function validateCharset () { return true }, + defaultCharset: defaultCharset + })) } diff --git a/test/text.js b/test/text.js index 3dbce3ef..021612aa 100644 --- a/test/text.js +++ b/test/text.js @@ -20,16 +20,6 @@ describe('bodyParser.text()', function () { .expect(200, '"user is tobi"', done) }) - it('should parse text/plain with a custom parser', function (done) { - request(createServer({ - parser: function (input) { return input.toUpperCase() } - })) - .post('/') - .set('Content-Type', 'text/plain') - .send('user is tobi') - .expect(200, '"USER IS TOBI"', done) - }) - it('should 400 when invalid content-length', function (done) { var textParser = bodyParser.text() var server = createServer(function (req, res, next) { From 7d7ab1d99b06172eb9d96eab364ae531d7e632d0 Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:11:47 -0500 Subject: [PATCH 12/27] converted urlencoded parser to use generic parser --- lib/types/urlencoded.js | 154 ++++++++++++++++++---------------------- 1 file changed, 70 insertions(+), 84 deletions(-) diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 93556d9e..acf62432 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -12,14 +12,10 @@ * @private */ -var bytes = require('bytes') -var contentType = require('content-type') +var assign = require('object-assign') var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') -var qs = require('qs') +var genericParser = require('../generic-parser.js') /** * Module exports. @@ -34,7 +30,6 @@ module.exports = urlencoded * @return {function} * @public */ - function urlencoded (options) { var opts = options || {} @@ -44,86 +39,24 @@ function urlencoded (options) { ? bytes.parse(opts.limit || '100kb') : opts.limit var type = opts.type || 'application/x-www-form-urlencoded' - var verify = opts.verify || false - var charsetSentinel = opts.charsetSentinel - var interpretNumericEntities = opts.interpretNumericEntities - - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } + var charset = opts.charset || 'utf-8' - var defaultCharset = opts.defaultCharset || 'utf-8' - if (defaultCharset !== 'utf-8' && defaultCharset !== 'iso-8859-1') { - throw new TypeError('option defaultCharset must be either utf-8 or iso-8859-1') - } - - // create the appropriate query parser - var parser = opts.parser || ( + var queryparse = opts.parser || ( extended ? extendedparser(opts) : simpleparser(opts) ) - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (body, encoding) { - return body.length - ? parser(body) - : {} - } - - return function urlencodedParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) + return genericParser(assign({}, opts, { + type: type, + charset: charset, - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return + parse: function parse (buf) { + return buf.length + ? queryparse(buf) + : {} } - - // assert charset - var charset = getCharset(req) || defaultCharset - if (charset !== 'utf-8' && charset !== 'iso-8859-1') { - debug('invalid charset') - next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { - charset: charset, - type: 'charset.unsupported' - })) - return - } - - // read - read(req, res, next, parse, debug, { - debug: debug, - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify, - charsetSentinel: charsetSentinel, - interpretNumericEntities: interpretNumericEntities - }) - } + })) } /** @@ -228,14 +161,67 @@ function parameterCount (body, limit) { } /** - * Get the simple type checker. + * Get parser for module name dynamically. * - * @param {string} type + * @param {string} name * @return {function} + * @api private */ -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) +function parser (name) { + var mod = parsers[name] + + if (mod !== undefined) { + return mod.parse + } + + // this uses a switch for static require analysis + switch (name) { + case 'qs': + mod = require('qs') + break + case 'querystring': + mod = require('querystring') + break + } + + // store to prevent invoking require() + parsers[name] = mod + + return mod.parse +} + +/** + * Get the simple query parser. + * + * @param {object} options + */ + +function simpleparser (options) { + var parameterLimit = options.parameterLimit !== undefined + ? options.parameterLimit + : 1000 + var parse = parser('querystring') + + if (isNaN(parameterLimit) || parameterLimit < 1) { + throw new TypeError('option parameterLimit must be a positive number') + } + + if (isFinite(parameterLimit)) { + parameterLimit = parameterLimit | 0 + } + + return function queryparse (body) { + var paramCount = parameterCount(body, parameterLimit) + + if (paramCount === undefined) { + debug('too many parameters') + throw createError(413, 'too many parameters', { + type: 'parameters.too.many' + }) + } + + debug('parse urlencoding') + return parse(body, undefined, undefined, { maxKeys: parameterLimit }) } } From eb881ef8e583941f5d2b8ed03b48c981acf7818b Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:13:11 -0500 Subject: [PATCH 13/27] cleanup / fix linter warnings --- lib/generic-parser.js | 3 +-- lib/types/urlencoded.js | 15 --------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/lib/generic-parser.js b/lib/generic-parser.js index 92dab429..8ddc52d4 100644 --- a/lib/generic-parser.js +++ b/lib/generic-parser.js @@ -26,7 +26,6 @@ var typeis = require('type-is') module.exports = generic - /** * Create a middleware to parse JSON bodies. * @@ -58,7 +57,7 @@ function generic (options) { : type // create the appropriate charset validating function - var validCharset = typeof charset !== "function" + var validCharset = typeof charset !== 'function' ? charsetValidator(charset) : charset diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index acf62432..755467ef 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -121,21 +121,6 @@ function createQueryParser (options, extended) { } } -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - /** * Count the number of parameters, stopping once limit reached * From 07def4191b329c1ac98858895711f1fd97bb71af Mon Sep 17 00:00:00 2001 From: Shawn Dellysse Date: Tue, 21 Nov 2017 18:15:15 -0500 Subject: [PATCH 14/27] removed items from README --- README.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/README.md b/README.md index 8a71c5a0..394d577c 100644 --- a/README.md +++ b/README.md @@ -173,15 +173,6 @@ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)` where `buf` is a `Buffer` of the raw request body and `encoding` is the encoding of the request. The parsing can be aborted by throwing an error. -##### parser - -The `parser` option, if supplied, is used to transform the body of a request -before being set to `req.body`. - -``` -parser(body) -> req.body -``` - ### bodyParser.text([options]) Returns middleware that parses all bodies as a string and only looks at @@ -232,15 +223,6 @@ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)` where `buf` is a `Buffer` of the raw request body and `encoding` is the encoding of the request. The parsing can be aborted by throwing an error. -##### parser - -The `parser` option, if supplied, is used to transform the body of a request -before being set to `req.body`. - -``` -parser(body) -> req.body -``` - ### bodyParser.urlencoded([options]) Returns middleware that only parses `urlencoded` bodies and only looks at From b4c3c52bd4e439a5394ffc66d7a8a80c63c88b4d Mon Sep 17 00:00:00 2001 From: S Dellysse Date: Thu, 16 Apr 2020 23:38:37 -0400 Subject: [PATCH 15/27] fixed tests after rebase --- lib/types/json.js | 11 ++++++++++- package.json | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/types/json.js b/lib/types/json.js index 2220524e..2261d88d 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -69,9 +69,18 @@ function json(options) { return {} } + if (strict) { + var first = firstchar(buf) + + if (first !== '{' && first !== '[') { + debug('strict violation') + throw createStrictSyntaxError(parser, reviver, buf, first) + } + } + try { debug('parse json') - return parser(body, reviver) + return parser(buf, reviver) } catch (e) { throw normalizeJsonSyntaxError(e, { message: e.message, diff --git a/package.json b/package.json index 3d9b7852..5b35fbf3 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,4 @@ -j{ +{ "name": "body-parser", "description": "Node.js body parsing middleware", "version": "2.0.2", @@ -48,4 +48,4 @@ j{ "test-ci": "nyc --reporter=lcov --reporter=text npm test", "test-cov": "nyc --reporter=html --reporter=text npm test" } -} \ No newline at end of file +} From 1e593377c12f2bac8bf659db79a5c4880ea6a32c Mon Sep 17 00:00:00 2001 From: S Dellysse Date: Thu, 16 Apr 2020 23:50:44 -0400 Subject: [PATCH 16/27] satisfying linter --- lib/types/json.js | 10 +++++----- lib/types/urlencoded.js | 4 ++-- package.json | 2 +- test/json.js | 8 ++++---- test/urlencoded.js | 8 ++++---- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/types/json.js b/lib/types/json.js index 2261d88d..f949382f 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -47,7 +47,7 @@ var JSON_SYNTAX_REGEXP = /#+/g * @public */ -function json(options) { +function json (options) { var opts = options || {} var reviver = opts.reviver @@ -58,11 +58,11 @@ function json(options) { return genericParser(assign({}, opts, { type: type, - charset: function validateCharset(charset) { + charset: function validateCharset (charset) { return charset.slice(0, 4) === 'utf-' }, - parse: function parse(buf) { + parse: function parse (buf) { if (buf.length === 0) { // special-case empty json body, as it's a common client-side mistake // TODO: maybe make this configurable or part of "strict" option @@ -100,7 +100,7 @@ function json(options) { * @private */ -function createStrictSyntaxError(parser, reviver, str, char) { +function createStrictSyntaxError (parser, reviver, str, char) { var index = str.indexOf(char) var partial = '' @@ -148,7 +148,7 @@ function firstchar (str) { * @return {SyntaxError} */ -function normalizeJsonSyntaxError(error, obj) { +function normalizeJsonSyntaxError (error, obj) { var keys = Object.getOwnPropertyNames(error) for (var i = 0; i < keys.length; i++) { diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 755467ef..f6c1d667 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -43,8 +43,8 @@ function urlencoded (options) { var queryparse = opts.parser || ( extended - ? extendedparser(opts) - : simpleparser(opts) + ? extendedparser(opts) + : simpleparser(opts) ) return genericParser(assign({}, opts, { diff --git a/package.json b/package.json index 5b35fbf3..a4f29891 100644 --- a/package.json +++ b/package.json @@ -48,4 +48,4 @@ "test-ci": "nyc --reporter=lcov --reporter=text npm test", "test-cov": "nyc --reporter=html --reporter=text npm test" } -} +} \ No newline at end of file diff --git a/test/json.js b/test/json.js index 208e5a5e..642c30b1 100644 --- a/test/json.js +++ b/test/json.js @@ -100,10 +100,10 @@ describe('bodyParser.json()', function () { return { foo: 'bar' } } })) - .post('/') - .set('Content-Type', 'application/json') - .send('{"str":') - .expect(200, '{"foo":"bar"}', done) + .post('/') + .set('Content-Type', 'application/json') + .send('{"str":') + .expect(200, '{"foo":"bar"}', done) }) describe('when JSON is invalid', function () { diff --git a/test/urlencoded.js b/test/urlencoded.js index 88d29ce5..e11fe8fa 100644 --- a/test/urlencoded.js +++ b/test/urlencoded.js @@ -24,10 +24,10 @@ describe('bodyParser.urlencoded()', function () { request(createServer({ parser: function (input) { return input.toUpperCase() } })) - .post('/') - .set('Content-Type', 'application/x-www-form-urlencoded') - .send('user=tobi') - .expect(200, '"USER=TOBI"', done) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '"USER=TOBI"', done) }) it('should 400 when invalid content-length', function (done) { From 3526affa45aa7de438ab28893beb681e7a17f8ae Mon Sep 17 00:00:00 2001 From: S Dellysse Date: Thu, 16 Apr 2020 23:57:49 -0400 Subject: [PATCH 17/27] Ref'd genParser via the bodyparser getter to signal how third party parsers should import genParser' --- lib/types/json.js | 2 +- lib/types/raw.js | 2 +- lib/types/text.js | 2 +- lib/types/urlencoded.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/types/json.js b/lib/types/json.js index f949382f..e8b233af 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -13,7 +13,7 @@ */ var assign = require('object-assign') -var genericParser = require('../generic-parser') +var genericParser = require('../..').generic var debug = require('debug')('body-parser:json') /** diff --git a/lib/types/raw.js b/lib/types/raw.js index 0e8b359c..f6120aaa 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -11,7 +11,7 @@ */ var assign = require('object-assign') -var genericParser = require('../generic-parser') +var genericParser = require('../..').generic /** * Module exports. diff --git a/lib/types/text.js b/lib/types/text.js index 2956684e..36405a1a 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -11,7 +11,7 @@ */ var assign = require('object-assign') -var genericParser = require('../generic-parser') +var genericParser = require('../..').generic /** * Module exports. diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index f6c1d667..3ae6e7d1 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -15,7 +15,7 @@ var assign = require('object-assign') var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') -var genericParser = require('../generic-parser.js') +var genericParser = require('../..').generic /** * Module exports. From 12dcaafb967cfe5b42724d2b32d504608427fee7 Mon Sep 17 00:00:00 2001 From: S Dellysse Date: Fri, 17 Apr 2020 00:40:36 -0400 Subject: [PATCH 18/27] removed dep on object-assign, which didnt support node < 0.10 --- lib/generic-parser.js | 17 +++++++++++++++-- lib/types/json.js | 5 ++--- lib/types/raw.js | 5 ++--- lib/types/text.js | 5 ++--- lib/types/urlencoded.js | 5 ++--- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/lib/generic-parser.js b/lib/generic-parser.js index 8ddc52d4..24c3563e 100644 --- a/lib/generic-parser.js +++ b/lib/generic-parser.js @@ -34,8 +34,21 @@ module.exports = generic * @public */ -function generic (options) { - var opts = options || {} +function generic (parserOptions, parserOverrides) { + var opts = {} + + // Squash the options and the overrides down into one object + var squashKey + for (squashKey in (parserOptions || {})) { + if (Object.prototype.hasOwnProperty.call(parserOptions, squashKey)) { + opts[squashKey] = parserOptions[squashKey] + } + } + for (squashKey in (parserOverrides || {})) { + if (Object.prototype.hasOwnProperty.call(parserOverrides, squashKey)) { + opts[squashKey] = parserOverrides[squashKey] + } + } var limit = typeof opts.limit !== 'number' ? bytes.parse(opts.limit || '100kb') diff --git a/lib/types/json.js b/lib/types/json.js index e8b233af..ae1af796 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -12,7 +12,6 @@ * @private */ -var assign = require('object-assign') var genericParser = require('../..').generic var debug = require('debug')('body-parser:json') @@ -55,7 +54,7 @@ function json (options) { var parser = opts.parser || JSON.parse var type = opts.type || 'application/json' - return genericParser(assign({}, opts, { + return genericParser(opts, { type: type, charset: function validateCharset (charset) { @@ -88,7 +87,7 @@ function json (options) { }) } } - })) + }) } /** diff --git a/lib/types/raw.js b/lib/types/raw.js index f6120aaa..b40cfd5d 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -10,7 +10,6 @@ * Module dependencies. */ -var assign = require('object-assign') var genericParser = require('../..').generic /** @@ -32,7 +31,7 @@ function raw (options) { var type = opts.type || 'application/octet-stream' - return genericParser(assign({}, opts, { + return genericParser(opts, { type: type - })) + }) } diff --git a/lib/types/text.js b/lib/types/text.js index 36405a1a..87e65793 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -10,7 +10,6 @@ * Module dependencies. */ -var assign = require('object-assign') var genericParser = require('../..').generic /** @@ -33,9 +32,9 @@ function text (options) { var defaultCharset = opts.defaultCharset || 'utf-8' var type = opts.type || 'text/plain' - return genericParser(assign({}, opts, { + return genericParser(opts, { type: type, charset: function validateCharset () { return true }, defaultCharset: defaultCharset - })) + }) } diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 3ae6e7d1..732c6214 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -12,7 +12,6 @@ * @private */ -var assign = require('object-assign') var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') var genericParser = require('../..').generic @@ -47,7 +46,7 @@ function urlencoded (options) { : simpleparser(opts) ) - return genericParser(assign({}, opts, { + return genericParser(opts, { type: type, charset: charset, @@ -56,7 +55,7 @@ function urlencoded (options) { ? queryparse(buf) : {} } - })) + }) } /** From 6f5ff23c201f74e3eb9b5f7258bd67d9c404298e Mon Sep 17 00:00:00 2001 From: S Dellysse Date: Fri, 17 Apr 2020 00:43:41 -0400 Subject: [PATCH 19/27] minor text cleanup --- lib/generic-parser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/generic-parser.js b/lib/generic-parser.js index 24c3563e..3c88628b 100644 --- a/lib/generic-parser.js +++ b/lib/generic-parser.js @@ -27,7 +27,7 @@ var typeis = require('type-is') module.exports = generic /** - * Create a middleware to parse JSON bodies. + * Use this to create a middleware that parses request bodies * * @param {object} [options] * @return {function} From 4978bab8be9dc2c6cf8bdaa20aad87086a96a9eb Mon Sep 17 00:00:00 2001 From: ctcpip Date: Fri, 24 May 2024 15:29:01 -0500 Subject: [PATCH 20/27] =?UTF-8?q?=F0=9F=94=A7=20add=20debug=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index a4f29891..1e8c4750 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "lint": "eslint .", "test": "mocha --require test/support/env --reporter spec --check-leaks --bail test/", "test-ci": "nyc --reporter=lcov --reporter=text npm test", - "test-cov": "nyc --reporter=html --reporter=text npm test" + "test-cov": "nyc --reporter=html --reporter=text npm test", + "test-debug": "npm test -- --timeout 0" } } \ No newline at end of file From e1e629bb715fa1955f465679c5764ce9f8698913 Mon Sep 17 00:00:00 2001 From: ctcpip Date: Fri, 24 May 2024 16:52:59 -0500 Subject: [PATCH 21/27] =?UTF-8?q?=F0=9F=90=9B=20fix=20object=20merging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/generic-parser.js | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/lib/generic-parser.js b/lib/generic-parser.js index 3c88628b..deb96d10 100644 --- a/lib/generic-parser.js +++ b/lib/generic-parser.js @@ -35,20 +35,9 @@ module.exports = generic */ function generic (parserOptions, parserOverrides) { - var opts = {} - // Squash the options and the overrides down into one object - var squashKey - for (squashKey in (parserOptions || {})) { - if (Object.prototype.hasOwnProperty.call(parserOptions, squashKey)) { - opts[squashKey] = parserOptions[squashKey] - } - } - for (squashKey in (parserOverrides || {})) { - if (Object.prototype.hasOwnProperty.call(parserOverrides, squashKey)) { - opts[squashKey] = parserOverrides[squashKey] - } - } + var opts = Object.create(parserOptions) + Object.assign(opts, parserOverrides) var limit = typeof opts.limit !== 'number' ? bytes.parse(opts.limit || '100kb') From d574f61998bcee4f352793ba16856c2f6232acf2 Mon Sep 17 00:00:00 2001 From: ctcpip Date: Fri, 24 May 2024 18:07:18 -0500 Subject: [PATCH 22/27] =?UTF-8?q?=F0=9F=94=A5=20clean=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/types/urlencoded.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 732c6214..6419a7db 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -33,10 +33,6 @@ function urlencoded (options) { var opts = options || {} var extended = Boolean(opts.extended) - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'application/x-www-form-urlencoded' var charset = opts.charset || 'utf-8' From 6f3dae1bd030e053186f0993492f033a32767425 Mon Sep 17 00:00:00 2001 From: Phillip9587 Date: Tue, 19 Nov 2024 15:19:35 +0100 Subject: [PATCH 23/27] Fix rebase --- lib/types/urlencoded.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 6419a7db..3b2e0724 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -22,6 +22,12 @@ var genericParser = require('../..').generic module.exports = urlencoded +/** + * Cache of parser modules. + */ + +var parsers = Object.create(null) + /** * Create a middleware to parse urlencoded bodies. * @@ -60,18 +66,17 @@ function urlencoded (options) { * @param {object} options */ -function createQueryParser (options, extended) { +function extendedparser (options) { var parameterLimit = options.parameterLimit !== undefined ? options.parameterLimit : 1000 var charsetSentinel = options.charsetSentinel var interpretNumericEntities = options.interpretNumericEntities - var depth = extended ? (options.depth !== undefined ? options.depth : 32) : 0 + var depth = (options.depth !== undefined ? options.depth : 32) if (isNaN(parameterLimit) || parameterLimit < 1) { throw new TypeError('option parameterLimit must be a positive number') } - if (isNaN(depth) || depth < 0) { throw new TypeError('option depth must be a zero or a positive number') } @@ -80,6 +85,8 @@ function createQueryParser (options, extended) { parameterLimit = parameterLimit | 0 } + var parse = parser('qs') + return function queryparse (body, encoding) { var paramCount = parameterCount(body, parameterLimit) @@ -90,11 +97,11 @@ function createQueryParser (options, extended) { }) } - var arrayLimit = extended ? Math.max(100, paramCount) : 0 + var arrayLimit = Math.max(100, paramCount) - debug('parse ' + (extended ? 'extended ' : '') + 'urlencoding') + debug('parse extended urlencoding') try { - return qs.parse(body, { + return parse(body, { allowPrototypes: true, arrayLimit: arrayLimit, depth: depth, From c166e6dfff682bffc43f5cb98d3f288e2533086b Mon Sep 17 00:00:00 2001 From: Phillip9587 Date: Tue, 19 Nov 2024 15:24:20 +0100 Subject: [PATCH 24/27] Remove added npm script --- package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 1e8c4750..5b35fbf3 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "lint": "eslint .", "test": "mocha --require test/support/env --reporter spec --check-leaks --bail test/", "test-ci": "nyc --reporter=lcov --reporter=text npm test", - "test-cov": "nyc --reporter=html --reporter=text npm test", - "test-debug": "npm test -- --timeout 0" + "test-cov": "nyc --reporter=html --reporter=text npm test" } -} \ No newline at end of file +} From debe0a328a9ed1beae30b05461451df460f7d731 Mon Sep 17 00:00:00 2001 From: Phillip9587 Date: Tue, 19 Nov 2024 15:35:58 +0100 Subject: [PATCH 25/27] Fix rebase --- lib/types/urlencoded.js | 98 +++++------------------------------------ 1 file changed, 11 insertions(+), 87 deletions(-) diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 3b2e0724..9ff13080 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -15,6 +15,7 @@ var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') var genericParser = require('../..').generic +var qs = require('qs') /** * Module exports. @@ -22,12 +23,6 @@ var genericParser = require('../..').generic module.exports = urlencoded -/** - * Cache of parser modules. - */ - -var parsers = Object.create(null) - /** * Create a middleware to parse urlencoded bodies. * @@ -42,19 +37,15 @@ function urlencoded (options) { var type = opts.type || 'application/x-www-form-urlencoded' var charset = opts.charset || 'utf-8' - var queryparse = opts.parser || ( - extended - ? extendedparser(opts) - : simpleparser(opts) - ) + var queryparse = opts.parser || createQueryParser(opts, extended) return genericParser(opts, { type: type, charset: charset, - parse: function parse (buf) { - return buf.length - ? queryparse(buf) + parse: function parse (body, encoding) { + return body.length + ? queryparse(body, encoding) : {} } }) @@ -66,17 +57,18 @@ function urlencoded (options) { * @param {object} options */ -function extendedparser (options) { +function createQueryParser (options, extended) { var parameterLimit = options.parameterLimit !== undefined ? options.parameterLimit : 1000 var charsetSentinel = options.charsetSentinel var interpretNumericEntities = options.interpretNumericEntities - var depth = (options.depth !== undefined ? options.depth : 32) + var depth = extended ? (options.depth !== undefined ? options.depth : 32) : 0 if (isNaN(parameterLimit) || parameterLimit < 1) { throw new TypeError('option parameterLimit must be a positive number') } + if (isNaN(depth) || depth < 0) { throw new TypeError('option depth must be a zero or a positive number') } @@ -85,8 +77,6 @@ function extendedparser (options) { parameterLimit = parameterLimit | 0 } - var parse = parser('qs') - return function queryparse (body, encoding) { var paramCount = parameterCount(body, parameterLimit) @@ -97,11 +87,11 @@ function extendedparser (options) { }) } - var arrayLimit = Math.max(100, paramCount) + var arrayLimit = extended ? Math.max(100, paramCount) : 0 - debug('parse extended urlencoding') + debug('parse ' + (extended ? 'extended ' : '') + 'urlencoding') try { - return parse(body, { + return qs.parse(body, { allowPrototypes: true, arrayLimit: arrayLimit, depth: depth, @@ -146,69 +136,3 @@ function parameterCount (body, limit) { return count } - -/** - * Get parser for module name dynamically. - * - * @param {string} name - * @return {function} - * @api private - */ - -function parser (name) { - var mod = parsers[name] - - if (mod !== undefined) { - return mod.parse - } - - // this uses a switch for static require analysis - switch (name) { - case 'qs': - mod = require('qs') - break - case 'querystring': - mod = require('querystring') - break - } - - // store to prevent invoking require() - parsers[name] = mod - - return mod.parse -} - -/** - * Get the simple query parser. - * - * @param {object} options - */ - -function simpleparser (options) { - var parameterLimit = options.parameterLimit !== undefined - ? options.parameterLimit - : 1000 - var parse = parser('querystring') - - if (isNaN(parameterLimit) || parameterLimit < 1) { - throw new TypeError('option parameterLimit must be a positive number') - } - - if (isFinite(parameterLimit)) { - parameterLimit = parameterLimit | 0 - } - - return function queryparse (body) { - var paramCount = parameterCount(body, parameterLimit) - - if (paramCount === undefined) { - debug('too many parameters') - throw createError(413, 'too many parameters', { - type: 'parameters.too.many' - }) - } - - debug('parse urlencoding') - return parse(body, undefined, undefined, { maxKeys: parameterLimit }) - } -} From 96fada887211201949297e043c5f8bc94100965c Mon Sep 17 00:00:00 2001 From: Phillip9587 Date: Tue, 19 Nov 2024 15:48:10 +0100 Subject: [PATCH 26/27] Fix charset validation for urlencoded --- lib/types/urlencoded.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 9ff13080..95520a26 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -35,13 +35,15 @@ function urlencoded (options) { var extended = Boolean(opts.extended) var type = opts.type || 'application/x-www-form-urlencoded' - var charset = opts.charset || 'utf-8' var queryparse = opts.parser || createQueryParser(opts, extended) return genericParser(opts, { type: type, - charset: charset, + + charset: function validateCharset (charset) { + return charset === 'utf-8' || charset === 'iso-8859-1' + }, parse: function parse (body, encoding) { return body.length From b83814b526e4a588ea1600ea5a279c2530a00cfa Mon Sep 17 00:00:00 2001 From: Phillip9587 Date: Wed, 20 Nov 2024 20:26:08 +0100 Subject: [PATCH 27/27] Refactor --- README.md | 4 ++-- lib/{generic-parser.js => factory.js} | 18 +++++++----------- lib/types/json.js | 23 ++++++++++++----------- lib/types/raw.js | 16 ++++++++-------- lib/types/text.js | 21 ++++++++++----------- lib/types/urlencoded.js | 24 +++++++++++------------- 6 files changed, 50 insertions(+), 56 deletions(-) rename lib/{generic-parser.js => factory.js} (90%) diff --git a/README.md b/README.md index 394d577c..f80f0b47 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ to `'100kb'`. ##### parser The `parser` option is the function called against the request body to convert -it to a Javascript object. If a `reviver` is supplied, it is supplied as the +it to a JavaScript object. If a `reviver` is supplied, it is supplied as the second argument to this function. ``` @@ -309,7 +309,7 @@ The `depth` option is used to configure the maximum depth of the `qs` library wh ##### parser The `parser` option, if supplied, is used to in place of the default parser to -convert the request body into a Javascript object. If this option is supplied, +convert the request body into a JavaScript object. If this option is supplied, both the `extended` and `parameterLimit` options are ignored. ``` diff --git a/lib/generic-parser.js b/lib/factory.js similarity index 90% rename from lib/generic-parser.js rename to lib/factory.js index deb96d10..5f04ab7a 100644 --- a/lib/generic-parser.js +++ b/lib/factory.js @@ -24,20 +24,21 @@ var typeis = require('type-is') * Module exports. */ -module.exports = generic +module.exports = createBodyParser /** * Use this to create a middleware that parses request bodies * - * @param {object} [options] + * @param {function} parse + * @param {object} options + * @param {object} defaultOptions * @return {function} * @public */ -function generic (parserOptions, parserOverrides) { +function createBodyParser (parse, options, defaultOptions) { // Squash the options and the overrides down into one object - var opts = Object.create(parserOptions) - Object.assign(opts, parserOverrides) + var opts = { ...defaultOptions || {}, ...options } var limit = typeof opts.limit !== 'number' ? bytes.parse(opts.limit || '100kb') @@ -45,7 +46,6 @@ function generic (parserOptions, parserOverrides) { var charset = opts.charset var inflate = opts.inflate !== false var verify = opts.verify || false - var parse = opts.parse || defaultParse var defaultReqCharset = opts.defaultCharset || 'utf-8' var type = opts.type @@ -63,7 +63,7 @@ function generic (parserOptions, parserOverrides) { ? charsetValidator(charset) : charset - return function genericParser (req, res, next) { + return function (req, res, next) { if (isFinished(req)) { debug('body already parsed') next() @@ -114,10 +114,6 @@ function generic (parserOptions, parserOverrides) { } } -function defaultParse (buf) { - return buf -} - /** * Get the charset of a request. * diff --git a/lib/types/json.js b/lib/types/json.js index ae1af796..44725fd1 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -12,7 +12,7 @@ * @private */ -var genericParser = require('../..').generic +var createBodyParser = require('../factory') var debug = require('debug')('body-parser:json') /** @@ -52,16 +52,9 @@ function json (options) { var reviver = opts.reviver var strict = opts.strict !== false var parser = opts.parser || JSON.parse - var type = opts.type || 'application/json' - return genericParser(opts, { - type: type, - - charset: function validateCharset (charset) { - return charset.slice(0, 4) === 'utf-' - }, - - parse: function parse (buf) { + return createBodyParser( + function (buf) { if (buf.length === 0) { // special-case empty json body, as it's a common client-side mistake // TODO: maybe make this configurable or part of "strict" option @@ -86,8 +79,16 @@ function json (options) { stack: e.stack }) } + }, + opts, + { + parser: JSON.parse, + type: 'application/json', + charset: function (charset) { + return charset.slice(0, 4) === 'utf-' + } } - }) + ) } /** diff --git a/lib/types/raw.js b/lib/types/raw.js index b40cfd5d..3a33e888 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -10,7 +10,7 @@ * Module dependencies. */ -var genericParser = require('../..').generic +var createBodyParser = require('../factory') /** * Module exports. @@ -27,11 +27,11 @@ module.exports = raw */ function raw (options) { - var opts = options || {} - - var type = opts.type || 'application/octet-stream' - - return genericParser(opts, { - type: type - }) + return createBodyParser( + function (buf) { return buf }, + options, + { + type: 'application/octet-stream' + } + ) } diff --git a/lib/types/text.js b/lib/types/text.js index 87e65793..2b8ef4ec 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -10,7 +10,7 @@ * Module dependencies. */ -var genericParser = require('../..').generic +var createBodyParser = require('../factory') /** * Module exports. @@ -27,14 +27,13 @@ module.exports = text */ function text (options) { - var opts = options || {} - - var defaultCharset = opts.defaultCharset || 'utf-8' - var type = opts.type || 'text/plain' - - return genericParser(opts, { - type: type, - charset: function validateCharset () { return true }, - defaultCharset: defaultCharset - }) + return createBodyParser( + function (buf) { return buf }, + options, + { + type: 'text/plain', + charset: function () { return true }, + defaultCharset: 'utf-8' + } + ) } diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 95520a26..cd93f149 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -14,7 +14,7 @@ var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') -var genericParser = require('../..').generic +var createBodyParser = require('../factory') var qs = require('qs') /** @@ -34,23 +34,21 @@ function urlencoded (options) { var opts = options || {} var extended = Boolean(opts.extended) - var type = opts.type || 'application/x-www-form-urlencoded' var queryparse = opts.parser || createQueryParser(opts, extended) - return genericParser(opts, { - type: type, - - charset: function validateCharset (charset) { - return charset === 'utf-8' || charset === 'iso-8859-1' + return createBodyParser( + function (body, encoding) { + return body.length ? queryparse(body, encoding) : {} }, - - parse: function parse (body, encoding) { - return body.length - ? queryparse(body, encoding) - : {} + opts, + { + type: 'application/x-www-form-urlencoded', + charset: function (charset) { + return charset === 'utf-8' || charset === 'iso-8859-1' + } } - }) + ) } /**