From b94f7c65e7d06565bed15e405e2e61f702ec4382 Mon Sep 17 00:00:00 2001 From: ctcpip Date: Tue, 14 Jan 2025 18:09:41 -0600 Subject: [PATCH] generic body parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit squashed the following commits: Added support for external parsers to bodyParser.json() removed test dependency on json-bigint reworked doc to describe json parser() func better added parser() option and doc for .text() added parser() option and doc for .raw() added parser() option and doc for .urlencoded() cleanup to satisfy linter added generic parser converted json parser to use generic parser converted raw parser to use generic parser converted text parser to use generic parser converted urlencoded parser to use generic parser cleanup / fix linter warnings removed items from README added bodyParser.generic() getter cleanup / fix linter warnings fixed tests after rebase satisfying linter Ref'd genParser via the bodyparser getter to signal how third party parsers should import genParser' removed dep on object-assign, which didnt support node < 0.10 minor text cleanup 🔧 add debug script 🐛 fix object merging 🔥 clean up 💚 remove node < 4 from CI Co-authored-by: S Dellysse Co-authored-by: ctcpip Co-authored-by: Phillip9587 --- README.md | 28 ++++++- lib/generic-parser.js | 160 ++++++++++++++++++++++++++++++++++++++++ lib/types/json.js | 147 ++++++++---------------------------- lib/types/raw.js | 75 +------------------ lib/types/text.js | 96 ++---------------------- lib/types/urlencoded.js | 121 +++--------------------------- package.json | 5 +- test/json.js | 12 +++ test/urlencoded.js | 10 +++ 9 files changed, 262 insertions(+), 392 deletions(-) create mode 100644 lib/generic-parser.js diff --git a/README.md b/README.md index 1eebdffd..8cdc735f 100644 --- a/README.md +++ b/README.md @@ -88,16 +88,27 @@ specifies the number of bytes; if it is a string, the value is passed to the [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults to `'100kb'`. +##### parser + +The `parser` option is the function called against the request body to convert +it to a Javascript object. If a `reviver` is supplied, it is supplied as the +second argument to this function. + +``` +parser(body, reviver) -> req.body +``` + +Defaults to `JSON.parse`. + ##### reviver -The `reviver` option is passed directly to `JSON.parse` as the second -argument. You can find more information on this argument +You can find more information on this argument [in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter). ##### strict When set to `true`, will only accept arrays and objects; when `false` will -accept anything `JSON.parse` accepts. Defaults to `true`. +accept anything the `parser` accepts. Defaults to `true`. ##### type @@ -290,11 +301,20 @@ of `✓`. Defaults to `false`. Whether to decode numeric entities such as `☺` when parsing an iso-8859-1 form. Defaults to `false`. - #### depth The `depth` option is used to configure the maximum depth of the `qs` library when `extended` is `true`. This allows you to limit the amount of keys that are parsed and can be useful to prevent certain types of abuse. Defaults to `32`. It is recommended to keep this value as low as possible. +##### parser + +The `parser` option, if supplied, is used to in place of the default parser to +convert the request body into a Javascript object. If this option is supplied, +both the `extended` and `parameterLimit` options are ignored. + +``` +parser(body) -> req.body +``` + ## Errors The middlewares provided by this module create errors using the diff --git a/lib/generic-parser.js b/lib/generic-parser.js new file mode 100644 index 00000000..deb96d10 --- /dev/null +++ b/lib/generic-parser.js @@ -0,0 +1,160 @@ +/*! + * body-parser + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var createError = require('http-errors') +var debug = require('debug')('body-parser:generic') +var isFinished = require('on-finished').isFinished +var read = require('./read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = generic + +/** + * Use this to create a middleware that parses request bodies + * + * @param {object} [options] + * @return {function} + * @public + */ + +function generic (parserOptions, parserOverrides) { + // Squash the options and the overrides down into one object + var opts = Object.create(parserOptions) + Object.assign(opts, parserOverrides) + + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var charset = opts.charset + var inflate = opts.inflate !== false + var verify = opts.verify || false + var parse = opts.parse || defaultParse + var defaultReqCharset = opts.defaultCharset || 'utf-8' + var type = opts.type + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + // create the appropriate charset validating function + var validCharset = typeof charset !== 'function' + ? charsetValidator(charset) + : charset + + return function genericParser (req, res, next) { + if (isFinished(req)) { + debug('body already parsed') + next() + return + } + + if (!('body' in req)) { + req.body = undefined + } + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // assert charset per RFC 7159 sec 8.1 + var reqCharset = null + if (charset !== undefined) { + reqCharset = getCharset(req) || defaultReqCharset + if (!validCharset(reqCharset)) { + debug('invalid charset') + next(createError(415, 'unsupported charset "' + reqCharset.toUpperCase() + '"', { + charset: reqCharset, + type: 'charset.unsupported' + })) + return + } + } + + // read + read(req, res, next, parse, debug, { + encoding: reqCharset, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +function defaultParse (buf) { + return buf +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return (contentType.parse(req).parameters.charset || '').toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} + +/** + * Get the simple charset validator. + * + * @param {string} type + * @return {function} + */ + +function charsetValidator (charset) { + return function validateCharset (reqCharset) { + return charset === reqCharset + } +} diff --git a/lib/types/json.js b/lib/types/json.js index 30bf8cab..b5172aa1 100644 --- a/lib/types/json.js +++ b/lib/types/json.js @@ -12,13 +12,8 @@ * @private */ -var bytes = require('bytes') -var contentType = require('content-type') -var createError = require('http-errors') +var genericParser = require('../generic-parser') var debug = require('debug')('body-parser:json') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') /** * Module exports. @@ -54,97 +49,45 @@ var JSON_SYNTAX_REGEXP = /#+/g function json (options) { var opts = options || {} - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit - var inflate = opts.inflate !== false var reviver = opts.reviver var strict = opts.strict !== false + var parser = opts.parser || JSON.parse var type = opts.type || 'application/json' - var verify = opts.verify || false - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type + return genericParser(opts, { + type: type, - function parse (body) { - if (body.length === 0) { - // special-case empty json body, as it's a common client-side mistake - // TODO: maybe make this configurable or part of "strict" option - return {} - } + charset: function validateCharset (charset) { + return charset.slice(0, 4) === 'utf-' + }, - if (strict) { - var first = firstchar(body) - - if (first !== '{' && first !== '[') { - debug('strict violation') - throw createStrictSyntaxError(body, first) + parse: function parse (buf) { + if (buf.length === 0) { + // special-case empty json body, as it's a common client-side mistake + // TODO: maybe make this configurable or part of "strict" option + return {} } - } - try { - debug('parse json') - return JSON.parse(body, reviver) - } catch (e) { - throw normalizeJsonSyntaxError(e, { - message: e.message, - stack: e.stack - }) - } - } + if (strict) { + var first = firstchar(buf) - return function jsonParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } + if (first !== '{' && first !== '[') { + debug('strict violation') + throw createStrictSyntaxError(parser, reviver, buf, first) + } + } - // assert charset per RFC 7159 sec 8.1 - var charset = getCharset(req) || 'utf-8' - if (charset.slice(0, 4) !== 'utf-') { - debug('invalid charset') - next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { - charset: charset, - type: 'charset.unsupported' - })) - return + try { + debug('parse json') + return parser(buf, reviver) + } catch (e) { + throw normalizeJsonSyntaxError(e, { + message: e.message, + stack: e.stack + }) + } } - - // read - read(req, res, next, parse, debug, { - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } + }) } /** @@ -156,7 +99,7 @@ function json (options) { * @private */ -function createStrictSyntaxError (str, char) { +function createStrictSyntaxError (parser, reviver, str, char) { var index = str.indexOf(char) var partial = '' @@ -169,7 +112,7 @@ function createStrictSyntaxError (str, char) { } try { - JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation') + parser(partial, reviver); /* istanbul ignore next */ throw new SyntaxError('strict violation') } catch (e) { return normalizeJsonSyntaxError(e, { message: e.message.replace(JSON_SYNTAX_REGEXP, function (placeholder) { @@ -196,21 +139,6 @@ function firstchar (str) { : undefined } -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - /** * Normalize a SyntaxError for JSON.parse. * @@ -235,16 +163,3 @@ function normalizeJsonSyntaxError (error, obj) { return error } - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } -} diff --git a/lib/types/raw.js b/lib/types/raw.js index bfe274cf..216affed 100644 --- a/lib/types/raw.js +++ b/lib/types/raw.js @@ -10,11 +10,7 @@ * Module dependencies. */ -var bytes = require('bytes') -var debug = require('debug')('body-parser:raw') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var genericParser = require('../generic-parser') /** * Module exports. @@ -33,72 +29,9 @@ module.exports = raw function raw (options) { var opts = options || {} - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'application/octet-stream' - var verify = opts.verify || false - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (buf) { - return buf - } - - return function rawParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // read - read(req, res, next, parse, debug, { - encoding: null, - inflate: inflate, - limit: limit, - verify: verify - }) - } -} - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } + return genericParser(opts, { + type: type + }) } diff --git a/lib/types/text.js b/lib/types/text.js index b153931b..92cfdeeb 100644 --- a/lib/types/text.js +++ b/lib/types/text.js @@ -10,12 +10,7 @@ * Module dependencies. */ -var bytes = require('bytes') -var contentType = require('content-type') -var debug = require('debug')('body-parser:text') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') +var genericParser = require('../generic-parser') /** * Module exports. @@ -35,90 +30,11 @@ function text (options) { var opts = options || {} var defaultCharset = opts.defaultCharset || 'utf-8' - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'text/plain' - var verify = opts.verify || false - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } - - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (buf) { - return buf - } - - return function textParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined - } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // get charset - var charset = getCharset(req) || defaultCharset - - // read - read(req, res, next, parse, debug, { - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify - }) - } -} - -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } + return genericParser(opts, { + type: type, + charset: function validateCharset () { return true }, + defaultCharset: defaultCharset + }) } diff --git a/lib/types/urlencoded.js b/lib/types/urlencoded.js index 687745f8..bab6ad4e 100644 --- a/lib/types/urlencoded.js +++ b/lib/types/urlencoded.js @@ -12,14 +12,10 @@ * @private */ -var bytes = require('bytes') -var contentType = require('content-type') var createError = require('http-errors') var debug = require('debug')('body-parser:urlencoded') -var isFinished = require('on-finished').isFinished -var read = require('../read') -var typeis = require('type-is') var qs = require('qs') +var genericParser = require('../generic-parser') /** * Module exports. @@ -34,92 +30,27 @@ module.exports = urlencoded * @return {function} * @public */ - function urlencoded (options) { var opts = options || {} var extended = Boolean(opts.extended) - var inflate = opts.inflate !== false - var limit = typeof opts.limit !== 'number' - ? bytes.parse(opts.limit || '100kb') - : opts.limit var type = opts.type || 'application/x-www-form-urlencoded' - var verify = opts.verify || false - var charsetSentinel = opts.charsetSentinel - var interpretNumericEntities = opts.interpretNumericEntities - if (verify !== false && typeof verify !== 'function') { - throw new TypeError('option verify must be function') - } + var queryparse = opts.parser || createQueryParser(opts, extended) - var defaultCharset = opts.defaultCharset || 'utf-8' - if (defaultCharset !== 'utf-8' && defaultCharset !== 'iso-8859-1') { - throw new TypeError('option defaultCharset must be either utf-8 or iso-8859-1') - } + return genericParser(opts, { + type: type, - // create the appropriate query parser - var queryparse = createQueryParser(opts, extended) + charset: function validateCharset (charset) { + return charset === 'utf-8' || charset === 'iso-8859-1' + }, - // create the appropriate type checking function - var shouldParse = typeof type !== 'function' - ? typeChecker(type) - : type - - function parse (body, encoding) { - return body.length - ? queryparse(body, encoding) - : {} - } - - return function urlencodedParser (req, res, next) { - if (isFinished(req)) { - debug('body already parsed') - next() - return - } - - if (!('body' in req)) { - req.body = undefined + parse: function parse (body, encoding) { + return body.length + ? queryparse(body, encoding) + : {} } - - // skip requests without bodies - if (!typeis.hasBody(req)) { - debug('skip empty body') - next() - return - } - - debug('content-type %j', req.headers['content-type']) - - // determine if request should be parsed - if (!shouldParse(req)) { - debug('skip parsing') - next() - return - } - - // assert charset - var charset = getCharset(req) || defaultCharset - if (charset !== 'utf-8' && charset !== 'iso-8859-1') { - debug('invalid charset') - next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { - charset: charset, - type: 'charset.unsupported' - })) - return - } - - // read - read(req, res, next, parse, debug, { - debug: debug, - encoding: charset, - inflate: inflate, - limit: limit, - verify: verify, - charsetSentinel: charsetSentinel, - interpretNumericEntities: interpretNumericEntities - }) - } + }) } /** @@ -184,21 +115,6 @@ function createQueryParser (options, extended) { } } -/** - * Get the charset of a request. - * - * @param {object} req - * @api private - */ - -function getCharset (req) { - try { - return (contentType.parse(req).parameters.charset || '').toLowerCase() - } catch (e) { - return undefined - } -} - /** * Count the number of parameters, stopping once limit reached * @@ -222,16 +138,3 @@ function parameterCount (body, limit) { return count } - -/** - * Get the simple type checker. - * - * @param {string} type - * @return {function} - */ - -function typeChecker (type) { - return function checkType (req) { - return Boolean(typeis(req, type)) - } -} diff --git a/package.json b/package.json index 5b35fbf3..1e8c4750 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "lint": "eslint .", "test": "mocha --require test/support/env --reporter spec --check-leaks --bail test/", "test-ci": "nyc --reporter=lcov --reporter=text npm test", - "test-cov": "nyc --reporter=html --reporter=text npm test" + "test-cov": "nyc --reporter=html --reporter=text npm test", + "test-debug": "npm test -- --timeout 0" } -} +} \ No newline at end of file diff --git a/test/json.js b/test/json.js index 3b5cc653..642c30b1 100644 --- a/test/json.js +++ b/test/json.js @@ -94,6 +94,18 @@ describe('bodyParser.json()', function () { .expect(200, '{"user":"tobi"}', done) }) + it('should use external parsers', function (done) { + request(createServer({ + parser: function (body) { + return { foo: 'bar' } + } + })) + .post('/') + .set('Content-Type', 'application/json') + .send('{"str":') + .expect(200, '{"foo":"bar"}', done) + }) + describe('when JSON is invalid', function () { before(function () { this.server = createServer() diff --git a/test/urlencoded.js b/test/urlencoded.js index dfe0eb9f..e11fe8fa 100644 --- a/test/urlencoded.js +++ b/test/urlencoded.js @@ -20,6 +20,16 @@ describe('bodyParser.urlencoded()', function () { .expect(200, '{"user":"tobi"}', done) }) + it('should parse x-www-form-urlencoded with custom parser', function (done) { + request(createServer({ + parser: function (input) { return input.toUpperCase() } + })) + .post('/') + .set('Content-Type', 'application/x-www-form-urlencoded') + .send('user=tobi') + .expect(200, '"USER=TOBI"', done) + }) + it('should 400 when invalid content-length', function (done) { var urlencodedParser = bodyParser.urlencoded() var server = createServer(function (req, res, next) {