A case-study on where recursion can be useful for enterprise Node.js applications and how to avoid its common pitfalls like RangeError: Maximum call stack size exceeded.
The full repository for this post is on GitHub: github.com/HugoDF/mongo-query-clause-modification
We’ll be implementing a solution to 2 real-world problems:
- Add an $or query clause to a MongoDB query
- Remove references to a field in an MongoDB query (potentially) using $or and $and
Add an $or query clause to a MongoDB query
See the final code at ./src/add-or-clause.js.
The parameters are query and orClause.
query is a MongoDB query which might or might not already contain an $or and/or $and clause.
orClause is an object containing and $or clause (it’s a fully-fledged MongoDB query in its own right) eg.
const orClause = { $or: [ {createdAt: {$exists: false}}, {createdAt: someDate} ] }; There is initially just 1 thing to look out for:1. the query does not contain an $or clause2. the query contains an $or clause
When there’s no $or clause in the query
If there is no $or clause, we can simply spread our orClause query and the query parameter, ie.
const newQuery = { ...query, ...orClause }; That is unless there’s and $and in there somewhere, in which case we want to add our orClause to the $and:
const newQuery = { ...query, $and: [...query.$and, orClause] }; When there’s an $or clause in the query
If there is an $or clause, we can’t just overwrite it, we need to $and the two $or queries.
We should also keep existing $and clause contents which yields:
const newQuery = { ...queryWithoutOrRemoved, $and: [ ...(query.$and || []), { $or: query.$or }, orClause ] }; Full solution
This is also available at ./src/add-or-clause.js.
function addOrClause(query, orClause) { const {$or, ...queryRest} = query; if ($or) { return { ...queryRest, $and: [...(queryRest.$and || []), {$or}, orClause] }; } if (queryRest.$and) { return { ...queryRest, $and: [...queryRest.$and, orClause] }; } return { ...query, ...orClause }; } module.exports = addOrClause; Corresponding Test Suite
We can observe how the different cases map pretty directly to test cases.
const addOrClause = require('./add-or-clause'); test('should add the passed or clause if no $or on the current query', () => { const orClause = {$or: [{myField: 'value'}, {myField: null}]}; const query = {foo: 'bar'}; expect(addOrClause(query, orClause)).toEqual({ $or: [{myField: 'value'}, {myField: null}], foo: 'bar' }); }); describe('when the query already has an $or', () => { test('should add the passed or clause to and $and that also contains the current query', () => { const orClause = {$or: [{myField: 'value'}, {myField: null}]}; const query = {$or: [{foo: 'bar'}, {foo: {$exists: false}}]}; expect(addOrClause(query, orClause)).toEqual({ $and: [ {$or: [{foo: 'bar'}, {foo: {$exists: false}}]}, { $or: [{myField: 'value'}, {myField: null}] } ] }); }); describe('when the query has an $and', () => { test('should keep the $and, add the $or and the current query', () => { const orClause = {$or: [{myField: 'value'}, {myField: null}]}; const query = { $or: [{hello: 'world'}], $and: [{foo: 'bar'}, {bar: 'baz'}] }; expect(addOrClause(query, orClause)).toEqual({ $and: [ {foo: 'bar'}, {bar: 'baz'}, {$or: [{hello: 'world'}]}, {$or: [{myField: 'value'}, {myField: null}]} ] }); }); }); }); describe('when the query has an $and query', () => { test('should add the new or clause to the $and', () => { const orClause = {$or: [{myField: 'value'}, {myField: null}]}; const query = {$and: [{foo: 'bar'}, {bar: 'baz'}]}; expect(addOrClause(query, orClause)).toEqual({ $and: [ {foo: 'bar'}, {bar: 'baz'}, {$or: [{myField: 'value'}, {myField: null}]} ] }); }); }); Remove references to a field in an MongoDB query (potentially) using $or and $and
See the full solution at ./src/remove-field-references.js.
In this case we’re creating a function that takes 2 parameters: query (MongoDB query as above) and fieldName (name of the field we want to remove references to).
Remove top-level fields
The simplest thing to do is remove references to the field at the top-level of the object.
We can create a simple omit function using destructuring and recursion
const omit = (obj, [field, ...nextFields]) => { const {[field]: ignore, ...rest} = obj; return nextFields.length > 0 ? omit(rest, nextFields) : rest; }; And use it:
const newQuery = omit(query, [fieldName]); Remove fields in any $or clause
To remove fields in an $or clause (which is a fully-fledged query) is as simple as taking the $or value (which is an array) and running a recursion of the function onto it.
This will remove fields at the top-level of the $or sub-queries and in nest $or fields’ sub-queries.
We want to make sure to remove empty $or sub-queries, since { $or: [{ }, {}]} is an invalid query.
We default the query’s $or to an empty array and check length before spreading it back into the newQuery. This is because { $or: [] } is an invalid query.
We’re also careful to remove the top-level $or when spreading filteredTopLevel so that if the new $or is an empty array, the old $or is ommitted.
function removeFieldReferences (query, fieldName) { const filteredTopLevel = omit(query, [fieldName]); const newOr = (filteredTopLevel.$or || []) .map(q => removeFieldReferences(q, fieldName)) .filter(q => Object.keys(q).length > 0); return { ...omit(filteredTopLevel, ['$or']), ...(newOr.length > 0 ? {$or: newOr} : {}) }; } Remove fields in any $and clause
The rationale for the $and solution is the same as for the $or solution.
We recurse and check that we’re not generating an invalid query by omitting empty arrays and objects:
function removeFieldReferences (query, fieldName) { const filteredTopLevel = omit(query, [fieldName]); const newAnd = (filteredTopLevel.$and || []) .map(q => removeFieldReferences(q, fieldName)) .filter(q => Object.keys(q).length > 0); return { ...omit(filteredTopLevel, ['$and']), ...(newAnd.length > 0 ? {$and: newAnd} : {}) }; } Check that we’re not likely to bust the stack
The actual implementation has a maxDepth 3rd parameter defaulted to 5.
When maxDepth is equal to 0, we return the query without any treatment (arguably we should run the top-level filter).
On recursive calls to removeFieldReferences we pass (q, fieldName, maxDepth - 1) so that we’re not going any deeper than we need to by accident.
This avoids RangeError: Maximum call stack size exceeded.
Final Implementation
This is also available at ./src/remove-field-references.js.
const omit = (obj, [field, ...nextFields]) => { const {[field]: ignore, ...rest} = obj; return nextFields.length > 0 ? omit(rest, nextFields) : rest; }; function removeFieldReferences(query, fieldName, maxDepth = 5) { if (maxDepth <= 0) { return query; } const filteredTopLevel = omit(query, [fieldName]); const newOr = (filteredTopLevel.$or || []) .map(q => removeFieldReferences(q, fieldName, maxDepth - 1)) .filter(q => Object.keys(q).length > 0); const newAnd = (filteredTopLevel.$and || []) .map(q => removeFieldReferences(q, fieldName, maxDepth - 1)) .filter(q => Object.keys(q).length > 0); return { ...omit(filteredTopLevel, ['$or', '$and']), ...(newOr.length > 0 ? {$or: newOr} : {}), ...(newAnd.length > 0 ? {$and: newAnd} : {}) }; } module.exports = removeFieldReferences; Corresponding Test Suite
We can observe how the different cases map pretty directly to test cases.
const removeFieldReferences = require('./remove-field-references'); test('should remove top-level fields', () => { const query = { hello: 'value' }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName).hello).toBeUndefined(); }); test('should return passed query when maxDepth is hit (avoids busting the stack by default)', () => { const query = { hello: 'value' }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName, 0)).toEqual(query); }); test('should remove references to the field in top-level $or queries', () => { const query = { $or: [ {hello: 'value', otherField: 'not-related'}, {hello: 'othervalue', otherField: 'even-less-related'} ] }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName)).toEqual({ $or: [{otherField: 'not-related'}, {otherField: 'even-less-related'}] }); }); test('should remove $or clauses where the query becomes empty on omission of a field', () => { const query = { $or: [{hello: 'value'}, {otherField: 'not-related'}] }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName)).toEqual({ $or: [{otherField: 'not-related'}] }); }); test('should remove references to field in top-level queries inside of $and', () => { const query = { $and: [ {hello: 'value', otherField: 'value'}, {hello: 'other-value', otherField: 'value'} ] }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName)).toEqual({ $and: [{otherField: 'value'}, {otherField: 'value'}] }); }); test('should remove $and clause if all queries end up filtered out', () => { const query = { foo: 'bar', $and: [{hello: 'value'}, {hello: 'other-value'}] }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName)).toEqual({foo: 'bar'}); }); test('should remove references to field in nested $or inside of $and', () => { const query = { $and: [ { $or: [{hello: 'value'}, {hello: null}] }, {otherField: 'not-related'} ] }; const fieldName = 'hello'; expect(removeFieldReferences(query, fieldName)).toEqual({ $and: [{otherField: 'not-related'}] }); }); 
Top comments (0)