fix(deps): Updates Mongo to 4.2 (LLC-234) (#1541)

BREAKING CHANGE: Removes maxScan parameter from API BREAKING CHANGE: Requires Mongo 4.2

Author: lydiamross

.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
  build:
  docker:
  - image: circleci/node:10
- - image: mongo:3.4-jessie
+ - image: mongo:4.2-bionic
  - image: redis:4-alpine
  environment:
  TERM: xterm

.env.example

@@ -87,10 +87,6 @@ REDIS_PREFIX=LEARNINGLOCKER
 # https://docs.mongodb.com/manual/reference/operator/meta/maxTimeMS/
 #MAX_TIME_MS=
 
-# Maximum number of documents an aggregation can scan
-# https://docs.mongodb.com/manual/reference/operator/meta/maxScan/
-#MAX_SCAN=
-
 # Turn off fuzzy scoring on persona matching
 # Set to true to make persona workers much faster at scale (default is true)
 #DISABLE_PERSONA_SCORING=true

api/src/controllers/PersonaController.js

@@ -4,7 +4,7 @@ import getJSONFromQuery from 'api/utils/getJSONFromQuery';
 import getFromQuery from 'api/utils/getFromQuery';
 import getOrgFromAuthInfo from 'lib/services/auth/authInfoSelectors/getOrgFromAuthInfo';
 import getScopeFilter from 'lib/services/auth/filters/getScopeFilter';
-import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions';
+import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions';
 import parseQuery from 'lib/helpers/parseQuery';
 import { CursorDirection } from '@learninglocker/persona-service/dist/service/constants';
 import { entityResponse, entitiesResponse } from 'api/controllers/utils/entitiesResponse';
@@ -47,8 +47,7 @@ const personaConnection = catchErrors(async (req, res) => {
  filter,
  project,
  hint,
- maxTimeMS: MAX_TIME_MS,
- maxScan: MAX_SCAN
+ maxTimeMS: MAX_TIME_MS
  };
 
  const result = await personaService.getPersonasConnection(params);

api/src/controllers/PersonaIdentifierController.js

@@ -9,7 +9,7 @@ import getAuthFromRequest from 'lib/helpers/getAuthFromRequest';
 import getScopeFilter from 'lib/services/auth/filters/getScopeFilter';
 import { CursorDirection } from '@learninglocker/persona-service/dist/service/constants';
 import Locked from '@learninglocker/persona-service/dist/errors/Locked';
-import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions';
+import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions';
 import parseQuery from 'lib/helpers/parseQuery';
 import asignIdentifierToStatements from 'lib/services/persona/asignIdentifierToStatements';
 import identifierHasStatements from 'lib/services/persona/identifierHasStatements';
@@ -67,8 +67,7 @@ const personaIdentifierConnection = catchErrors(async (req, res) => {
  filter: filterNoUndefined,
  project,
  hint,
- maxTimeMS: MAX_TIME_MS,
- maxScan: MAX_SCAN
+ maxTimeMS: MAX_TIME_MS
  });
 
  return res.status(200).send(result);

api/src/controllers/StatementController.js

@@ -6,15 +6,13 @@ import catchErrors from 'api/controllers/utils/catchErrors';
 import defaultTo from 'lodash/defaultTo';
 
 const MAX_TIME_MS = defaultTo(Number(process.env.MAX_TIME_MS), 0);
-const MAX_SCAN = defaultTo(Number(process.env.MAX_SCAN), 0);
 
 const aggregate = (req) => {
  const authInfo = req.user.authInfo || {};
  const limit = Number(req.query.limit) || -1;
  const skip = Number(req.query.skip) || 0;
  const cache = (!!req.query.cache && req.query.cache !== 'false') || false;
  const maxTimeMS = Number(req.query.maxTimeMS) || MAX_TIME_MS;
- const maxScan = Number(req.query.maxScan) || MAX_SCAN;
  const pipeline = JSON.parse(req.query.pipeline);
  const sampleSize = Number(req.query.sampleSize) || undefined;
  const out = statementsService.aggregate({
@@ -23,7 +21,6 @@ const aggregate = (req) => {
  skip,
  cache,
  maxTimeMS,
- maxScan,
  pipeline,
  sampleSize
  });
@@ -87,15 +84,13 @@ const countStatements = catchErrors(async (req, res) => {
  getJSONFromQuery(req, 'query', {})
  );
  const maxTimeMS = getFromQuery(req, 'maxTimeMS', MAX_TIME_MS, Number);
- const maxScan = getFromQuery(req, 'maxScan', MAX_SCAN, Number);
  const hint = getJSONFromQuery(req, 'hint', null);
  const authInfo = getAuthFromRequest(req);
 
  const count = await statementsService.countStatements({
  authInfo,
  filter,
  maxTimeMS,
- maxScan,
  hint
  });
 

api/src/utils/exports.js

@@ -49,7 +49,6 @@ export const exportCSV = ({ authInfo, pipelines }) => new Promise((resolve, reje
  limit: 500000,
  getStream: true,
  maxTimeMS: 0,
- maxScan: 0,
  }, next),
  (err, streams) => {
  if (err) reject(err);

cli/src/commands/v2-migrations/20171122100800_common_indexes.js

@@ -32,7 +32,6 @@ const createStatementIndexes = (connection) => {
  createIndexWithVoid('timestamp__id', { timestamp: -1, _id: -1 });
  createIndexWithVoid('stored__id', { stored: -1, _id: -1 });
  createIndex(stmts, 'statementId_lrs_id', { 'statement.id': 1, lrs_id: 1 });
- createIndexWithOrg('timestamp__id', { timestamp: -1, _id: 1 });
  createIndexWithOrg('stored__id', { stored: -1, _id: 1 });
  createIndexWithOrg('objId', objectIdIndex);
  createIndexWithOrg('verbId_objId', { ...verbIdIndex, ...objectIdIndex });

lib/helpers/convert$personaIdent.js

@@ -1,7 +1,7 @@
 import _, { map, without } from 'lodash';
 import getService from 'lib/connections/personaService';
 import logger from 'lib/logger';
-import { MAX_TIME_MS, MAX_SCAN } from 'lib/models/plugins/addCRUDFunctions';
+import { MAX_TIME_MS } from 'lib/models/plugins/addCRUDFunctions';
 import getOrgFromAuthInfo from 'lib/services/auth/authInfoSelectors/getOrgFromAuthInfo';
 
 /**
@@ -30,8 +30,7 @@ const convertOpValue = async (identifier, {
  filter,
  organisation,
  limit: Number.MAX_SAFE_INTEGER,
- maxTimeMS: MAX_TIME_MS,
- maxScan: MAX_SCAN
+ maxTimeMS: MAX_TIME_MS
  });
  attributes = map(result.edges, ({ node }) => node);
  } catch (err) {

lib/models/plugins/addCRUDFunctions.js

@@ -13,7 +13,6 @@ const FORWARDS = 'forwards';
 export const MAX_TIME_MS = process.env.MAX_TIME_MS
  ? Number(process.env.MAX_TIME_MS)
  : 0;
-export const MAX_SCAN = process.env.MAX_SCAN ? Number(process.env.MAX_SCAN) : 0;
 
 const sortDirectionToOperator = (direction, paginationDirection) => {
  if (paginationDirection === FORWARDS) {
@@ -71,9 +70,9 @@ const modelToCursor = (model, sort) => {
  return cursor;
 };
 
-const getLrsStatementCount = ({ orgFilter, maxTimeMS, maxScan, hint }) => {
+const getLrsStatementCount = ({ orgFilter, maxTimeMS, hint }) => {
  const LRS = getDBConnection().model('Lrs');
- const query = LRS.collection.find(orgFilter, {}, { maxTimeMS, maxScan });
+ const query = LRS.collection.find(orgFilter, {}, { maxTimeMS });
  if (hint) query.hint(hint);
  return query
  .project({ statementCount: 1 })
@@ -149,8 +148,7 @@ async function getConnection({
  authInfo,
  hint,
  project,
- maxTimeMS = MAX_TIME_MS,
- maxScan = MAX_SCAN
+ maxTimeMS = MAX_TIME_MS
 }) {
  const modelName = this.modelName;
  const actionName = 'view';
@@ -162,7 +160,7 @@ async function getConnection({
  actionName,
  });
 
- const query = this.find(parsedScopeFilter, {}, { maxTimeMS, maxScan });
+ const query = this.find(parsedScopeFilter, {}, { maxTimeMS });
 
  // apply sort
  query.sort(sort);
@@ -265,7 +263,6 @@ async function getCount({
  filter,
  authInfo,
  maxTimeMS = MAX_TIME_MS,
- maxScan = MAX_SCAN,
  hint
 }) {
  const parsedScopeFilter = await getParsedScopedFilter({
@@ -283,11 +280,10 @@ async function getCount({
  return getLrsStatementCount({
  orgFilter: parsedScopeFilter,
  maxTimeMS,
- maxScan,
  hint
  });
  }
- const query = this.find(parsedScopeFilter, {}, { maxTimeMS, maxScan });
+ const query = this.find(parsedScopeFilter, {}, { maxTimeMS });
  if (hint) query.hint(hint);
  return query.select({ _id: 0, organisation: 1 }).countDocuments();
 }

lib/models/statement.js

@@ -33,7 +33,6 @@ const ASYNC_AGGREGATION_CACHE_SECONDS = defaultTo(Number(process.env.ASYNC_AGGRE
 const ASYNC_AGGREGATION_TIMEOUT_MS = defaultTo(Number(process.env.ASYNC_AGGREGATION_TIMEOUT_MS), 0);
 const ASYNC_AGGREGATION_REFRESH_AFTER_SECONDS = defaultTo(Number(process.env.ASYNC_AGGREGATION_REFRESH_AFTER_SECONDS), 60); // 1 minute
 const MAX_TIME_MS = defaultTo(Number(process.env.MAX_TIME_MS), 0);
-const MAX_SCAN = defaultTo(Number(process.env.MAX_SCAN), 0);
 
 let Statement;
 const schema = new mongoose.Schema({
@@ -107,11 +106,10 @@ schema.post('remove', async (statement, next) => {
  * limit
  * batchSize
  * maxTimeMS
- * maxScan
  * }
  * @return {Promise}
  */
-const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS, maxScan }) => {
+const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS }) => {
  let query = Statement
  .aggregate(pipeline)
  .read('secondaryPreferred')
@@ -122,7 +120,6 @@ const streamAggregation = ({ pipeline, skip, limit, batchSize, maxTimeMS, maxSca
  query.options = {};
  }
  query.options.maxTimeMS = maxTimeMS;
- query.options.maxScan = maxScan;
 
  return Promise.resolve(query
  .cursor({ batchSize })
@@ -187,7 +184,6 @@ schema.statics.aggregateByAuth = function aggregateByAuth(
  batchSize = 100,
  getStream = false,
  maxTimeMS = MAX_TIME_MS,
- maxScan = MAX_SCAN,
  sampleSize = undefined
  },
  cb = () => { }
@@ -220,13 +216,12 @@ schema.statics.aggregateByAuth = function aggregateByAuth(
  limit,
  batchSize,
  maxTimeMS,
- maxScan,
  }).then(stream => cb(null, stream));
  }
 
  if (cache === false) {
  return streamAggregation({
- pipeline: finalPipeline, skip, limit, batchSize, maxTimeMS, maxScan
+ pipeline: finalPipeline, skip, limit, batchSize, maxTimeMS
  }).then(stream =>
  streamToStringResult(stream)
  ).then((result) => {
@@ -322,7 +317,6 @@ export const runAggregationAsync = async (
  limit,
  batchSize: 100,
  maxTimeMS: ASYNC_AGGREGATION_TIMEOUT_MS,
- maxScan: MAX_SCAN,
  }).then(streamToStringResult);
 
  const startedAt = await redisClient.get(`${prefix}-STARTED-AT`);