Updated changelog [ci skip]

Author: ht2

.env.example

@@ -193,5 +193,10 @@ FS_REPO=local
 ########
 
 # Location of virus scanning binary (ClamAV - https://www.clamav.net/)
+<<<<<<< HEAD
 #CLAMSCAN_BINARY=/usr/bin/clamscan
-RESTRICT_CREATE_ORGANISATION=true
+RESTRICT_CREATE_ORGANISATION=true
+=======
+#CLAMDSCAN_BINARY=/usr/bin/clamdscan
+#CLAMDSCAN_CONF=/etc/clamav/clamd.conf
+>>>>>>> origin/release/v2.2.5

CHANGELOG.md

@@ -11,19 +11,27 @@ Format based on [Keep a Changelog](http://keepachangelog.com/)
 
 ## [2.3.0]
 ### Added
- - Multiple shareable links per dashboard
- - Requires migration to be run
- - Aggregations now can read from secondary members on replica set
- - Sentinel Redis support
- - New role to allow organisation creation (via site admin)
+ - Multiple shareable links per dashboard (#1096)
+ - Requires migration to be run - `yarn migrate`
+ - Aggregations now can read from secondary members on replica set (#1095)
+ - Sentinel Redis support (#1119)
+ - New role to allow organisation creation (via site admin) (#1110)
+ - Widgets now auto pick visualisation name when populated (#1126)
 ### Security
- - Passwords can only be changed for the user logged in or by site admins
+ - Passwords can only be changed for the user logged in or by site admins (#1112)
 ### Fixes
- - Unicode data now pulled from dependency
- - Ensure order on personaIdentifier IFI values (fixes issue with multiple personaIdents for the same actor)
+ - Unicode data now pulled from dependency (#1125)
+ - Ensure order on personaIdentifier IFI values (fixes issue with multiple personaIdents for the same actor) (#1120)
  - Fix for personaIdentifier migration
+ - Client can select more than 10 xAPI stores (#1130)
+ - Server side validation of Statement Forward queries (#1138)
+ - Statement forwards decode `&46;` in statement keys (#1134)
+ - Persona Attribute and Identifier APIs now parse $oid values for `persona` filters (#1133)
+ - Fixed issue with hanging file imports on persona data
+ - Switch to `clamdscan` as primary AV scanner
+ - Requires updated .env settings - refer to .env.example
 ### Performance and build
- - Webpack 3 
+ - Webpack 3 - improved build speed (#1094)
 ### Migrations
 **This update requires a migration which can be run using `yarn migrate`.**
 

lib/helpers/getDateFromMongoID.js

@@ -0,0 +1,18 @@
+import isString from 'lodash/isString';
+
+export default (_id) => {
+ if (!isString(_id) || _id.length !== 24) {
+ // not a mongo id
+ throw Error('Not a Mongo ID');
+ }
+ const timehex = _id.substring(0, 8);
+
+ // convert to a number... base 16
+ const secondsSinceEpoch = parseInt(timehex, 16);
+ if (isNaN(secondsSinceEpoch)) {
+ throw Error('Invalid date in Mongo ID');
+ }
+
+ // convert to milliseconds, and create a new date
+ return new Date(secondsSinceEpoch * 1000);
+};

lib/helpers/tests/getDateFromMongoID-test.js

@@ -0,0 +1,29 @@
+import { expect } from 'chai';
+import getDateFromMongoID from '../getDateFromMongoID';
+
+const _id = '5ad52af00000000000000000';
+describe('getDateFromMongoID', () => {
+ it('should return date for Mongo ID as string', () => {
+ const date = getDateFromMongoID(_id);
+ const expectedDate = new Date('2018-04-16T23:00:00');
+ expect(date.getTime()).to.equal(expectedDate.getTime());
+ });
+
+ it('should error when _id is not a string', () => {
+ expect(getDateFromMongoID.bind(null, 123)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, false)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, undefined)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, null)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, {})).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, [])).to.throw(Error);
+ });
+
+ it('should error when _id is not 24 chars', () => {
+ expect(getDateFromMongoID.bind(null, 'abc')).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, `${_id}0`)).to.throw(Error);
+ });
+
+ it('should error when _id does not contain a valid hex date', () => {
+ expect(getDateFromMongoID.bind(null, 'Zad52af00000000000000000')).to.throw(Error);
+ });
+});

lib/models/statementForwarding.js

@@ -166,15 +166,15 @@ const schema = new mongoose.Schema({
  query: {
  type: String,
  validate: {
- validator: value => {
+ validator: (value) => {
  try {
  JSON.parse(value);
  } catch (err) {
  return false;
  }
  return true;
  },
- message: "Invalid query"
+ message: 'Invalid query'
  }
  },
  isPublic: { type: Boolean, default: false }

lib/services/files/clamscan.js

@@ -1,32 +1,35 @@
 import Promise from 'bluebird';
 import logger from 'lib/logger';
+import defaultTo from 'lodash/defaultTo';
 
-const scanError = err => (
- new Error(`Something went wrong when scanning the file. ${err.message}`)
-);
 const virusError = () => (
  new Error('This file has not passed the virus scan and will be deleted.')
 );
 export default filePath => new Promise((resolve, reject) => {
- if (!process.env.CLAMSCAN_BINARY) {
- logger.warn('CLAMSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.');
+ if (!process.env.CLAMDSCAN_BINARY) {
+ logger.warn('CLAMDSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.');
  return resolve(filePath);
  }
  try {
  const clam = require('clamscan')({
  remove_infected: true,
- clamscan: {
- path: process.env.CLAMSCAN_BINARY,
+ clamdscan: {
+ path: process.env.CLAMDSCAN_BINARY,
+ config_file: defaultTo(process.env.CLAMDSCAN_CONF, '/etc/clamav/clamd.conf')
  },
- preference: 'clamscan',
+ preference: 'clamdscan',
  });
 
  clam.is_infected(filePath, (err, file, isInfected) => {
- if (err) return reject(scanError(err));
+ if (err) {
+ logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err);
+ return resolve(filePath);
+ }
  if (isInfected) return reject(virusError());
  return resolve(filePath);
  });
  } catch (err) {
- return reject(scanError(err));
+ logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err);
+ return resolve(filePath);
  }
 });

lib/services/importPersonas/getCsvHeaders.js

@@ -1,29 +1,28 @@
-import * as fileService from 'lib/services/files';
 import csv from 'fast-csv';
 import { isUndefined, uniq } from 'lodash';
 import EmptyCsvError from 'lib/errors/EmptyCsvError';
 import DuplicateCsvHeadersError from 'lib/errors/DuplicateCsvHeadersError';
 
-const getCsvHeaders = async (handle) => {
- const csvStream = csv.parse({
+const getCsvHeaders = async (fileStream) => {
+ const csvStreamHandler = csv.parse({
  headers: false,
  quoteHeaders: true
  });
 
  // read the first row.
  let headers;
 
- csvStream.on('data', (data) => {
+ csvStreamHandler.on('data', (data) => {
  // we're just interested in the first one.
  if (isUndefined(headers)) headers = data;
  });
 
  const csvPromise = new Promise((resolve, reject) => {
- csvStream.on('error', reject);
- csvStream.on('finish', resolve);
+ csvStreamHandler.on('error', reject);
+ csvStreamHandler.on('finish', resolve);
  });
 
- fileService.downloadToStream(handle)(csvStream);
+ fileStream.pipe(csvStreamHandler);
 
  await csvPromise;
 

lib/services/importPersonas/persistPersonas.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import clamscan from 'lib/services/files/clamscan';
 import { uploadFromStream } from 'lib/services/files/storage';
 
-export const PERSONAS_CSV_PATH = '/personasCsvs';
+export const PERSONAS_CSV_PATH = 'personasCsvs';
 
 export default async ({
  file,

lib/services/importPersonas/uploadPersonas.js

@@ -1,3 +1,4 @@
+import fs from 'fs';
 import persistPersonas from 'lib/services/importPersonas/persistPersonas';
 import PersonasImport from 'lib/models/personasImport';
 import { STAGE_CONFIGURE_FIELDS } from 'lib/constants/personasImport';
@@ -29,7 +30,10 @@ const uploadPersonas = async ({
  id
  });
 
- const csvHeaders = await getCsvHeaders(handle);
+ // instead of going back out to s3, use the local file we already have!
+ const filePath = file.path;
+ const fileStream = fs.createReadStream(filePath);
+ const csvHeaders = await getCsvHeaders(fileStream);
 
  const structure = await getStructure({
  csvHeaders,

ui/src/containers/Owner/CreatedAt.js

@@ -1,4 +1,6 @@
 import moment from 'moment';
 
-export default createdAt =>
- `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`;
+export default (createdAt) => {
+ if (!createdAt) return '';
+ return `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`;
+};