fix: Use local file when importing csvs, and use clamd (LearningLocker#1141)

* fix: Use local file when importing csvs * ci(lint): Remove unused import * fix: Use clamdscan for faster scanning * Update .env.example * Update .env.example * fix: Handle clamscan failures greacefully * ci(lint): Linting fixes * fix: Show persona dates * ci(lint): Fix linting

Author: ht2

.env.example

@@ -193,4 +193,5 @@ FS_REPO=local
 ########
 
 # Location of virus scanning binary (ClamAV - https://www.clamav.net/)
-#CLAMSCAN_BINARY=/usr/bin/clamscan
+#CLAMDSCAN_BINARY=/usr/bin/clamdscan
+#CLAMDSCAN_CONF=/etc/clamav/clamd.conf

lib/helpers/getDateFromMongoID.js

@@ -0,0 +1,18 @@
+import isString from 'lodash/isString';
+
+export default (_id) => {
+ if (!isString(_id) || _id.length !== 24) {
+ // not a mongo id
+ throw Error('Not a Mongo ID');
+ }
+ const timehex = _id.substring(0, 8);
+
+ // convert to a number... base 16
+ const secondsSinceEpoch = parseInt(timehex, 16);
+ if (isNaN(secondsSinceEpoch)) {
+ throw Error('Invalid date in Mongo ID');
+ }
+
+ // convert to milliseconds, and create a new date
+ return new Date(secondsSinceEpoch * 1000);
+};

lib/helpers/tests/getDateFromMongoID-test.js

@@ -0,0 +1,29 @@
+import { expect } from 'chai';
+import getDateFromMongoID from '../getDateFromMongoID';
+
+const _id = '5ad52af00000000000000000';
+describe('getDateFromMongoID', () => {
+ it('should return date for Mongo ID as string', () => {
+ const date = getDateFromMongoID(_id);
+ const expectedDate = new Date('2018-04-16T23:00:00');
+ expect(date.getTime()).to.equal(expectedDate.getTime());
+ });
+
+ it('should error when _id is not a string', () => {
+ expect(getDateFromMongoID.bind(null, 123)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, false)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, undefined)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, null)).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, {})).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, [])).to.throw(Error);
+ });
+
+ it('should error when _id is not 24 chars', () => {
+ expect(getDateFromMongoID.bind(null, 'abc')).to.throw(Error);
+ expect(getDateFromMongoID.bind(null, `${_id}0`)).to.throw(Error);
+ });
+
+ it('should error when _id does not contain a valid hex date', () => {
+ expect(getDateFromMongoID.bind(null, 'Zad52af00000000000000000')).to.throw(Error);
+ });
+});

lib/models/statementForwarding.js

@@ -166,15 +166,15 @@ const schema = new mongoose.Schema({
  query: {
  type: String,
  validate: {
- validator: value => {
+ validator: (value) => {
  try {
  JSON.parse(value);
  } catch (err) {
  return false;
  }
  return true;
  },
- message: "Invalid query"
+ message: 'Invalid query'
  }
  },
  isPublic: { type: Boolean, default: false }

lib/services/files/clamscan.js

@@ -1,32 +1,35 @@
 import Promise from 'bluebird';
 import logger from 'lib/logger';
+import defaultTo from 'lodash/defaultTo';
 
-const scanError = err => (
- new Error(`Something went wrong when scanning the file. ${err.message}`)
-);
 const virusError = () => (
  new Error('This file has not passed the virus scan and will be deleted.')
 );
 export default filePath => new Promise((resolve, reject) => {
- if (!process.env.CLAMSCAN_BINARY) {
- logger.warn('CLAMSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.');
+ if (!process.env.CLAMDSCAN_BINARY) {
+ logger.warn('CLAMDSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.');
  return resolve(filePath);
  }
  try {
  const clam = require('clamscan')({
  remove_infected: true,
- clamscan: {
- path: process.env.CLAMSCAN_BINARY,
+ clamdscan: {
+ path: process.env.CLAMDSCAN_BINARY,
+ config_file: defaultTo(process.env.CLAMDSCAN_CONF, '/etc/clamav/clamd.conf')
  },
- preference: 'clamscan',
+ preference: 'clamdscan',
  });
 
  clam.is_infected(filePath, (err, file, isInfected) => {
- if (err) return reject(scanError(err));
+ if (err) {
+ logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err);
+ return resolve(filePath);
+ }
  if (isInfected) return reject(virusError());
  return resolve(filePath);
  });
  } catch (err) {
- return reject(scanError(err));
+ logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err);
+ return resolve(filePath);
  }
 });

lib/services/importPersonas/getCsvHeaders.js

@@ -1,29 +1,28 @@
-import * as fileService from 'lib/services/files';
 import csv from 'fast-csv';
 import { isUndefined, uniq } from 'lodash';
 import EmptyCsvError from 'lib/errors/EmptyCsvError';
 import DuplicateCsvHeadersError from 'lib/errors/DuplicateCsvHeadersError';
 
-const getCsvHeaders = async (handle) => {
- const csvStream = csv.parse({
+const getCsvHeaders = async (fileStream) => {
+ const csvStreamHandler = csv.parse({
  headers: false,
  quoteHeaders: true
  });
 
  // read the first row.
  let headers;
 
- csvStream.on('data', (data) => {
+ csvStreamHandler.on('data', (data) => {
  // we're just interested in the first one.
  if (isUndefined(headers)) headers = data;
  });
 
  const csvPromise = new Promise((resolve, reject) => {
- csvStream.on('error', reject);
- csvStream.on('finish', resolve);
+ csvStreamHandler.on('error', reject);
+ csvStreamHandler.on('finish', resolve);
  });
 
- fileService.downloadToStream(handle)(csvStream);
+ fileStream.pipe(csvStreamHandler);
 
  await csvPromise;
 

lib/services/importPersonas/persistPersonas.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import clamscan from 'lib/services/files/clamscan';
 import { uploadFromStream } from 'lib/services/files/storage';
 
-export const PERSONAS_CSV_PATH = '/personasCsvs';
+export const PERSONAS_CSV_PATH = 'personasCsvs';
 
 export default async ({
  file,

lib/services/importPersonas/uploadPersonas.js

@@ -1,3 +1,4 @@
+import fs from 'fs';
 import persistPersonas from 'lib/services/importPersonas/persistPersonas';
 import PersonasImport from 'lib/models/personasImport';
 import { STAGE_CONFIGURE_FIELDS } from 'lib/constants/personasImport';
@@ -29,7 +30,10 @@ const uploadPersonas = async ({
  id
  });
 
- const csvHeaders = await getCsvHeaders(handle);
+ // instead of going back out to s3, use the local file we already have!
+ const filePath = file.path;
+ const fileStream = fs.createReadStream(filePath);
+ const csvHeaders = await getCsvHeaders(fileStream);
 
  const structure = await getStructure({
  csvHeaders,

ui/src/containers/Owner/CreatedAt.js

@@ -1,4 +1,6 @@
 import moment from 'moment';
 
-export default createdAt =>
- `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`;
+export default (createdAt) => {
+ if (!createdAt) return '';
+ return `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`;
+};

ui/src/containers/Owner/index.js

@@ -1,24 +1,41 @@
 import React from 'react';
+import getDateFromMongoID from 'lib/helpers/getDateFromMongoID';
 import { withProps, compose } from 'recompose';
 import { withModel } from 'ui/utils/hocs';
 import createdAtFormatter from './CreatedAt';
 
+const getCreatedDate = (createdAt, _id) => {
+ if (!createdAt && _id) {
+ try {
+ return getDateFromMongoID(_id);
+ } catch (err) {
+ // error getting date from id (might not be a valid mongo id)
+ return;
+ }
+ }
+ return createdAt;
+};
+
 const Creator = compose(
  withProps(({ model }) => ({ schema: 'user', id: model.get('owner') })),
  withModel
-)(({ model, createdAt }) => {
+)(({ model, date }) => {
  const email = model.get('email', false);
  const name = model.get('name', false);
  return (
  <div>
  <div style={{ fontWeight: 'bold' }}>Made by { email ? <i>{name || email}</i> : <i>deleted user</i> }</div>
- <div>{ createdAtFormatter(createdAt) }</div>
+ <div>{ date }</div>
  </div>
  );
 });
 
-export default ({ model }) => (
- model.has('owner')
- ? <Creator model={model} createdAt={model.get('createdAt', false)} />
- : <div style={{ marginTop: 8 }}>{ createdAtFormatter(model.get('createdAt')) }</div>
-);
+export default ({ model }) => {
+ const date = getCreatedDate(model.get('createdAt', false), model.get('_id'));
+ const formattedDate = createdAtFormatter(date);
+ return (
+ model.has('owner')
+ ? <Creator model={model} date={formattedDate} />
+ : <div style={{ marginTop: 8 }}>{ formattedDate }</div>
+ );
+};