Merge branch '2.0.x' into 2.1.x

Author: AntoineRelief

migrations/1687184194639-remove-old-forms.ts

@@ -0,0 +1,48 @@
+import { Form, Record, Resource } from '@models';
+import { startDatabaseForMigration } from '../src/utils/migrations/database.helper';
+import { deleteFolder } from '@utils/files/deleteFolder';
+import { logger } from '@services/logger.service';
+
+/** This migration removes forms that are not linked to any resources */
+export const up = async () => {
+ await startDatabaseForMigration();
+
+ const forms = await Form.find({});
+ const resources = await Resource.find({});
+
+ const formsToDelete = forms.filter(
+ (f) => !resources.find((r) => r._id.equals(f.resource))
+ );
+
+ try {
+ for (const form of forms) {
+ await deleteFolder('forms', form.id || form._id);
+ logger.info(`Files from form ${form.id} successfully removed.`);
+ }
+ } catch (err) {
+ logger.error(`Deletion of files from forms failed: ${err.message}`);
+ }
+
+ // delete and log how many forms were deleted
+ const { deletedCount } = await Form.deleteMany({
+ _id: { $in: formsToDelete.map((f) => f._id) },
+ });
+ logger.info(`${deletedCount} forms were deleted.`);
+
+ // delete records that are linked to the deleted forms and log how many were deleted
+ const { deletedCount: deletedRecordsCount } = await Record.deleteMany({
+ form: { $in: formsToDelete.map((f) => f._id) },
+ });
+ logger.info(`${deletedRecordsCount} records were deleted.`);
+};
+
+/**
+ * Sample function of down migration
+ *
+ * @returns just migrate data.
+ */
+export const down = async () => {
+ /*
+ Code you downgrade script here!
+ */
+};

src/models/resource.model.ts

@@ -115,15 +115,23 @@ const resourceSchema = new Schema<Resource>(
 
 // handle cascading deletion for resources
 addOnBeforeDeleteMany(resourceSchema, async (resources) => {
+ const resourcesIds = resources.map((r) => r._id);
  try {
- const forms = await Form.find({ resource: { $in: resources } });
- for (const form of forms) {
- await deleteFolder('forms', form.id);
- logger.info(`Files from form ${form.id} successfully removed.`);
+ const forms = await Form.find({ resource: { $in: resourcesIds } });
+
+ // adding try catch because on envs without storage this was throwing an error
+ // and preventing the deletion of forms and records
+ try {
+ for (const form of forms) {
+ await deleteFolder('forms', form.id);
+ logger.info(`Files from form ${form.id} successfully removed.`);
+ }
+ } catch (err) {
+ logger.error(`Deletion of files from forms failed: ${err.message}`);
  }
 
- await Form.deleteMany({ resource: { $in: resources } });
- await Record.deleteMany({ resource: { $in: resources } });
+ await Form.deleteMany({ resource: { $in: resourcesIds } });
+ await Record.deleteMany({ resource: { $in: resourcesIds } });
  } catch (err) {
  logger.error(`Deletion of resources failed: ${err.message}`);
  }

src/utils/files/exportBatch.ts

@@ -365,7 +365,7 @@ const getRowsCsv = async (
  // todo: optimize in order to avoid using graphQL?
  const query = buildQuery(params.query);
  let offset = 0;
- const batchSize = 2000;
+ const batchSize = 1000;
  const csvData = [];
  do {
  await axios({

src/utils/schema/resolvers/Query/all.ts

@@ -607,6 +607,7 @@ export default (entityName: string, fieldsByName: any, idsByName: any) =>
  const relatedFilters = [];
  for (const item of items as any) {
  item._relatedRecords = {};
+ item.data = item.data || {};
  for (const field of resourcesFields) {
  if (field.type === 'resource') {
  const record = item.data[field.name];

src/utils/schema/resolvers/Query/getFilter.ts

@@ -2,6 +2,7 @@ import mongoose from 'mongoose';
 import { getDateForMongo } from '@utils/filter/getDateForMongo';
 import { getTimeForMongo } from '@utils/filter/getTimeForMongo';
 import { MULTISELECT_TYPES, DATE_TYPES } from '@const/fieldTypes';
+import { Resource } from '@models';
 
 /** The default fields */
 const DEFAULT_FIELDS = [
@@ -64,16 +65,20 @@ export const extractFilterFields = (filter: any): string[] => {
  * @param prefix prefix to access field
  * @returns Mongo filter.
  */
-const buildMongoFilter = (
+const buildMongoFilter = async (
  filter: any,
  fields: any[],
  context: any,
  prefix = ''
-): any => {
+): Promise<any> => {
  if (filter.filters) {
- const filters = filter.filters
- .map((x: any) => buildMongoFilter(x, fields, context, prefix))
- .filter((x) => x);
+ const filters = (
+ await Promise.all(
+ filter.filters.map((x: any) =>
+ buildMongoFilter(x, fields, context, prefix)
+ )
+ )
+ ).filter((x) => x);
  if (filters.length > 0) {
  switch (filter.logic) {
  case 'and': {
@@ -101,6 +106,28 @@ const buildMongoFilter = (
  (x) =>
  x.name === filter.field || x.name === filter.field.split('.')[0]
  )?.type || '';
+
+ // If type is resource and refers to a nested field, get the type of the nested field
+ if (type === 'resource') {
+ // find the resource field
+ const resourceField = fields.find(
+ (x) => x.name === filter.field.split('.')[0]
+ );
+
+ if (resourceField?.resource) {
+ const nestedResource = await Resource.findById(
+ resourceField.resource
+ );
+
+ // find the nested field
+ const nestedField = nestedResource?.fields.find(
+ (x) => x.name === filter.field.split('.')[1]
+ );
+
+ // get the type of the nested field
+ type = nestedField?.type || type;
+ }
+ }
  if (filter.field === 'ids') {
  return {
  _id: { $in: filter.value.map((x) => mongoose.Types.ObjectId(x)) },