Merge pull request #4726 from payloadcms/feat/migration-transactions

feat: improve transaction support by passing req to migrations

Author: jmikrut

docs/database/migrations.mdx

@@ -6,7 +6,8 @@ keywords: database, migrations, ddl, sql, mongodb, postgres, documentation, Cont
 desc: Payload features first-party database migrations all done in TypeScript.
 ---
 
-Payload exposes a full suite of migration controls available for your use. Migration commands are accessible via the `npm run payload` command in your project directory.
+Payload exposes a full suite of migration controls available for your use. Migration commands are accessible via
+the `npm run payload` command in your project directory.
 
 Ensure you have an npm script called "payload" in your `package.json` file.
 
@@ -24,33 +25,42 @@ Ensure you have an npm script called "payload" in your `package.json` file.
 
 ### Migration file contents
 
-Payload stores all created migrations in a folder that you can specify. By default, migrations are stored in `./src/migrations`.
+Payload stores all created migrations in a folder that you can specify. By default, migrations are stored
+in `./src/migrations`.
 
-A migration file has two exports - an `up` function, which is called when a migration is executed, and a `down` function that will be called if for some reason the migration fails to complete successfully. The `up` function should contain all changes that you attempt to make within the migration, and the `down` should ideally revert any changes you make.
+A migration file has two exports - an `up` function, which is called when a migration is executed, and a `down` function
+that will be called if for some reason the migration fails to complete successfully. The `up` function should contain
+all changes that you attempt to make within the migration, and the `down` should ideally revert any changes you make.
 
-For an added level of safety, migrations should leverage Payload [transactions](/docs/database/transactions).
+For an added level of safety, migrations should leverage Payload [transactions](/docs/database/transactions). Migration
+functions should make use of the `req` by adding it to the arguments of your payload local API calls such
+as `payload.create` and database adapter methods like `payload.db.create`.
 
 Here is an example migration file:
 
 ```ts
 import { MigrateUpArgs, MigrateDownArgs } from '@payloadcms/your-db-adapter'
 
-export async function up({ payload }: MigrateUpArgs): Promise<void> {
+export async function up ({ payload, req }: MigrateUpArgs): Promise<void> {
  // Perform changes to your database here.
  // You have access to `payload` as an argument, and
  // everything is done in TypeScript.
 };
 
-export async function down({ payload }: MigrateDownArgs): Promise<void> {
+export async function down ({ payload, req }: MigrateDownArgs): Promise<void> {
  // Do whatever you need to revert changes if the `up` function fails
 };
 ```
 
 ### Migrations Directory
 
-Each DB adapter has an optional property `migrationDir` where you can override where you want your migrations to be stored/read. If this is not specified, Payload will check the default and possibly make a best effort to find your migrations directory by searching in common locations ie. `./src/migrations`, `./dist/migrations`, `./migrations`, etc.
+Each DB adapter has an optional property `migrationDir` where you can override where you want your migrations to be
+stored/read. If this is not specified, Payload will check the default and possibly make a best effort to find your
+migrations directory by searching in common locations ie. `./src/migrations`, `./dist/migrations`, `./migrations`, etc.
 
-All database adapters should implement similar migration patterns, but there will be small differences based on the adapter and its specific needs. Below is a list of all migration commands that should be supported by your database adapter.
+All database adapters should implement similar migration patterns, but there will be small differences based on the
+adapter and its specific needs. Below is a list of all migration commands that should be supported by your database
+adapter.
 
 ## Commands
 
@@ -64,15 +74,17 @@ npm run payload migrate
 
 ### Create
 
-Create a new migration file in the migrations directory. You can optionally name the migration that will be created. By default, migrations will be named using a timestamp.
+Create a new migration file in the migrations directory. You can optionally name the migration that will be created. By
+default, migrations will be named using a timestamp.
 
 ```text
 npm run payload migrate:create optional-name-here
 ```
 
 ### Status
 
-The `migrate:status` command will check the status of migrations and output a table of which migrations have been run, and which migrations have not yet run.
+The `migrate:status` command will check the status of migrations and output a table of which migrations have been run,
+and which migrations have not yet run.
 
 `payload migrate:status`
 

packages/db-mongodb/src/migrateFresh.ts

@@ -1,6 +1,9 @@
 import type { PayloadRequest } from 'payload/types'
 
 import { readMigrationFiles } from 'payload/database'
+import { commitTransaction } from 'payload/dist/utilities/commitTransaction'
+import { initTransaction } from 'payload/dist/utilities/initTransaction'
+import { killTransaction } from 'payload/dist/utilities/killTransaction'
 import prompts from 'prompts'
 
 import type { MongooseAdapter } from '.'
@@ -14,9 +17,9 @@ export async function migrateFresh(this: MongooseAdapter): Promise<void> {
  const { confirm: acceptWarning } = await prompts(
  {
  name: 'confirm',
+ type: 'confirm',
  initial: false,
  message: `WARNING: This will drop your database and run all migrations. Are you sure you want to proceed?`,
- type: 'confirm',
  },
  {
  onCancel: () => {
@@ -40,29 +43,29 @@ export async function migrateFresh(this: MongooseAdapter): Promise<void> {
  msg: `Found ${migrationFiles.length} migration files.`,
  })
 
- let transactionID
+ const req = {} as PayloadRequest
+
  // Run all migrate up
  for (const migration of migrationFiles) {
  payload.logger.info({ msg: `Migrating: ${migration.name}` })
  try {
  const start = Date.now()
- transactionID = await this.beginTransaction()
- await migration.up({ payload })
+ await initTransaction(req)
+ await migration.up({ payload, req })
  await payload.create({
  collection: 'payload-migrations',
  data: {
  name: migration.name,
  batch: 1,
  },
- req: {
- transactionID,
- } as PayloadRequest,
+ req,
  })
- await this.commitTransaction(transactionID)
+
+ await commitTransaction(req)
 
  payload.logger.info({ msg: `Migrated: ${migration.name} (${Date.now() - start}ms)` })
  } catch (err: unknown) {
- await this.rollbackTransaction(transactionID)
+ await killTransaction(req)
  payload.logger.error({
  err,
  msg: `Error running migration ${migration.name}. Rolling back.`,

packages/db-postgres/src/migrate.ts

@@ -1,8 +1,12 @@
 /* eslint-disable no-restricted-syntax, no-await-in-loop */
 import type { Payload } from 'payload'
 import type { Migration } from 'payload/database'
+import type { PayloadRequest } from 'payload/dist/express/types'
 
 import { readMigrationFiles } from 'payload/database'
+import { commitTransaction } from 'payload/dist/utilities/commitTransaction'
+import { initTransaction } from 'payload/dist/utilities/initTransaction'
+import { killTransaction } from 'payload/dist/utilities/killTransaction'
 import prompts from 'prompts'
 
 import type { PostgresAdapter } from './types'
@@ -42,11 +46,11 @@ export async function migrate(this: PostgresAdapter): Promise<void> {
  const { confirm: runMigrations } = await prompts(
  {
  name: 'confirm',
+ type: 'confirm',
  initial: false,
  message:
  "It looks like you've run Payload in dev mode, meaning you've dynamically pushed changes to your database.\n\n" +
  "If you'd like to run migrations, data loss will occur. Would you like to proceed?",
- type: 'confirm',
  },
  {
  onCancel: () => {
@@ -79,14 +83,16 @@ async function runMigrationFile(payload: Payload, migration: Migration, batch: n
  const { generateDrizzleJson } = require('drizzle-kit/utils')
 
  const start = Date.now()
+ const req = {} as PayloadRequest
 
  payload.logger.info({ msg: `Migrating: ${migration.name}` })
 
  const pgAdapter = payload.db
  const drizzleJSON = generateDrizzleJson(pgAdapter.schema)
 
  try {
- await migration.up({ payload })
+ await initTransaction(req)
+ await migration.up({ payload, req })
  payload.logger.info({ msg: `Migrated: ${migration.name} (${Date.now() - start}ms)` })
  await payload.create({
  collection: 'payload-migrations',
@@ -95,8 +101,11 @@ async function runMigrationFile(payload: Payload, migration: Migration, batch: n
  batch,
  schema: drizzleJSON,
  },
+ req,
  })
+ await commitTransaction(req)
  } catch (err: unknown) {
+ await killTransaction(req)
  payload.logger.error({
  err,
  msg: parseError(err, `Error running migration ${migration.name}`),

packages/db-postgres/src/migrateDown.ts

@@ -2,6 +2,9 @@
 import type { PayloadRequest } from 'payload/types'
 
 import { getMigrations, readMigrationFiles } from 'payload/database'
+import { commitTransaction } from 'payload/dist/utilities/commitTransaction'
+import { initTransaction } from 'payload/dist/utilities/initTransaction'
+import { killTransaction } from 'payload/dist/utilities/killTransaction'
 
 import type { PostgresAdapter } from './types'
 
@@ -32,12 +35,12 @@ export async function migrateDown(this: PostgresAdapter): Promise<void> {
  }
 
  const start = Date.now()
- let transactionID
+ const req = {} as PayloadRequest
 
  try {
  payload.logger.info({ msg: `Migrating down: ${migrationFile.name}` })
- transactionID = await this.beginTransaction()
- await migrationFile.down({ payload })
+ await initTransaction(req)
+ await migrationFile.down({ payload, req })
  payload.logger.info({
  msg: `Migrated down: ${migrationFile.name} (${Date.now() - start}ms)`,
  })
@@ -47,15 +50,13 @@ export async function migrateDown(this: PostgresAdapter): Promise<void> {
  await payload.delete({
  id: migration.id,
  collection: 'payload-migrations',
- req: {
- transactionID,
- } as PayloadRequest,
+ req,
  })
  }
 
- await this.commitTransaction(transactionID)
+ await commitTransaction(req)
  } catch (err: unknown) {
- await this.rollbackTransaction(transactionID)
+ await killTransaction(req)
 
  payload.logger.error({
  err,

packages/db-postgres/src/migrateFresh.ts

@@ -2,6 +2,9 @@ import type { PayloadRequest } from 'payload/types'
 
 import { sql } from 'drizzle-orm'
 import { readMigrationFiles } from 'payload/database'
+import { commitTransaction } from 'payload/dist/utilities/commitTransaction'
+import { initTransaction } from 'payload/dist/utilities/initTransaction'
+import { killTransaction } from 'payload/dist/utilities/killTransaction'
 import prompts from 'prompts'
 
 import type { PostgresAdapter } from './types'
@@ -17,9 +20,9 @@ export async function migrateFresh(this: PostgresAdapter): Promise<void> {
  const { confirm: acceptWarning } = await prompts(
  {
  name: 'confirm',
+ type: 'confirm',
  initial: false,
  message: `WARNING: This will drop your database and run all migrations. Are you sure you want to proceed?`,
- type: 'confirm',
  },
  {
  onCancel: () => {
@@ -36,36 +39,35 @@ export async function migrateFresh(this: PostgresAdapter): Promise<void> {
  msg: `Dropping database.`,
  })
 
- await this.drizzle.execute(sql`drop schema public cascade;\ncreate schema public;`)
+ await this.drizzle.execute(sql`drop schema public cascade;
+ create schema public;`)
 
  const migrationFiles = await readMigrationFiles({ payload })
  payload.logger.debug({
  msg: `Found ${migrationFiles.length} migration files.`,
  })
 
- let transactionID
+ const req = {} as PayloadRequest
  // Run all migrate up
  for (const migration of migrationFiles) {
  payload.logger.info({ msg: `Migrating: ${migration.name}` })
  try {
  const start = Date.now()
- transactionID = await this.beginTransaction()
- await migration.up({ payload })
+ await initTransaction(req)
+ await migration.up({ payload, req })
  await payload.create({
  collection: 'payload-migrations',
  data: {
  name: migration.name,
  batch: 1,
  },
- req: {
- transactionID,
- } as PayloadRequest,
+ req,
  })
- await this.commitTransaction(transactionID)
+ await commitTransaction(req)
 
  payload.logger.info({ msg: `Migrated: ${migration.name} (${Date.now() - start}ms)` })
  } catch (err: unknown) {
- await this.rollbackTransaction(transactionID)
+ await killTransaction(req)
  payload.logger.error({
  err,
  msg: parseError(err, `Error running migration ${migration.name}. Rolling back`),

packages/db-postgres/src/migrateRefresh.ts

@@ -2,7 +2,9 @@
 import type { PayloadRequest } from 'payload/types'
 
 import { getMigrations, readMigrationFiles } from 'payload/database'
-import { DatabaseError } from 'pg'
+import { commitTransaction } from 'payload/dist/utilities/commitTransaction'
+import { initTransaction } from 'payload/dist/utilities/initTransaction'
+import { killTransaction } from 'payload/dist/utilities/killTransaction'
 
 import type { PostgresAdapter } from './types'
 
@@ -29,7 +31,7 @@ export async function migrateRefresh(this: PostgresAdapter) {
  msg: `Rolling back batch ${latestBatch} consisting of ${existingMigrations.length} migration(s).`,
  })
 
- let transactionID
+ const req = {} as PayloadRequest
 
  // Reverse order of migrations to rollback
  existingMigrations.reverse()
@@ -43,8 +45,8 @@ export async function migrateRefresh(this: PostgresAdapter) {
 
  payload.logger.info({ msg: `Migrating down: ${migration.name}` })
  const start = Date.now()
- transactionID = await this.beginTransaction()
- await migrationFile.down({ payload })
+ await initTransaction(req)
+ await migrationFile.down({ payload, req })
  payload.logger.info({
  msg: `Migrated down: ${migration.name} (${Date.now() - start}ms)`,
  })
@@ -53,18 +55,17 @@ export async function migrateRefresh(this: PostgresAdapter) {
  if (tableExists) {
  await payload.delete({
  collection: 'payload-migrations',
- req: {
- transactionID,
- } as PayloadRequest,
+ req,
  where: {
  name: {
  equals: migration.name,
  },
  },
  })
  }
+ await commitTransaction(req)
  } catch (err: unknown) {
- await this.rollbackTransaction(transactionID)
+ await killTransaction(req)
  payload.logger.error({
  err,
  msg: parseError(err, `Error running migration ${migration.name}. Rolling back.`),
@@ -78,23 +79,21 @@ export async function migrateRefresh(this: PostgresAdapter) {
  payload.logger.info({ msg: `Migrating: ${migration.name}` })
  try {
  const start = Date.now()
- transactionID = await this.beginTransaction()
- await migration.up({ payload })
+ await initTransaction(req)
+ await migration.up({ payload, req })
  await payload.create({
  collection: 'payload-migrations',
  data: {
  name: migration.name,
  executed: true,
  },
- req: {
- transactionID,
- } as PayloadRequest,
+ req,
  })
- await this.commitTransaction(transactionID)
+ await commitTransaction(req)
 
  payload.logger.info({ msg: `Migrated: ${migration.name} (${Date.now() - start}ms)` })
  } catch (err: unknown) {
- await this.rollbackTransaction(transactionID)
+ await killTransaction(req)
  payload.logger.error({
  err,
  msg: parseError(err, `Error running migration ${migration.name}. Rolling back.`),