We are using a lambda function with provisioned concurrency. We see two issues:
- We are trying to connect to the database in the init phase and even though the function to connect is being called (seeing Kicking off connect at module load), the connection is not set up until an actual request happens through API gateway that invokes the function.
- After the first attempt, and connecting to the database correctly, subsequent calls work fine and reuse the connection. However, even though the lambda is still warm (since it has provisioned concurrency), if there are some idle minutes without requests to the database the mongoose connection is lost and needs to reconnect. We are only seeing this issue in lambda and not locally with ts-node and the same setup.
We are more focused on solving #2.
This is a simplified stub of our implementation. The connect function is being called as well in the lambda invocation phase.
Any help / debugging / tests I could get would be great
import { logger } from '@baselime/lambda-logger'; import * as mongoose from 'mongoose'; let conn: Promise<typeof mongoose> | null = null; export const connect = async () => { if (conn == null) { const uri = 'mongodb://localhost:27017/test'; // In actual implementation it goes to secret manager to fetch connection string // Disable autoIndex in prod; build indexes separately. Recommended in docs for prod environments const shouldEnableAutoIndex = false; const minPoolSize = 5; const maxPoolSize = 100; // Retry initial connection to the database 3 times since // Mongoose doesn't retry automatically on initial connection failure const maxRetries = 3; for (let i = 1; i <= maxRetries; ++i) { try { // See the mongoose docs for more info on options: https://mongoosejs.com/docs/connections.html#options conn = mongoose .connect(uri, { minPoolSize: minPoolSize, maxPoolSize: maxPoolSize, serverSelectionTimeoutMS: 5_000, socketTimeoutMS: 20_000, autoIndex: shouldEnableAutoIndex, }) .then(() => mongoose); // `await`ing connection after assigning to the `conn` variable // to avoid multiple function calls creating new connections await conn; logger.info('Connection to database setup correctly'); setupConnectionLogging(mongoose.connection); return conn; } catch (err) { logger.warn(`Failed to connect to database on attempt ${i}/${maxRetries}`); if (i >= maxRetries) { logger.error(`Failed to connect to database after max retries (${maxRetries})`); logger.error(JSON.stringify(err)); throw err; } } } } logger.info('Connected to database'); return conn; }; const setupConnectionLogging = (connection: mongoose.Connection) => { if (connection) { connection.on('connecting', () => { logger.info('Mongoose connecting'); }); connection.on('connected', () => { logger.info('Mongoose connected'); }); connection.on('open', () => { logger.info('Mongoose open'); }); connection.on('close', () => { logger.warn('Mongoose close'); }); connection.on('disconnecting', () => { logger.warn('Mongoose disconnecting'); }); connection.on('disconnected', () => { logger.warn('Mongoose disconnected'); }); connection.on('reconnected', () => { logger.warn('Mongoose reconnected'); }); connection.on('error', (err) => { logger.error('Mongoose connection error:', err); }); } else { logger.warn('No valid Mongoose connection object available'); } }; // Kick off connect at module load (Init phase) so the connection is established before the first request logger.info('Kicking off connect at module load'); connect();