Authentication issue with postgres14 : InternalClientError: unexpected error while performing authentication: Incorrect padding #1066
Description
- asyncpg version: 0.27.1
- PostgreSQL version: 14.6
- Do you use a PostgreSQL SaaS? If so, which? Can you reproduce
the issue with a local PostgreSQL install?: No. - Python version: python 3.8
- Platform: linux/x86-64
- Do you use pgbouncer?: No
- Did you install asyncpg with pip?: Yes
- If you built asyncpg locally, which version of Cython did you use?: No
- Can the issue be reproduced under both asyncio and
uvloop?: No, we are unable to reproduce the issue consistently
We have run into an exception InternalClientError: unexpected error while performing authentication: Incorrect padding while trying to establish connection to Aurora PostgresSQL.
This issue is not happening consistently, as more often than not, request to DB goes through successfully, but scouring through the logs, we've found few instances in past as well.
We are using username/password authentication to connect to postgres instance.
Below is the error trace
asyncpg.exceptions._base.InternalClientError: unexpected error while performing authentication: Incorrect padding return fut.result() File "/usr/local/lib/python3.8/site-packages/asyncpg/connection.py", line 2092, in connect return dialect.connect(*cargs, **cparams) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ dbapi_connection = rec.get_connection() File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_ File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 424, in checkout File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3166, in connect return super(Engine, self).connect() File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind result = context.throw(*sys.exc_info()) return await greenlet_spawn( File "/usr/local/lib/python3.8/site-packages/sqlalchemy/ext/asyncio/session.py", line 158, in scalar File "/app/./service_handler/api/deps.py", line 80, in get_account_or_404 solved_result = await solve_dependencies( raise exc from None return await self.app(scope, receive, send) result = await app(self.scope, self.receive, self.send) File "/usr/local/lib/python3.8/site-packages/asyncpg/compat.py", line 56, in wait_for value = await result File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 115, in greenlet_spawn return current.driver.switch(awaitable) await_only(self.asyncpg.connect(*arg, **kw)), return self.dbapi.connect(*cargs, **cparams) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 599, in __connect File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_ compat.raise_( compat.raise_( return fn() conn = bind.connect() return self._transaction._connection_for_bind( File "/usr/local/lib/python3.8/site-packages/sqlalchemy/ext/asyncio/session.py", line 139, in execute response = await func(request) File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 52, in app await self.app(scope, receive, sender) await self.app(scope, receive, send) asyncpg.exceptions._base.InternalClientError: unexpected error while performing authentication: Incorrect padding return await _connect_addr( await_only(self.asyncpg.connect(*arg, **kw)), raise exception compat.raise_( pool.logger.debug("Error on connect(): %s", e) self.__connect() File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 574, in get_connection File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 421, in checkout File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_ File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 424, in checkout File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 301, in connect conn = self._connection_for_bind(bind, close_with_result=True) result = context.throw(*sys.exc_info()) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/ext/asyncio/session.py", line 139, in execute account: Optional[models.Account] = await db.scalar(statement=get_account_stmt) solved_result = await solve_dependencies( await route.handle(scope, receive, send) raise exc from None return fut.result() File "/usr/local/lib/python3.8/asyncio/tasks.py", line 494, in wait_for File "/usr/local/lib/python3.8/site-packages/asyncpg/connect_utils.py", line 881, in _connect return await connect_utils._connect( value = await result File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 115, in greenlet_spawn connection = pool._invoke_creator(self) return fn() return self._wrap_pool_connect(self.pool.connect, _connection) return await greenlet_spawn( File "/app/./service_handler/crud/account.py", line 83, in lookup_account_by_company_id File "/app/./service_handler/api/deps.py", line 80, in get_account_or_404 File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 580, in __call__ File "/usr/local/lib/python3.8/site-packages/asyncpg/compat.py", line 56, in wait_for await compat.wait_for(connected, timeout=timeout) return current.driver.switch(awaitable) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 584, in connect return dialect.connect(*cargs, **cparams) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/create.py", line 578, in connect File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 599, in __connect raise exception compat.raise_( File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ return super(Engine, self).connect() File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1676, in execute File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 120, in greenlet_spawn File "/usr/local/lib/python3.8/site-packages/sqlalchemy/ext/asyncio/session.py", line 158, in scalar solved = await call(**sub_values) File "/usr/local/lib/python3.8/site-packages/fastapi/routing.py", line 204, in app await self.app(scope, receive, send) return await asyncio.wait_for(fut, timeout) File "/usr/local/lib/python3.8/site-packages/asyncpg/connect_utils.py", line 831, in __connect_addr return await __connect_addr(params, timeout, True, *args) File "/usr/local/lib/python3.8/site-packages/asyncpg/connection.py", line 2092, in connect File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/_concurrency_py3k.py", line 62, in await_only return self.dbapi.connect(*cargs, **cparams) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 605, in __connect dbapi_connection = rec.get_connection() rec._checkin_failed(err, _fairy_was_created=False) return _ConnectionFairy._checkout(self) else engine.raw_connection() return self._transaction._connection_for_bind( result = await self.execute( result = await account.lookup_account_by_company_id(db=session, account_id=account_id, company_id=company_id) File "/usr/local/lib/python3.8/site-packages/fastapi/dependencies/utils.py", line 548, in solve_dependencies response = await func(request) File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 52, in app File "/usr/local/lib/python3.8/site-packages/starlette/routing.py", line 241, in handle File "/usr/local/lib/python3.8/site-packages/starlette/exceptions.py", line 71, in __call__ File "/usr/local/lib/python3.8/site-packages/asyncpg/connect_utils.py", line 773, in _connect_addr File "/usr/local/lib/python3.8/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py", line 747, in connect File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 207, in raise_ File "/usr/local/lib/python3.8/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__ fairy = _ConnectionRecord.checkout(pool) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/pool/base.py", line 761, in _checkout File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3212, in _wrap_pool_connect File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3245, in raw_connection File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 96, in __init__ return self._connection_cls(self, close_with_result=close_with_result) File "/usr/local/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3166, in connect File "/usr/local/lib/python3.8/site-packages/sqlalchemy/future/engine.py", line 419, in connect conn = bind.connect() File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1526, in _connection_for_bind await self.app(scope, receive, sender) await self.app(scope, receive, send) File "/usr/local/lib/python3.8/site-packages/starlette/middleware/errors.py", line 159, in __call__ Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/starlette/exceptions.py", line 82, in __call__ File "/usr/local/lib/python3.8/site-packages/starlette/middleware/cors.py", line 78, in __call__ await self.app(scope, receive, _send)