chore(testinfra): check for connectivity

Author: soedirgo

.github/workflows/testinfra.yml

@@ -127,7 +127,7 @@ jobs:
  run: |
  # TODO: use poetry for pkg mgmt
  pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
- pytest -vv testinfra/test_ami.py
+ pytest -vv -s testinfra/test_ami.py
 
  - name: Cleanup resources on build cancellation
  if: ${{ cancelled() }}

testinfra/test_ami.py

@@ -5,10 +5,10 @@
 import os
 import pytest
 import requests
+import socket
 import testinfra
 from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger
 from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey
-from paramiko.ssh_exception import NoValidConnectionsError
 from time import sleep
 
 RUN_ID = os.environ.get("GITHUB_RUN_ID", "unknown-ci-run")
@@ -243,7 +243,18 @@ def gzip_then_base64_encode(s: str) -> str:
  assert response["Success"]
 
  # instance doesn't have public ip yet
- instance.reload()
+ while not instance.public_ip_address:
+ logger.warning("waiting for ip to be available")
+ sleep(5)
+ instance.reload()
+
+ while True:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ if sock.connect_ex((instance.public_ip_address, 22)) == 0:
+ break
+ else:
+ logger.warning("waiting for ssh to be available")
+ sleep(10)
 
  host = testinfra.get_host(
  # paramiko is an ssh backend
@@ -252,51 +263,46 @@ def gzip_then_base64_encode(s: str) -> str:
  )
 
  def is_healthy(host) -> bool:
- try:
- cmd = host.run("pg_isready -U postgres")
- if cmd.failed is True:
- logger.warn("pg not ready")
- return False
+ cmd = host.run("pg_isready -U postgres")
+ if cmd.failed is True:
+ logger.warning("pg not ready")
+ return False
 
-  cmd = host.run(f"curl -sf -k https://localhost:8085/health -H 'apikey: {supabase_admin_key}'")
-  if cmd.failed is True:
-  logger.warn("adminapi not ready")
-  return False
+ cmd = host.run(f"curl -sf -k https://localhost:8085/health -H 'apikey: {supabase_admin_key}'")
+ if cmd.failed is True:
+ logger.warning("adminapi not ready")
+ return False
 
-  cmd = host.run("curl -sf http://localhost:3001/ready")
-  if cmd.failed is True:
-  logger.warn("postgrest not ready")
-  return False
+ cmd = host.run("curl -sf http://localhost:3001/ready")
+ if cmd.failed is True:
+ logger.warning("postgrest not ready")
+ return False
 
-  cmd = host.run("curl -sf http://localhost:8081/health")
-  if cmd.failed is True:
-  logger.warn("gotrue not ready")
-  return False
+ cmd = host.run("curl -sf http://localhost:8081/health")
+ if cmd.failed is True:
+ logger.warning("gotrue not ready")
+ return False
 
-  cmd = host.run("sudo kong health")
-  if cmd.failed is True:
-  logger.warn("kong not ready")
-  return False
+ cmd = host.run("sudo kong health")
+ if cmd.failed is True:
+ logger.warning("kong not ready")
+ return False
 
-  cmd = host.run("printf \\\\0 > '/dev/tcp/localhost/6543'")
-  if cmd.failed is True:
-  logger.warn("pgbouncer not ready")
-  return False
+ cmd = host.run("printf \\\\0 > '/dev/tcp/localhost/6543'")
+ if cmd.failed is True:
+ logger.warning("pgbouncer not ready")
+ return False
 
- cmd = host.run("sudo fail2ban-client status")
- if cmd.failed is True:
- logger.warn("fail2ban not ready")
- return False
- except NoValidConnectionsError:
- logger.warn("unable to connect via ssh")
+ cmd = host.run("sudo fail2ban-client status")
+ if cmd.failed is True:
+ logger.warning("fail2ban not ready")
  return False
 
  return True
 
  while True:
  if is_healthy(host):
  break
- print("waiting until healthy")
  sleep(1)
 
  # return a testinfra connection to the instance