CI small fix (#616)

* wait for principals to be removed before deleting iot thing in CI

Author: sbSteveK

utils/ci_iot_thing.py

@@ -3,8 +3,46 @@
 
 import sys
 
-import boto3
-
+import boto3, time
+from botocore.exceptions import ClientError, WaiterError
+
+class ThingDetachedWaiter:
+ """
+ Wait until principal (cert or Cognito identity) is detached from a thing.
+ Raise WaiterError after timeout seconds.
+ """
+
+ def __init__(self, client, delay=2.0, max_delay=10.0, timeout=60.0):
+ self._client = client
+ self._delay = delay
+ self._max_delay = max_delay
+ self._timeout = timeout
+
+ def wait(self, thing_name):
+ start = time.monotonic()
+ sleep = self._delay
+
+ while True:
+ try:
+ resp = self._client.list_thing_principals(thingName=thing_name)
+ except ClientError as e:
+ if e.response["Error"]["Code"] == "ResourceNotFoundException":
+ return
+ raise
+
+ if not resp.get("principals"):
+ # No principals, we can move on.
+ return
+
+ if time.monotonic() - start > self._timeout:
+ raise WaiterError(
+ name="ThingDetached",
+ reason="timeout",
+ last_response=resp,
+ )
+
+ time.sleep(sleep)
+ sleep = min(sleep * 1.6, self._max_delay) # exponential backoff on retrys
 
 def create_iot_thing(thing_name, region, policy_name, certificate_path, key_path, thing_group=None):
  """ Create IoT thing along with policy and credentials. """
@@ -57,6 +95,8 @@ def delete_iot_thing(thing_name, region):
  print(f"ERROR: Could not make Boto3 client. Credentials likely could not be sourced", file=sys.stderr)
  raise
 
+ cert_ids = []
+
  # Detach and delete thing's principals.
  try:
  thing_principals = iot_client.list_thing_principals(thingName=thing_name)
@@ -65,12 +105,24 @@ def delete_iot_thing(thing_name, region):
  certificate_id = principal.split("/")[1]
  iot_client.detach_thing_principal(thingName=thing_name, principal=principal)
  iot_client.update_certificate(certificateId=certificate_id, newStatus='INACTIVE')
- iot_client.delete_certificate(certificateId=certificate_id, forceDelete=True)
+ cert_ids.append(certificate_id)
  except Exception:
- print("ERROR: Could not delete certificate for IoT thing {thing_name}, probably thing does not exist",
+ print("ERROR: Could not detatch principal or set its certificate to INACTIVE for {thing_name}, probably thing does not exist",
  file=sys.stderr)
  raise
 
+ # Wait for thing to be free of principals
+ ThingDetachedWaiter(iot_client, timeout=10).wait(thing_name)
+
+ # Delete all the certificates
+ for cert in cert_ids:
+ try:
+ iot_client.delete_certificate(certificateId=cert, forceDelete=True)
+ except Exception:
+ print("ERROR: Could not delete certificate for IoT thing {thing_name}.",
+ file=sys.stderr)
+ raise
+
  # Delete thing.
  try:
  iot_client.delete_thing(thingName=thing_name)