Merge branch 'develop' of https://github.com/awslabs/aws-lambda-powertools-python into develop

* 'develop' of https://github.com/awslabs/aws-lambda-powertools-python: feat(feature-flags): improve "IN/NOT_IN"; new rule actions (aws-powertools#710) feat(idempotency): makes customers unit testing easier (aws-powertools#719) feat(feature-flags): get_raw_configuration property in Store (aws-powertools#720)

Author: heitorlessa

aws_lambda_powertools/shared/constants.py

@@ -21,3 +21,5 @@
 
 XRAY_SDK_MODULE: str = "aws_xray_sdk"
 XRAY_SDK_CORE_MODULE: str = "aws_xray_sdk.core"
+
+IDEMPOTENCY_DISABLED_ENV: str = "POWERTOOLS_IDEMPOTENCY_DISABLED"

aws_lambda_powertools/utilities/feature_flags/feature_flags.py

@@ -48,6 +48,10 @@ def _match_by_action(action: str, condition_value: Any, context_value: Any) -> b
  schema.RuleAction.ENDSWITH.value: lambda a, b: a.endswith(b),
  schema.RuleAction.IN.value: lambda a, b: a in b,
  schema.RuleAction.NOT_IN.value: lambda a, b: a not in b,
+ schema.RuleAction.KEY_IN_VALUE.value: lambda a, b: a in b,
+ schema.RuleAction.KEY_NOT_IN_VALUE.value: lambda a, b: a not in b,
+ schema.RuleAction.VALUE_IN_KEY.value: lambda a, b: b in a,
+ schema.RuleAction.VALUE_NOT_IN_KEY.value: lambda a, b: b not in a,
  }
 
  try:

aws_lambda_powertools/utilities/feature_flags/schema.py

@@ -22,6 +22,10 @@ class RuleAction(str, Enum):
  ENDSWITH = "ENDSWITH"
  IN = "IN"
  NOT_IN = "NOT_IN"
+ KEY_IN_VALUE = "KEY_IN_VALUE"
+ KEY_NOT_IN_VALUE = "KEY_NOT_IN_VALUE"
+ VALUE_IN_KEY = "VALUE_IN_KEY"
+ VALUE_NOT_IN_KEY = "VALUE_NOT_IN_KEY"
 
 
 class SchemaValidator(BaseValidator):
@@ -80,7 +84,9 @@ class SchemaValidator(BaseValidator):
  The value MUST contain the following members:
 
  * **action**: `str`. Operation to perform to match a key and value.
- The value MUST be either EQUALS, STARTSWITH, ENDSWITH, IN, NOT_IN
+ The value MUST be either EQUALS, STARTSWITH, ENDSWITH,
+ KEY_IN_VALUE KEY_NOT_IN_VALUE VALUE_IN_KEY VALUE_NOT_IN_KEY
+
  * **key**: `str`. Key in given context to perform operation
  * **value**: `Any`. Value in given context that should match action operation.
 

aws_lambda_powertools/utilities/idempotency/idempotency.py

@@ -3,9 +3,11 @@
 """
 import functools
 import logging
+import os
 from typing import Any, Callable, Dict, Optional, cast
 
 from aws_lambda_powertools.middleware_factory import lambda_handler_decorator
+from aws_lambda_powertools.shared.constants import IDEMPOTENCY_DISABLED_ENV
 from aws_lambda_powertools.shared.types import AnyCallableT
 from aws_lambda_powertools.utilities.idempotency.base import IdempotencyHandler
 from aws_lambda_powertools.utilities.idempotency.config import IdempotencyConfig
@@ -56,6 +58,9 @@ def idempotent(
  >>> return {"StatusCode": 200}
  """
 
+ if os.getenv(IDEMPOTENCY_DISABLED_ENV):
+ return handler(event, context)
+
  config = config or IdempotencyConfig()
  args = event, context
  idempotency_handler = IdempotencyHandler(
@@ -122,6 +127,9 @@ def process_order(customer_id: str, order: dict, **kwargs):
 
  @functools.wraps(function)
  def decorate(*args, **kwargs):
+ if os.getenv(IDEMPOTENCY_DISABLED_ENV):
+ return function(*args, **kwargs)
+
  payload = kwargs.get(data_keyword_argument)
 
  if payload is None:

aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py

@@ -62,18 +62,37 @@ def __init__(
  >>> return {"StatusCode": 200}
  """
 
- boto_config = boto_config or Config()
- session = boto3_session or boto3.session.Session()
- self._ddb_resource = session.resource("dynamodb", config=boto_config)
+ self._boto_config = boto_config or Config()
+ self._boto3_session = boto3_session or boto3.session.Session()
+
+ self._table = None
  self.table_name = table_name
- self.table = self._ddb_resource.Table(self.table_name)
  self.key_attr = key_attr
  self.expiry_attr = expiry_attr
  self.status_attr = status_attr
  self.data_attr = data_attr
  self.validation_key_attr = validation_key_attr
  super(DynamoDBPersistenceLayer, self).__init__()
 
+ @property
+ def table(self):
+ """
+ Caching property to store boto3 dynamodb Table resource
+
+ """
+ if self._table:
+ return self._table
+ ddb_resource = self._boto3_session.resource("dynamodb", config=self._boto_config)
+ self._table = ddb_resource.Table(self.table_name)
+ return self._table
+
+ @table.setter
+ def table(self, table):
+ """
+ Allow table instance variable to be set directly, primarily for use in tests
+ """
+ self._table = table
+
  def _item_to_data_record(self, item: Dict[str, Any]) -> DataRecord:
  """
  Translate raw item records from DynamoDB to DataRecord
@@ -125,7 +144,7 @@ def _put_record(self, data_record: DataRecord) -> None:
  ExpressionAttributeNames={"#id": self.key_attr, "#now": self.expiry_attr},
  ExpressionAttributeValues={":now": int(now.timestamp())},
  )
- except self._ddb_resource.meta.client.exceptions.ConditionalCheckFailedException:
+ except self.table.meta.client.exceptions.ConditionalCheckFailedException:
  logger.debug(f"Failed to put record for already existing idempotency key: {data_record.idempotency_key}")
  raise IdempotencyItemAlreadyExistsError
 

docs/utilities/feature_flags.md

@@ -366,7 +366,7 @@ You can use `get_enabled_features` method for scenarios where you need a list of
   "when_match": true,
   "conditions": [
   {
-  "action": "IN",
+  "action": "KEY_IN_VALUE",
   "key": "CloudFront-Viewer-Country",
   "value": ["NL", "IE", "UK", "PL", "PT"]
   }
@@ -450,9 +450,20 @@ The `conditions` block is a list of conditions that contain `action`, `key`, and
  }
  ```
 
-The `action` configuration can have 5 different values: `EQUALS`, `STARTSWITH`, `ENDSWITH`, `IN`, `NOT_IN`.
+The `action` configuration can have the following values, where the expressions **`a`** is the `key` and **`b`** is the `value` above:
 
-The `key` and `value` will be compared to the input from the context parameter.
+Action | Equivalent expression
+------------------------------------------------- | ---------------------------------------------------------------------------------
+**EQUALS** | `lambda a, b: a == b`
+**STARTSWITH** | `lambda a, b: a.startswith(b)`
+**ENDSWITH** | `lambda a, b: a.endswith(b)`
+**KEY_IN_VALUE** | `lambda a, b: a in b`
+**KEY_NOT_IN_VALUE** | `lambda a, b: a not in b`
+**VALUE_IN_KEY** | `lambda a, b: b in a`
+**VALUE_NOT_IN_KEY** | `lambda a, b: b not in a`
+
+
+!!! info "The `**key**` and `**value**` will be compared to the input from the `**context**` parameter."
 
 **For multiple conditions**, we will evaluate the list of conditions as a logical `AND`, so all conditions needs to match to return `when_match` value.
 
@@ -529,6 +540,27 @@ For this to work, you need to use a JMESPath expression via the `envelope` param
  }
  ```
 
+### Getting fetched configuration
+
+You can access the configuration fetched from the store via `get_raw_configuration` property within the store instance.
+
+=== "app.py"
+
+ ```python hl_lines="12"
+ from aws_lambda_powertools.utilities.feature_flags import FeatureFlags, AppConfigStore
+
+ app_config = AppConfigStore(
+  environment="dev",
+  application="product-catalogue",
+  name="configuration",
+  envelope = "feature_flags"
+ )
+
+feature_flags = FeatureFlags(store=app_config)
+
+config = app_config.get_raw_configuration
+ ```
+
 ### Built-in store provider
 
 !!! info "For GA, you'll be able to bring your own store."
@@ -650,3 +682,11 @@ Method | When to use | Requires new deployment on changes | Supported services
 **[Environment variables](https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html){target="_blank"}** | Simple configuration that will rarely if ever change, because changing it requires a Lambda function deployment. | Yes | Lambda
 **[Parameters utility](parameters.md)** | Access to secrets, or fetch parameters in different formats from AWS System Manager Parameter Store or Amazon DynamoDB. | No | Parameter Store, DynamoDB, Secrets Manager, AppConfig
 **Feature flags utility** | Rule engine to define when one or multiple features should be enabled depending on the input. | No | AppConfig
+
+
+## Deprecation list when GA
+
+Breaking change | Recommendation
+------------------------------------------------- | ---------------------------------------------------------------------------------
+`IN` RuleAction | Use `KEY_IN_VALUE` instead
+`NOT_IN` RuleAction | Use `KEY_NOT_IN_VALUE` instead

docs/utilities/idempotency.md

@@ -765,6 +765,123 @@ The idempotency utility can be used with the `validator` decorator. Ensure that
 !!! tip "JMESPath Powertools functions are also available"
  Built-in functions known in the validation utility like `powertools_json`, `powertools_base64`, `powertools_base64_gzip` are also available to use in this utility.
 
+
+## Testing your code
+
+The idempotency utility provides several routes to test your code.
+
+### Disabling the idempotency utility
+When testing your code, you may wish to disable the idempotency logic altogether and focus on testing your business logic. To do this, you can set the environment variable `POWERTOOLS_IDEMPOTENCY_DISABLED`
+with a truthy value. If you prefer setting this for specific tests, and are using Pytest, you can use [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) fixture:
+
+=== "tests.py"
+
+ ```python hl_lines="2 3"
+ def test_idempotent_lambda_handler(monkeypatch):
+  # Set POWERTOOLS_IDEMPOTENCY_DISABLED before calling decorated functions
+  monkeypatch.setenv("POWERTOOLS_IDEMPOTENCY_DISABLED", 1)
+
+  result = handler()
+  ...
+ ```
+=== "app.py"
+
+ ```python
+ from aws_lambda_powertools.utilities.idempotency import (
+ DynamoDBPersistenceLayer, idempotent
+ )
+
+ persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+ @idempotent(persistence_store=persistence_layer)
+ def handler(event, context):
+  print('expensive operation')
+  return {
+  "payment_id": 12345,
+  "message": "success",
+  "statusCode": 200,
+  }
+ ```
+
+### Testing with DynamoDB Local
+
+To test with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html), you can replace the `Table` resource used by the persistence layer with one you create inside your tests. This allows you to set the endpoint_url.
+
+=== "tests.py"
+
+ ```python hl_lines="6 7 8"
+ import boto3
+
+ import app
+
+ def test_idempotent_lambda():
+  # Create our own Table resource using the endpoint for our DynamoDB Local instance
+  resource = boto3.resource("dynamodb", endpoint_url='http://localhost:8000')
+  table = resource.Table(app.persistence_layer.table_name)
+  app.persistence_layer.table = table
+
+  result = app.handler({'testkey': 'testvalue'}, {})
+  assert result['payment_id'] == 12345
+ ```
+
+=== "app.py"
+
+ ```python
+ from aws_lambda_powertools.utilities.idempotency import (
+ DynamoDBPersistenceLayer, idempotent
+ )
+
+ persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+ @idempotent(persistence_store=persistence_layer)
+ def handler(event, context):
+  print('expensive operation')
+  return {
+  "payment_id": 12345,
+  "message": "success",
+  "statusCode": 200,
+  }
+ ```
+
+### How do I mock all DynamoDB I/O operations
+
+The idempotency utility lazily creates the dynamodb [Table](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#table) which it uses to access DynamoDB.
+This means it is possible to pass a mocked Table resource, or stub various methods.
+
+=== "tests.py"
+
+ ```python hl_lines="6 7 8 9"
+ from unittest.mock import MagicMock
+
+ import app
+
+ def test_idempotent_lambda():
+  table = MagicMock()
+  app.persistence_layer.table = table
+  result = app.handler({'testkey': 'testvalue'}, {})
+  table.put_item.assert_called()
+  ...
+ ```
+
+=== "app.py"
+
+ ```python
+ from aws_lambda_powertools.utilities.idempotency import (
+ DynamoDBPersistenceLayer, idempotent
+ )
+
+ persistence_layer = DynamoDBPersistenceLayer(table_name="idempotency")
+
+ @idempotent(persistence_store=persistence_layer)
+ def handler(event, context):
+  print('expensive operation')
+  return {
+  "payment_id": 12345,
+  "message": "success",
+  "statusCode": 200,
+  }
+ ```
+
 ## Extra resources
 
 If you're interested in a deep dive on how Amazon uses idempotency when building our APIs, check out