Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
2e00554
feat: add batch module
gmcrocetti Aug 13, 2020
9015d43
feat: include base processors
gmcrocetti Aug 13, 2020
48a7687
feat: add sqs failure processors
gmcrocetti Aug 13, 2020
9c7324b
fix: include proposed suggestions
gmcrocetti Aug 18, 2020
e7e36a9
feat(sqs): improve validation for queue_url
gmcrocetti Aug 25, 2020
8dbaf67
feat: add package level import for batch utility
gmcrocetti Aug 25, 2020
524d27f
refactor: change return for failure/success handlers
gmcrocetti Aug 25, 2020
7d32e8c
test: add unit tests for partial sqs processor
gmcrocetti Aug 25, 2020
f9c53e0
test: add unit tests for partial sqs processor
gmcrocetti Aug 25, 2020
680ee69
test: functional tests for partial sqs processor and its middleware
gmcrocetti Aug 25, 2020
6cddf76
feat(sqs): add optional config parameter
gmcrocetti Aug 25, 2020
13e3132
refactor(tests): processor using default config
gmcrocetti Aug 25, 2020
4275fb2
refactor(sqs): change methods to protected
gmcrocetti Aug 25, 2020
54a76df
fix(base-partial): append record instead of entry
gmcrocetti Aug 25, 2020
438ecd5
docs(sqs): docstrings for PartialSQS
gmcrocetti Aug 27, 2020
c7582f3
docs(sqs-base): docstring for base class
gmcrocetti Aug 27, 2020
f419ef6
refactor(sqs): add module middlewares
gmcrocetti Aug 27, 2020
c999f96
refactor: changes partial_sqs middleware in favor of a generic interf…
gmcrocetti Aug 27, 2020
5cb6b89
refactor(tests): update tests to new batch processor middleware
gmcrocetti Aug 27, 2020
be06149
refactor: batch middleware
gmcrocetti Aug 27, 2020
8ba81b4
docs(partial-processor): add simple docstrings to success/failure han…
gmcrocetti Aug 27, 2020
edcc14a
fix: typo in example
gmcrocetti Aug 27, 2020
ded3d75
docs: user specific documentation
gmcrocetti Aug 27, 2020
14cc383
docs: fix suggestions made by @heitorlessa
gmcrocetti Aug 29, 2020
25d67ca
refactor: remove references to BaseProcessor. Left BasePartialProcessor
gmcrocetti Aug 31, 2020
9caf3d1
docs: refactor example; improve docs about creating your own processor
gmcrocetti Aug 31, 2020
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
docs(sqs): docstrings for PartialSQS
  • Loading branch information
gmcrocetti committed Aug 27, 2020
commit 438ecd544b9ffd3251e67481ff8644a965dd3cd3
71 changes: 52 additions & 19 deletions aws_lambda_powertools/utilities/batch/sqs.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,63 @@
"""
Batch SQS utilities
"""

from typing import List, Optional
from typing import List, Optional, Tuple

import boto3
from botocore.config import Config

from aws_lambda_powertools.middleware_factory import lambda_handler_decorator

from .base import BasePartialProcessor


class PartialSQSProcessor(BasePartialProcessor):
"""
Amazon SQS batch processor to delete successes from the Queue.

Only the **special** case of partial failure is handled, thus a batch in
which all records failed is **not** going to be removed from the queue, and
the same is valid for a full success.

Parameters
----------
config: Config
botocore config object

Example
-------
**Process batch triggered by SQS**

>>> from aws_lambda_powertools.utilities.batch import PartialSQSProcessor
>>>
>>> def record_handler(record):
>>> return record["body"]
>>>
>>> def handler(event, context):
>>> records = event["Records"]
>>> processor = PartialSQSProcessor()
>>>
>>> with processor(records=records, handler=record_handler) as ctx:
>>> result = ctx.process()
>>>
>>> # Case a partial failure occurred, all successful executions
>>> # have been deleted from the queue after context's exit.
>>>
>>> return result
"""

def __init__(self, config: Optional[Config] = None):
"""
Initializes sqs client and also success and failure lists
to keep track of record's execution status.
"""
config = config or Config()
self.client = boto3.client("sqs", config=config)

self.success_messages: List = []
self.fail_messages: List = []

super().__init__()

def _get_queue_url(self):
def _get_queue_url(self) -> str:
"""
Format QueueUrl from first records entry
"""
Expand All @@ -33,13 +69,21 @@ def _get_queue_url(self):
*_, account_id, queue_name = self.records[0]["eventSourceARN"].split(":")
return f"{self.client._endpoint.host}/{account_id}/{queue_name}"

def _get_entries_to_clean(self):
def _get_entries_to_clean(self) -> List:
"""
Format messages to use in batch deletion
"""
return [{"Id": msg["messageId"], "ReceiptHandle": msg["receiptHandle"]} for msg in self.success_messages]

def _process_record(self, record):
def _process_record(self, record) -> Tuple:
"""
Process a record with instance's handler

Parameters
----------
record: Any
An object to be processed.
"""
try:
result = self.handler(record)
return self.success_handler(record, result)
Expand All @@ -48,7 +92,7 @@ def _process_record(self, record):

def _prepare(self):
"""
Remove results from previous executions.
Remove results from previous execution.
"""
self.success_messages.clear()
self.fail_messages.clear()
Expand All @@ -64,14 +108,3 @@ def _clean(self):
entries_to_remove = self._get_entries_to_clean()

return self.client.delete_message_batch(QueueUrl=queue_url, Entries=entries_to_remove)


@lambda_handler_decorator
def partial_sqs_processor(handler, event, context, record_handler, processor=None):
records = event["Records"]
processor = processor or PartialSQSProcessor()

with processor(records, record_handler) as ctx:
ctx.process()

return handler(event, context)