✨ Source S3: Add handling NoSuchBucket error (#31383)

Author: tolik0

airbyte-integrations/connectors/source-s3/metadata.yaml

@@ -10,7 +10,7 @@ data:
  connectorSubtype: file
  connectorType: source
  definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
- dockerImageTag: 4.1.1
+ dockerImageTag: 4.1.2
  dockerRepository: airbyte/source-s3
  documentationUrl: https://docs.airbyte.com/integrations/sources/s3
  githubIssueLabel: source-s3

airbyte-integrations/connectors/source-s3/setup.py

@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
- "airbyte-cdk>=0.51.35",
+ "airbyte-cdk>=0.52.0",
  "pyarrow==12.0.1",
  "smart-open[s3]==5.1.0",
  "wcmatch==8.4",

airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py

@@ -10,11 +10,13 @@
 import boto3.session
 import pytz
 import smart_open
-from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from botocore.client import BaseClient
 from botocore.client import Config as ClientConfig
+from botocore.exceptions import ClientError
 from source_s3.v4.config import Config
 from source_s3.v4.zip_reader import DecompressedStream, RemoteFileInsideArchive, ZipContentReader, ZipFileHandler
 
@@ -68,25 +70,30 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo
  total_n_keys = 0
 
  try:
- if prefixes:
- for prefix in prefixes:
- for remote_file in self._page(s3, globs, self.config.bucket, prefix, seen, logger):
- total_n_keys += 1
- yield remote_file
- else:
- for remote_file in self._page(s3, globs, self.config.bucket, None, seen, logger):
+ for current_prefix in prefixes if prefixes else [None]:
+ for remote_file in self._page(s3, globs, self.config.bucket, current_prefix, seen, logger):
  total_n_keys += 1
  yield remote_file
 
  logger.info(f"Finished listing objects from S3. Found {total_n_keys} objects total ({len(seen)} unique objects).")
+ except ClientError as exc:
+ if exc.response["Error"]["Code"] == "NoSuchBucket":
+ raise CustomFileBasedException(
+ f"The bucket {self.config.bucket} does not exist.", failure_type=FailureType.config_error, exception=exc
+ )
+ self._raise_error_listing_files(globs, exc)
  except Exception as exc:
- raise ErrorListingFiles(
- FileBasedSourceError.ERROR_LISTING_FILES,
- source="s3",
- bucket=self.config.bucket,
- globs=globs,
- endpoint=self.config.endpoint,
- ) from exc
+ self._raise_error_listing_files(globs, exc)
+
+ def _raise_error_listing_files(self, globs: List[str], exc: Optional[Exception] = None):
+ """Helper method to raise the ErrorListingFiles exception."""
+ raise ErrorListingFiles(
+ FileBasedSourceError.ERROR_LISTING_FILES,
+ source="s3",
+ bucket=self.config.bucket,
+ globs=globs,
+ endpoint=self.config.endpoint,
+ ) from exc
 
  def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase:
  try:

docs/integrations/sources/s3.md

@@ -250,7 +250,8 @@ One record will be emitted for each document. Keep in mind that large files can
 ## Changelog
 
 | Version | Date | Pull Request | Subject |
-| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- |
+|:--------|:-----------| :-------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- |
+| 4.1.2 | 2023-10-23 | [31383](https://github.com/airbytehq/airbyte/pull/31383) | Add handling NoSuchBucket error |
 | 4.1.1 | 2023-10-19 | [31601](https://github.com/airbytehq/airbyte/pull/31601) | Base image migration: remove Dockerfile and use the python-connector-base image |
 | 4.1.0 | 2023-10-17 | [31340](https://github.com/airbytehq/airbyte/pull/31340) | Add reading files inside zip archive |
 | 4.0.5 | 2023-10-16 | [31209](https://github.com/airbytehq/airbyte/pull/31209) | Add experimental Markdown/PDF/Docx file format |