airbytehq · davydov-d · Oct 14, 2022 · Oct 14, 2022 · Oct 14, 2022 · Oct 14, 2022
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -915,7 +915,7 @@
 - name: S3
  sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
  dockerRepository: airbyte/source-s3
- dockerImageTag: 0.1.23
+ dockerImageTag: 0.1.24
  documentationUrl: https://docs.airbyte.com/integrations/sources/s3
  icon: s3.svg
  sourceType: file

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -9436,7 +9436,7 @@
  supportsNormalization: false
  supportsDBT: false
  supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-s3:0.1.23"
+- dockerImage: "airbyte/source-s3:0.1.24"
  spec:
  documentationUrl: "https://docs.airbyte.com/integrations/sources/s3"
  changelogUrl: "https://docs.airbyte.com/integrations/sources/s3"

diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile
@@ -17,5 +17,5 @@ COPY source_s3 ./source_s3
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.23
+LABEL io.airbyte.version=0.1.24
 LABEL io.airbyte.name=airbyte/source-s3
diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json
@@ -6,7 +6,7 @@
  "aws_access_key_id": "123456",
  "aws_secret_access_key": "123456key",
  "path_prefix": "",
- "endpoint": "http://10.0.167.14:9000"
+ "endpoint": "http://10.0.92.4:9000"
  },
  "format": {
  "filetype": "csv"

diff --git a/...ions/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py b/...ions/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py
@@ -13,7 +13,7 @@
 class AbstractFileParser(ABC):
  logger = AirbyteLogger()
 
- NON_SCALAR_TYPES = {"struct": "struct"}
+ NON_SCALAR_TYPES = {"struct": "struct", "list": "list"}
  TYPE_MAP = {
  "boolean": ("bool_", "bool"),
  "integer": ("int64", "int8", "int16", "int32", "uint8", "uint16", "uint32", "uint64"),

diff --git a/...integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py b/...integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py
@@ -24,7 +24,10 @@ class JsonlParser(AbstractFileParser):
  "large_string",
  ),
  # TODO: support array type rather than coercing to string
- "array": ("large_string",),
+ "array": (
+ "list",
+ "large_string",
+ ),
  "null": ("large_string",),
  }
 
@@ -80,6 +83,8 @@ def get_inferred_schema(self, file: Union[TextIO, BinaryIO]) -> Mapping[str, Any
  def field_type_to_str(type_: Any) -> str:
  if isinstance(type_, pa.lib.StructType):
  return "struct"
+ if isinstance(type_, pa.lib.ListType):
+ return "list"
  if isinstance(type_, pa.lib.DataType):
  return str(type_)
  raise Exception(f"Unknown PyArrow Type: {type_}")

diff --git a/...ons/connectors/source-s3/unit_tests/sample_files/jsonl/test_file_11_array_in_schema.jsonl b/...ons/connectors/source-s3/unit_tests/sample_files/jsonl/test_file_11_array_in_schema.jsonl
@@ -0,0 +1,3 @@
+{"id": 1, "name": "Erich", "books": ["Shadows in Paradise", "The Dream Room", "The Night in Lisbon"]}
+{"id": 2, "name": "Maria", "books": ["All Quiet on the Western Front"]}
+{"id": 3, "name": "Remarque", "books": ["The Road Back", "Three Comrades"]}
diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_jsonl_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_jsonl_parser.py
@@ -168,4 +168,12 @@ def cases(cls) -> Mapping[str, Any]:
  "line_checks": {},
  "fails": [],
  },
+ "array_in_schema_test": {
+ "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}),
+ "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_11_array_in_schema.jsonl"),
+ "num_records": 3,
+ "inferred_schema": {"id": "integer", "name": "string", "books": "array"},
+ "line_checks": {},
+ "fails": [],
+ },
  }
diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md
@@ -204,7 +204,8 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo
 ## Changelog
 
 | Version | Date | Pull Request | Subject |
-| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------- |
+|:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------|
+| 0.1.23 | 2022-10-10 | [17991](https://github.com/airbytehq/airbyte/pull/17991) | Fix pyarrow to JSON schema type conversion for arrays |
 | 0.1.23 | 2022-10-10 | [17800](https://github.com/airbytehq/airbyte/pull/17800) | Deleted `use_ssl` and `verify_ssl_cert` flags and hardcoded to `True` |
 | 0.1.22 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state |
 | 0.1.21 | 2022-09-20 | [16921](https://github.com/airbytehq/airbyte/pull/16921) | Upgrade pyarrow |