🐛 Source HubSpot: Fix empty string inside number / float datatype (#5334)

#5293 - Source Hubspot fails in normalization step Co-authored-by: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>

Author: bazarnov

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json

@@ -2,7 +2,7 @@
  "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c",
  "name": "Hubspot",
  "dockerRepository": "airbyte/source-hubspot",
- "dockerImageTag": "0.1.8",
+ "dockerImageTag": "0.1.9",
  "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot",
  "icon": "hubspot.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -137,7 +137,7 @@
 - sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
  name: Hubspot
  dockerRepository: airbyte/source-hubspot
- dockerImageTag: 0.1.8
+ dockerImageTag: 0.1.9
  documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
  icon: hubspot.svg
 - sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde

airbyte-integrations/connectors/source-hubspot/Dockerfile

@@ -14,5 +14,5 @@ RUN pip install .
 
 ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-hubspot

airbyte-integrations/connectors/source-hubspot/README.md

@@ -53,12 +53,51 @@ python main_dev.py discover --config secrets/config.json
 python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
 ```
 
+## Testing
+Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+First install test dependencies into your virtual environment:
+```
+pip install .[tests]
+```
+
 ### Unit Tests
 To run unit tests locally, from the connector directory run:
 ```
 python -m pytest unit_tests
 ```
 
+### Integration Tests
+There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector).
+
+#### Custom Integration tests
+Place custom tests inside `integration_tests/` folder, then, from the connector root, run
+```
+python -m pytest integration_tests
+```
+
+#### Acceptance Tests
+Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information.
+If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
+
+To run your integration tests with acceptance tests, from the connector root, run
+```
+python -m pytest integration_tests -p integration_tests.acceptance
+```
+
+To run your integration tests with docker
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unit tests:
+```
+./gradlew :airbyte-integrations:connectors:source-hubspot:unitTest
+```
+
+To run acceptance and custom integration tests:
+```
+./gradlew :airbyte-integrations:connectors:source-hubspot:integrationTest
+```
+
 ### Locally running the connector docker image
 
 #### Build
@@ -85,16 +124,16 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files
 
 ### Integration Tests
 1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-hubspot:integrationTest` to run the standard integration test suite.
-1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
- Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+2. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
+Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 
 ## Dependency Management
 All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
 
 ### Publishing a new version of the connector
 You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
 1. Make sure your changes are passing unit and integration tests
-1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer).
-1. Create a Pull Request
-1. Pat yourself on the back for being an awesome contributor
-1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master
+2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer).
+3. Create a Pull Request
+4. Pat yourself on the back for being an awesome contributor
+5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master

airbyte-integrations/connectors/source-hubspot/setup.py

@@ -28,9 +28,9 @@
 MAIN_REQUIREMENTS = [
  "airbyte-protocol",
  "base-python",
- "backoff==1.10.0",
- "pendulum==1.2.0",
- "requests==2.25.1",
+ "backoff==1.11.1",
+ "pendulum==2.1.2",
+ "requests==2.26.0",
 ]
 
 TEST_REQUIREMENTS = [

airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py

@@ -273,12 +273,17 @@ def _cast_value(declared_field_types: List, field_name: str, field_value):
  target_type = CUSTOM_FIELD_VALUE_TYPE_CAST_REVERSED.get(target_type_name)
 
  if target_type_name == "number":
- if field_name.endswith("_id"):
- # do not cast numeric IDs into float, use integer instead
- target_type = int
+ # do not cast numeric IDs into float, use integer instead
+ target_type = int if field_name.endswith("_id") else target_type
+
+ if target_type_name != "string" and field_value == '':
+ # do not cast empty strings, return None instead to be properly casted.
+ field_value = None
+ return field_value
 
  try:
  casted_value = target_type(field_value)
+ print(casted_value)
  except ValueError:
  logger.exception(f"Could not cast `{field_value}` to `{target_type}`")
  return field_value
@@ -451,7 +456,7 @@ def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator:
  self._start_date = self._state
 
  def read_chunked(
- self, getter: Callable, params: Mapping[str, Any] = None, chunk_size: pendulum.Interval = pendulum.interval(days=1)
+ self, getter: Callable, params: Mapping[str, Any] = None, chunk_size: pendulum.duration = pendulum.duration(days=1)
  ) -> Iterator:
  params = {**params} if params else {}
  now_ts = int(pendulum.now().timestamp() * 1000)

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json

@@ -86,7 +86,7 @@
  "type": ["null", "string"]
  },
  "lastUpdatedTime": {
- "type": "integer"
+ "type": ["null", "integer"]
  }
  }
 }

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json

@@ -6,7 +6,7 @@
  "type": ["null", "integer"]
  },
  "companyId": {
- "type": "integer"
+ "type": ["null", "integer"]
  },
  "isDeleted": {
  "type": ["null", "boolean"]
@@ -24,10 +24,10 @@
  "type": ["null", "array"]
  },
  "createdAt": {
- "type": "string"
+ "type": ["null", "string"]
  },
  "updatedAt": {
- "type": "string"
+ "type": ["null", "string"]
  }
  }
 }

airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json

@@ -6,7 +6,7 @@
  "type": ["null", "integer"]
  },
  "metaData": {
- "type": "object",
+ "type": ["null", "object"],
  "properties": {
  "processing": {
  "type": ["null", "string"]
@@ -21,7 +21,7 @@
  "type": ["null", "integer"]
  },
  "lastSizeChangeAt": {
- "type": "integer"
+ "type": ["null", "integer"]
  }
  }
  },
@@ -32,11 +32,11 @@
  "type": ["null", "string"]
  },
  "filters": {
- "type": "array",
+ "type": ["null", "array"],
  "items": {
- "type": "array",
+ "type": ["null", "array"],
  "items": {
- "type": "object",
+ "type": ["null", "object"],
  "properties": {
  "filterFamily": {
  "type": ["null", "string"]
@@ -67,13 +67,13 @@
  "type": ["null", "integer"]
  },
  "createdAt": {
- "type": "integer"
+ "type": ["null", "integer"]
  },
  "listId": {
  "type": ["null", "integer"]
  },
  "updatedAt": {
- "type": "integer"
+ "type": ["null", "integer"]
  },
  "internalListId": {
  "type": ["null", "integer"]