🎉 Source Shopify - change the default stream cursor_field to 'updated_at' where possible (#4472)

#4472 - Source Shopify - change the default stream cursor_field to 'updated_at' where possible Co-authored-by: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>

Author: bazarnov

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json

@@ -2,6 +2,6 @@
  "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9",
  "name": "Shopify",
  "dockerRepository": "airbyte/source-shopify",
- "dockerImageTag": "0.1.8",
+ "dockerImageTag": "0.1.9",
  "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -133,7 +133,7 @@
 - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9
  name: Shopify
  dockerRepository: airbyte/source-shopify
- dockerImageTag: 0.1.8
+ dockerImageTag: 0.1.9
  documentationUrl: https://docs.airbyte.io/integrations/sources/shopify
 - sourceDefinitionId: 9845d17a-45f1-4070-8a60-50914b1c8e2b
  name: HTTP Request

airbyte-integrations/connectors/source-shopify/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-shopify

airbyte-integrations/connectors/source-shopify/README.md

@@ -70,6 +70,7 @@ You can also build the connector image via Gradle:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:airbyteDocker
 ```
+
 When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in
 the Dockerfile.
 
@@ -81,12 +82,14 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev check --co
 docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev discover --config /secrets/config.json
 docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shopify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json
 ```
+
 ## Testing
 Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 First install test dependencies into your virtual environment:
 ```
 pip install .[tests]
 ```
+
 ### Unit Tests
 To run unit tests locally, from the connector directory run:
 ```
@@ -100,6 +103,7 @@ Place custom tests inside `integration_tests/` folder, then, from the connector
 ```
 python3 -m pytest integration_tests
 ```
+
 #### Acceptance Tests
 Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information.
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
@@ -115,10 +119,12 @@ To run unit tests:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:unitTest
 ```
+
 To run acceptance and custom integration tests:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:integrationTest
 ```
+
 To build final build the connector:
 ```
 ./gradlew :airbyte-integrations:connectors:source-shopify:build

airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json

@@ -1,44 +1,44 @@
 {
+ "customers": {
+ "updated_at": "2020-06-08T21:09:26-07:00"
+ },
  "orders": {
- "id": 2497723957406484
+ "updated_at": "2020-08-03T14:42:06-07:00"
  },
- "collects": {
- "id": 1727945110336789
+ "draft_orders": {
+ "updated_at": "2020-06-08T21:09:25-07:00"
  },
  "products": {
- "id": 505931153421589
+ "updated_at": "2021-06-25T06:13:37-07:00"
  },
- "customers": {
- "id": 366477377551899
+ "abandoned_checkouts": {
+ "updated_at": "2021-06-29T02:14:14-07:00"
  },
  "metafields": {
- "id": 1126916852947498
- },
- "transactions": {
- "id": 312557948124698
+ "updated_at": "2019-11-03T21:15:00-08:00"
  },
- "order_refunds": {
- "id": 63997837325598
+ "collects": {
+ "id": 29523654213791
  },
  "custom_collections": {
- "id": 15327628499498
+ "updated_at": "2021-06-29T09:41:33-07:00"
  },
- "abandoned_checkouts": {
- "id": 142548663666229
+ "order_refunds": {
+ "created_at": "2020-02-18T19:49:23-08:00"
  },
  "order_risks": {
- "id": 593300439055898
+ "id": 5933004390559
+ },
+ "transactions": {
+ "created_at": "2020-06-08T21:09:20-07:00"
  },
  "pages": {
- "id": 131092082366699
+ "updated_at": "2020-06-01T17:08:25-07:00"
  },
  "price_rules": {
- "id": 935355809987999
+ "updated_at": "2021-06-29T06:35:08-07:00"
  },
  "discount_codes": {
- "id": 980808098689434
- },
- "draft_orders": {
- "id": 94068132465507
+ "updated_at": "2021-06-29T06:35:08-07:00"
  }
 }

airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json

@@ -201,10 +201,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -1638,10 +1638,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -2335,10 +2335,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -2592,10 +2592,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -3373,10 +3373,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -3425,6 +3425,45 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
+ "default_cursor_field": ["updated_at"]
+ },
+ "sync_mode": "incremental",
+ "cursor_field": ["updated_at"],
+ "destination_sync_mode": "append"
+ },
+ {
+ "stream": {
+ "name": "collects",
+ "json_schema": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": ["null", "integer"]
+ },
+ "collection_id": {
+ "type": ["null", "integer"]
+ },
+ "created_at": {
+ "type": ["null", "string"],
+ "format": "date-time"
+ },
+ "position": {
+ "type": ["null", "integer"]
+ },
+ "product_id": {
+ "type": ["null", "integer"]
+ },
+ "sort_value": {
+ "type": ["null", "string"]
+ },
+ "updated_at": {
+ "type": ["null", "string"],
+ "format": "date-time"
+ }
+ }
+ },
+ "supported_sync_modes": ["incremental", "full_refresh"],
+ "source_defined_cursor": true,
  "default_cursor_field": ["id"]
  },
  "sync_mode": "incremental",
@@ -3491,10 +3530,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -3897,10 +3936,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["created_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["created_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -4054,10 +4093,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["created_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["created_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -4103,10 +4142,10 @@
  },
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -4265,10 +4304,10 @@
  },
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -4301,10 +4340,10 @@
  },
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  }
  ]

airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json

@@ -2490,10 +2490,10 @@
  },
  "supported_sync_modes": ["incremental", "full_refresh"],
  "source_defined_cursor": true,
- "default_cursor_field": ["id"]
+ "default_cursor_field": ["updated_at"]
  },
  "sync_mode": "incremental",
- "cursor_field": ["id"],
+ "cursor_field": ["updated_at"],
  "destination_sync_mode": "append"
  },
  {
@@ -3586,6 +3586,45 @@
  "cursor_field": ["id"],
  "destination_sync_mode": "append"
  },
+ {
+ "stream": {
+ "name": "collects",
+ "json_schema": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": ["null", "integer"]
+ },
+ "collection_id": {
+ "type": ["null", "integer"]
+ },
+ "created_at": {
+ "type": ["null", "string"],
+ "format": "date-time"
+ },
+ "position": {
+ "type": ["null", "integer"]
+ },
+ "product_id": {
+ "type": ["null", "integer"]
+ },
+ "sort_value": {
+ "type": ["null", "string"]
+ },
+ "updated_at": {
+ "type": ["null", "string"],
+ "format": "date-time"
+ }
+ }
+ },
+ "supported_sync_modes": ["incremental", "full_refresh"],
+ "source_defined_cursor": true,
+ "default_cursor_field": ["id"]
+ },
+ "sync_mode": "incremental",
+ "cursor_field": ["id"],
+ "destination_sync_mode": "append"
+ },
  {
  "stream": {
  "name": "custom_collections",