🐛 Source Tiktok: unnest cursor and primary key to the root-level (#16137)

Author: Baz

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -1012,7 +1012,7 @@
 - name: TikTok Marketing
  sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
  dockerRepository: airbyte/source-tiktok-marketing
- dockerImageTag: 0.1.14
+ dockerImageTag: 0.1.15
  documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing
  icon: tiktok.svg
  sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -10050,7 +10050,7 @@
  supportsNormalization: false
  supportsDBT: false
  supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-tiktok-marketing:0.1.14"
+- dockerImage: "airbyte/source-tiktok-marketing:0.1.15"
  spec:
  documentationUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing"
  changelogUrl: "https://docs.airbyte.io/integrations/sources/tiktok-marketing"

airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile

@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.14
+LABEL io.airbyte.version=0.1.15
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/README.md

@@ -99,7 +99,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
 To run your integration tests with acceptance tests, from the connector root, run
 ```
-python -m pytest integration_tests -p integration_tests.acceptance
+docker build . --no-cache -t airbyte/source-tiktok-marketing:dev \
+&& python -m pytest -p source_acceptance_test.plugin
 ```
 To run your integration tests with docker
 

airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml

@@ -13,11 +13,6 @@ tests:
  status: "succeed"
  - config_path: "secrets/prod_config_with_lifetime_granularity.json"
  status: "succeed"
-# Sandbox creds are not working temporary because of issues from APi side
-# - config_path: "secrets/config.json"
-# status: "succeed"
-# - config_path: "secrets/new_config_sandbox.json"
-# status: "succeed"
  - config_path: "secrets/new_config_prod.json"
  status: "succeed"
  - config_path: "secrets/config_oauth.json"
@@ -31,11 +26,23 @@ tests:
 
  discovery:
  - config_path: "secrets/prod_config.json"
+ backward_compatibility_tests_config:
+ disable_for_version: "0.1.14"
  - config_path: "secrets/prod_config_with_day_granularity.json"
+ backward_compatibility_tests_config:
+ disable_for_version: "0.1.14"
  - config_path: "secrets/prod_config_with_lifetime_granularity.json"
+ backward_compatibility_tests_config:
+ disable_for_version: "0.1.14"
  - config_path: "secrets/config.json"
+ backward_compatibility_tests_config:
+ disable_for_version: "0.1.14"
  - config_path: "secrets/new_config_prod.json"
+ backward_compatibility_tests_config:
+ disable_for_version: "0.1.14"
  - config_path: "secrets/config_oauth.json"
+ backward_compatibility_tests_config:
+ disable_for_version: "0.1.14"
 
  basic_read:
  # New style streams (for >= 0.1.13):

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json

@@ -10,42 +10,50 @@
  },
 
  "ads_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "advertisers_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "ad_groups_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "campaigns_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
 
  "ads_audience_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "advertisers_audience_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "ad_group_audience_reports_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "campaigns_audience_reports_by_country_daily": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state_with_granullarity.json

@@ -10,42 +10,49 @@
  },
 
  "ads_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "advertisers_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "ad_groups_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "campaigns_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
-
  "ads_audience_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "advertisers_audience_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "ad_group_audience_reports": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }
  },
  "campaigns_audience_reports_by_country": {
+ "stat_time_day": "2030-01-01",
  "dimensions": {
  "stat_time_day": "2030-01-01"
  }

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json

@@ -16,7 +16,9 @@
  "stream": {
  "name": "ads_reports",
  "json_schema": {},
- "supported_sync_modes": ["full_refresh"]
+ "supported_sync_modes": ["full_refresh"],
+ "source_defined_cursor": true,
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -27,7 +29,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams.json

@@ -37,14 +37,14 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "ads_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "ads_reports_lifetime",
@@ -56,14 +56,14 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "ad_groups_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "ad_groups_reports_lifetime",
@@ -75,14 +75,14 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "campaigns_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "campaigns_reports_lifetime",
@@ -94,14 +94,14 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "advertisers_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "advertisers_reports_lifetime",
@@ -113,14 +113,14 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "advertisers_audience_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "advertisers_audience_reports_lifetime",
@@ -132,42 +132,42 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "ads_audience_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "ad_group_audience_reports_hourly",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "ad_group_audience_reports_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  {
  "name": "campaigns_audience_reports_by_country_hourly",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_hour"]
+ "default_cursor_field": ["stat_time_hour"]
  },
  {
  "name": "campaigns_audience_reports_by_country_daily",
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  }
  ]
  }

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/streams_all.json

@@ -54,7 +54,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -65,7 +65,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -76,7 +76,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -87,7 +87,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -99,7 +99,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "incremental",
  "destination_sync_mode": "append"
@@ -110,7 +110,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -121,7 +121,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"
@@ -132,7 +132,7 @@
  "json_schema": {},
  "supported_sync_modes": ["full_refresh", "incremental"],
  "source_defined_cursor": true,
- "default_cursor_field": ["dimensions", "stat_time_day"]
+ "default_cursor_field": ["stat_time_day"]
  },
  "sync_mode": "full_refresh",
  "destination_sync_mode": "append"