🐛 Source Google Analytics: Fix issue with custom error and simplify configuration

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json

@@ -2,7 +2,7 @@
  "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003",
  "name": "Google Analytics v4",
  "dockerRepository": "airbyte/source-google-analytics-v4",
- "dockerImageTag": "0.1.9",
+ "dockerImageTag": "0.1.12",
  "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-analytics-v4",
  "icon": "google-analytics.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -232,7 +232,7 @@
 - name: Google Analytics
  sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003
  dockerRepository: airbyte/source-google-analytics-v4
- dockerImageTag: 0.1.11
+ dockerImageTag: 0.1.12
  documentationUrl: https://docs.airbyte.io/integrations/sources/source-google-analytics-v4
  icon: google-analytics.svg
  sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2011,7 +2011,7 @@
  oauthFlowOutputParameters:
  - - "access_token"
  - - "refresh_token"
-- dockerImage: "airbyte/source-google-analytics-v4:0.1.11"
+- dockerImage: "airbyte/source-google-analytics-v4:0.1.12"
  spec:
  documentationUrl: "https://docs.airbyte.io/integrations/sources/google-analytics-v4"
  connectionSpecification:

airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.11
+LABEL io.airbyte.version=0.1.12
 LABEL io.airbyte.name=airbyte/source-google-analytics-v4

airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py

@@ -119,6 +119,20 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str,
  if next_page:
  return {"pageToken": next_page}
 
+ def should_retry(self, response: requests.Response) -> bool:
+ """When the connector gets a custom report which has unknown metric(s) or dimension(s)
+ and API returns an error with 400 code, the connector ignores an error with 400 code
+ to finish successfully sync and inform the user about an error in logs with an error message."""
+
+ if response.status_code == 400:
+ self.logger.info(f"{response.json()['error']['message']}")
+ self.raise_on_http_errors = False
+
+ return super().should_retry(response)
+
+ def raise_on_http_errors(self) -> bool:
+ return True
+
  def request_body_json(
  self, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
  ) -> Optional[Mapping]:
@@ -522,6 +536,9 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
  def streams(self, config: Mapping[str, Any]) -> List[Stream]:
  streams: List[GoogleAnalyticsV4Stream] = []
 
+ if "window_in_days" not in config:
+ config["window_in_days"] = 90
+
  authenticator = self.get_authenticator(config)
 
  config["authenticator"] = authenticator

airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json

@@ -8,30 +8,28 @@
  "additionalProperties": true,
  "properties": {
  "view_id": {
+ "order": 2,
  "type": "string",
  "title": "View ID",
- "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the <a href=\"https://ga-dev-tools.appspot.com/account-explorer/\">Google Analytics Account Explorer</a>.",
- "airbyte_secret": true
+ "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the <a href=\"https://ga-dev-tools.appspot.com/account-explorer/\">Google Analytics Account Explorer</a>."
  },
  "start_date": {
+ "order": 1,
  "type": "string",
  "title": "Start Date",
  "description": "A date in the format YYYY-MM-DD.",
  "examples": ["2020-06-01"]
  },
- "window_in_days": {
- "type": "integer",
- "description": "The amount of days for each data-chunk begining from start_date. Bigger the value - faster the fetch. (Min=1, as for a Day; Max=364, as for a Year).",
- "examples": [30, 60, 90, 120, 200, 364],
- "default": 90
- },
  "custom_reports": {
- "title": "Custom Reports",
+ "order": 3,
  "type": "string",
+ "title": "Custom Reports (optional)",
  "description": "A JSON array describing the custom reports you want to sync from GA. Check out the <a href=\"https://docs.airbyte.io/integrations/sources/google-analytics-v4\">docs</a> to get more information about this field."
  },
  "credentials": {
+ "order": 0,
  "type": "object",
+ "title": "Credentials",
  "oneOf": [
  {
  "title": "Authenticate via Google (Oauth)",

airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py

@@ -40,6 +40,20 @@ def mock_auth_call(requests_mock):
  yield requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": "", "expires_in": 0})
 
 
+@pytest.fixture
+def mock_auth_check_connection(requests_mock):
+ yield requests_mock.post("https://analyticsreporting.googleapis.com/v4/reports:batchGet", json={"data": {"test": "value"}})
+
+
+@pytest.fixture
+def mock_unknown_metrics_or_dimensions_error(requests_mock):
+ yield requests_mock.post(
+ "https://analyticsreporting.googleapis.com/v4/reports:batchGet",
+ status_code=400,
+ json={"error": {"message": "Unknown metrics or dimensions"}},
+ )
+
+
 @pytest.fixture
 def mock_api_returns_no_records(requests_mock):
  """API returns empty data for given date based slice"""
@@ -200,6 +214,11 @@ def test_check_connection_success_oauth(
  assert mock_api_returns_valid_records.called
 
 
+def test_unknown_metrics_or_dimensions_error_validation(mock_metrics_dimensions_type_list_link, mock_unknown_metrics_or_dimensions_error):
+ records = GoogleAnalyticsV4Stream(MagicMock()).read_records(sync_mode=None)
+ assert records
+
+
 @freeze_time("2021-11-30")
 def test_stream_slices_limited_by_current_date(test_config):
  g = GoogleAnalyticsV4IncrementalObjectsBase(config=test_config)

docs/integrations/sources/google-analytics-v4.md

@@ -132,6 +132,7 @@ The Google Analytics connector should not run into Google Analytics API limitati
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.12 | 2021-12-03 | [8175](https://github.com/airbytehq/airbyte/pull/8175) | Fix validation of unknown metric(s) or dimension(s) error |
 | 0.1.11 | 2021-11-30 | [8264](https://github.com/airbytehq/airbyte/pull/8264) | Corrected date range |
 | 0.1.10 | 2021-11-19 | [8087](https://github.com/airbytehq/airbyte/pull/8087) | Support `start_date` before the account has any data |
 | 0.1.9 | 2021-10-27 | [7410](https://github.com/airbytehq/airbyte/pull/7410) | Add check for correct permission for requested `view_id` |