🐛 Source Google Analytics (UA): fix discovery stage, when custom_reports is provided as single dict (#18965)

Author: Baz

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -471,7 +471,7 @@
 - name: Google Analytics (Universal Analytics)
  sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003
  dockerRepository: airbyte/source-google-analytics-v4
- dockerImageTag: 0.1.31
+ dockerImageTag: 0.1.32
  documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics
  icon: google-analytics.svg
  sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4369,7 +4369,7 @@
  oauthFlowOutputParameters:
  - - "access_token"
  - - "refresh_token"
-- dockerImage: "airbyte/source-google-analytics-v4:0.1.31"
+- dockerImage: "airbyte/source-google-analytics-v4:0.1.32"
  spec:
  documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics"
  connectionSpecification:

airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile

@@ -12,5 +12,5 @@ COPY main.py ./
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.31
+LABEL io.airbyte.version=0.1.32
 LABEL io.airbyte.name=airbyte/source-google-analytics-v4

airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py

@@ -6,7 +6,7 @@
 from dataclasses import dataclass
 from typing import Dict, List, Optional, Union
 
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, Field, ValidationError, validator
 
 
 class Model(BaseModel):
@@ -18,6 +18,16 @@ class Config:
  metrics: list[str]
  filter: Optional[str]
 
+ @validator("dimensions", "metrics")
+ def check_field_reference_forrmat(cls, value):
+ """
+ Defines rules for nested strings, for fields: dimensions, metrics.
+ General rule: the `ga:` prefix is defined for each field
+ """
+ for v in value:
+ if "ga:" not in v:
+ raise ValueError(v)
+
 
 class Explainer:
  """
@@ -33,16 +43,21 @@ class Explainer:
  "value_error.missing": ("fields required", []),
  "value_error.extra": ("fields not permitted", []),
  "type_error": ("type errors", []),
+ "value_error": ("incorrect field reference, expected format `ga:MY_FIELD_NAME`, but got", []),
  }
 
  def parse(self, errors: List[Dict]) -> str:
  for error in errors:
- field_name, error_type = error.get("loc")[0], error.get("type")
+ field_name, error_type, error_msg = error.get("loc")[0], error.get("type"), error.get("msg")
 
  # general errors
  if error_type in self.errors_mapping:
- self.errors_mapping.get(error_type)[1].append(field_name)
-
+ # value errors
+ if error_type == "value_error":
+ self.errors_mapping.get(error_type)[1].append({"field": field_name, "reference": error_msg})
+ # general errors
+ else:
+ self.errors_mapping.get(error_type)[1].append(field_name)
  # type errors
  if "type_error" in error_type:
  error_type, _type = error_type.split(".")

airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py

@@ -588,10 +588,8 @@ def check_connection(self, logger: logging.Logger, config: MutableMapping) -> Tu
  False,
  f"Please check the permissions for the requested view_id: {config['view_id']}. Cannot retrieve data from that view ID.",
  )
-
  except ValueError as e:
  return False, f"Invalid custom reports json structure. {e}"
-
  except requests.exceptions.RequestException as e:
  error_msg = e.response.json().get("error")
  if e.response.status_code == 403:
@@ -608,8 +606,10 @@ def streams(self, config: MutableMapping[str, Any]) -> List[Stream]:
 
  reports = json.loads(pkgutil.get_data("source_google_analytics_v4", "defaults/default_reports.json"))
 
- if config.get("custom_reports"):
- custom_reports = json.loads(config["custom_reports"])
+ custom_reports = config.get("custom_reports")
+ if custom_reports:
+ custom_reports = json.loads(custom_reports)
+ custom_reports = [custom_reports] if not isinstance(custom_reports, list) else custom_reports
  reports += custom_reports
 
  config["ga_streams"] = reports

airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py

@@ -10,11 +10,12 @@
 @pytest.mark.parametrize(
  "custom_reports, expected",
  (
- ([{"name": [], "dimensions": ["test"], "metrics": ["test"]}], "errors: type errors"),
- ([{"name": "test", "dimensions": ["test"], "metrics": ["test"], "added_field": "test"}], "errors: fields not permitted"),
- ([{"missing_name": "test", "dimensions": ["test"], "metrics": ["test"]}], "errors: fields required"),
+ ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"),
+ ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"),
+ ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"),
+ ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"),
  ),
- ids=["type_error", "not_permitted", "missing"],
+ ids=["incorrrect field reference", "type_error", "not_permitted", "missing"],
 )
 def test_custom_reports_validator(custom_reports, expected):
  try:

docs/integrations/sources/google-analytics-universal-analytics.md

@@ -155,6 +155,7 @@ Incremental sync is supported only if you add `ga:date` dimension to your custom
 
 | Version | Date | Pull Request | Subject |
 |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------|
+| 0.1.32 | 2022-11-04 | [18965](https://github.com/airbytehq/airbyte/pull/18965) | Fix for `discovery` stage, when `custom_reports` are provided with single stream as `dict` | 
 | 0.1.31 | 2022-10-30 | [18670](https://github.com/airbytehq/airbyte/pull/18670) | Add `Custom Reports` schema validation on `check connection` |
 | 0.1.30 | 2022-10-13 | [17943](https://github.com/airbytehq/airbyte/pull/17943) | Fix pagination |
 | 0.1.29 | 2022-10-12 | [17905](https://github.com/airbytehq/airbyte/pull/17905) | Handle exceeded daily quota gracefully |