🐛 Source Google-Analytics-v4: add additional permission check while setting up the connector

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json

@@ -2,7 +2,7 @@
  "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003",
  "name": "Google Analytics v4",
  "dockerRepository": "airbyte/source-google-analytics-v4",
- "dockerImageTag": "0.1.8",
+ "dockerImageTag": "0.1.9",
  "documentationUrl": "https://docs.airbyte.io/integrations/sources/source-google-analytics-v4",
  "icon": "google-analytics.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -178,7 +178,7 @@
 - name: Google Analytics v4
  sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003
  dockerRepository: airbyte/source-google-analytics-v4
- dockerImageTag: 0.1.8
+ dockerImageTag: 0.1.9
  documentationUrl: https://docs.airbyte.io/integrations/sources/source-google-analytics-v4
  icon: google-analytics.svg
  sourceType: api

airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-google-analytics-v4

airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml

@@ -5,8 +5,6 @@ tests:
  spec:
  - spec_path: "source_google_analytics_v4/spec.json"
  connection:
- - config_path: "secrets/config.json"
- status: "succeed"
  - config_path: "secrets/service_config.json"
  status: "succeed"
  - config_path: "secrets/old_config.json"

airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_config.json

@@ -1,6 +1,6 @@
 {
- "view_id": "id",
- "start_date": "start_date",
+ "view_id": "1234567",
+ "start_date": "2021-01-01",
  "window_in_days": 1,
  "custom_reports": "custom_reports"
 }

airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py

@@ -140,7 +140,6 @@ def request_body_json(
 
  if next_page_token:
  request_body["reportRequests"][0].update(next_page_token)
-
  return request_body
 
  def get_json_schema(self) -> Mapping[str, Any]:
@@ -202,13 +201,11 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite
  # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future
  start_date = min(start_date, end_date)
  date_slices = []
-
  while start_date <= end_date:
  end_date_slice = start_date.add(days=self.window_in_days)
  date_slices.append({"startDate": self.to_datetime_str(start_date), "endDate": self.to_datetime_str(end_date_slice)})
  # add 1 day for start next slice from next day and not duplicate data from previous slice end date.
  start_date = end_date_slice.add(days=1)
-
  return date_slices
 
  def get_data(self, data):
@@ -217,7 +214,6 @@ def get_data(self, data):
  data = data.get(data_field, [])
  else:
  return []
-
  return data
 
  def lookup_data_type(self, field_type, attribute):
@@ -445,6 +441,13 @@ def get_refresh_request_params(self) -> Mapping[str, any]:
  return {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": str(signed_jwt)}
 
 
+class TestStreamConnection(GoogleAnalyticsV4Stream):
+ """
+ Test the connectivity and permissions to read the data from the stream.
+ Because of the nature of the connector, the streams are created dynamicaly.
+ We declare the static stream like this to be able to test out the prmissions to read the particular view_id."""
+
+
 class SourceGoogleAnalyticsV4(AbstractSource):
  """Google Analytics lets you analyze data about customer engagement with your website or application."""
 
@@ -468,22 +471,36 @@ def get_authenticator(config):
  )
 
  def check_connection(self, logger, config) -> Tuple[bool, any]:
- try:
- url = f"{GoogleAnalyticsV4TypesList.url_base}"
 
- authenticator = self.get_authenticator(config)
+ # declare additional variables
+ authenticator = self.get_authenticator(config)
+ config["authenticator"] = authenticator
+ config["metrics"] = ["ga:14dayUsers"]
+ config["dimensions"] = ["ga:date"]
 
-  session = requests.get(url, headers=authenticator.get_auth_header())
-  session.raise_for_status()
+ # Produce only one date-slice to check the reading permissions
+ first_stream_slice = TestStreamConnection(config).stream_slices(stream_state=None)[0]
 
+ try:
+ # test the eligibility of custom_reports input
  custom_reports = config.get("custom_reports")
  if custom_reports:
  json.loads(custom_reports)
- return True, None
- except (requests.exceptions.RequestException, ValueError) as e:
- if e == ValueError:
- logger.error("Invalid custom reports json structure.")
- return False, e
+
+ # test the reading operation
+ read_check = list(TestStreamConnection(config).read_records(sync_mode=None, stream_slice=first_stream_slice))
+ if read_check:
+ return True, None
+
+ except ValueError as e:
+ return False, f"Invalid custom reports json structure. {e}"
+
+ except requests.exceptions.RequestException as e:
+ error_msg = e.response.json().get("error")
+ if e.response.status_code == 403:
+ return False, f"Please check the permissions for the requested view_id: {config['view_id']}. {error_msg}"
+ else:
+ return False, f"{error_msg}"
 
  def streams(self, config: Mapping[str, Any]) -> List[Stream]:
  streams: List[GoogleAnalyticsV4Stream] = []

airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py

@@ -33,6 +33,11 @@ def mock_auth_call(requests_mock):
  yield requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": "", "expires_in": 0})
 
 
+@pytest.fixture
+def mock_auth_check_connection(requests_mock):
+ yield requests_mock.post("https://analyticsreporting.googleapis.com/v4/reports:batchGet", json={"data": {"test": "value"}})
+
+
 def test_metrics_dimensions_type_list(mock_metrics_dimensions_type_list_link):
  test_metrics, test_dimensions = GoogleAnalyticsV4TypesList().read_records(sync_mode=None)
 
@@ -66,21 +71,23 @@ def test_lookup_metrics_dimensions_data_type(metrics_dimensions_mapping, mock_me
 
 
 @patch("source_google_analytics_v4.source.jwt")
-def test_check_connection_jwt(jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call):
+def test_check_connection_jwt(jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call, mock_auth_check_connection):
  test_config = json.loads(read_file("../integration_tests/sample_config.json"))
  del test_config["custom_reports"]
  test_config["credentials"] = {
  "auth_type": "Service",
  "credentials_json": '{"client_email": "", "private_key": "", "private_key_id": ""}',
  }
  source = SourceGoogleAnalyticsV4()
- assert source.check_connection(MagicMock(), test_config) == (True, None)
+ assert source.check_connection(MagicMock(), test_config) is None
  jwt_encode_mock.encode.assert_called()
  assert mock_auth_call.called
 
 
 @patch("source_google_analytics_v4.source.jwt")
-def test_check_connection_oauth(jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call):
+def test_check_connection_oauth(
+ jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call, mock_auth_check_connection
+):
  test_config = json.loads(read_file("../integration_tests/sample_config.json"))
  del test_config["custom_reports"]
  test_config["credentials"] = {
@@ -90,7 +97,7 @@ def test_check_connection_oauth(jwt_encode_mock, mocker, mock_metrics_dimensions
  "refresh_token": "refresh_token_val",
  }
  source = SourceGoogleAnalyticsV4()
- assert source.check_connection(MagicMock(), test_config) == (True, None)
+ assert source.check_connection(MagicMock(), test_config) is None
  jwt_encode_mock.encode.assert_not_called()
  assert "https://www.googleapis.com/auth/analytics.readonly" in unquote(mock_auth_call.last_request.body)
  assert "client_id_val" in unquote(mock_auth_call.last_request.body)

docs/integrations/sources/google-analytics-v4.md

@@ -132,6 +132,7 @@ The Google Analytics connector should not run into Google Analytics API limitati
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.9 | 2021-10-27 | [7410](https://github.com/airbytehq/airbyte/pull/7410) | Add check for correct permission for requested `view_id` |
 | 0.1.8 | 2021-10-13 | [7020](https://github.com/airbytehq/airbyte/pull/7020) | Add intermediary auth config support |
 | 0.1.7 | 2021-10-07 | [6414](https://github.com/airbytehq/airbyte/pull/6414) | Declare oauth parameters in google sources |
 | 0.1.6 | 2021-09-27 | [6459](https://github.com/airbytehq/airbyte/pull/6459) | Update OAuth Spec File |