🎉 Make credentials optional in BigQuery connector (issue #3657) (#3947)

* Make credentials optional for BigQuery connector * Bump destination-bigquery version 0.3.5 -> 0.3.6 Co-authored-by: Sabolc Franjo <sabolc.franjo@ev-box.com>

Author: sabifranjo

airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/22f6c74f-5699-40ff-833c-4a879ea40133.json

@@ -2,6 +2,6 @@
  "destinationDefinitionId": "22f6c74f-5699-40ff-833c-4a879ea40133",
  "name": "BigQuery",
  "dockerRepository": "airbyte/destination-bigquery",
- "dockerImageTag": "0.3.5",
+ "dockerImageTag": "0.3.6",
  "documentationUrl": "https://docs.airbyte.io/integrations/destinations/bigquery"
 }

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -17,7 +17,7 @@
 - destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
  name: BigQuery
  dockerRepository: airbyte/destination-bigquery
- dockerImageTag: 0.3.5
+ dockerImageTag: 0.3.6
  documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
 - destinationDefinitionId: 424892c4-daac-4491-b35d-c6688ba547ba
  name: Snowflake

airbyte-integrations/connectors/destination-bigquery/CHANGELOG.md

@@ -2,3 +2,6 @@
 
 ## 0.3.4
 Added option to choose dataset location
+
+## 0.3.6
+Service account credentials are now optional. Default credentials will be used if the field is empty.

airbyte-integrations/connectors/destination-bigquery/Dockerfile

@@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar
 
 RUN tar xf ${APPLICATION}.tar --strip-components=1
 
-LABEL io.airbyte.version=0.3.5
+LABEL io.airbyte.version=0.3.6
 LABEL io.airbyte.name=airbyte/destination-bigquery

airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java

@@ -69,6 +69,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.util.Objects.isNull;
+
 public class BigQueryDestination extends BaseConnector implements Destination {
 
  private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestination.class);
@@ -130,24 +132,32 @@ private void createSchemaTable(BigQuery bigquery, String datasetId, String datas
 
  private BigQuery getBigQuery(JsonNode config) {
  final String projectId = config.get(CONFIG_PROJECT_ID).asText();
- // handle the credentials json being passed as a json object or a json object already serialized as
- // a string.
- final String credentialsString =
- config.get(CONFIG_CREDS).isObject() ? Jsons.serialize(config.get(CONFIG_CREDS)) : config.get(CONFIG_CREDS).asText();
- try {
- final ServiceAccountCredentials credentials = ServiceAccountCredentials
- .fromStream(new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8)));
 
- return BigQueryOptions.newBuilder()
+ try {
+ BigQueryOptions.Builder bigQueryBuilder = BigQueryOptions.newBuilder();
+ ServiceAccountCredentials credentials = null;
+ if (isUsingJsonCredentials(config)) {
+ // handle the credentials json being passed as a json object or a json object already serialized as
+ // a string.
+ final String credentialsString =
+ config.get(CONFIG_CREDS).isObject() ? Jsons.serialize(config.get(CONFIG_CREDS)) : config.get(CONFIG_CREDS).asText();
+ credentials = ServiceAccountCredentials
+ .fromStream(new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8)));
+ }
+ return bigQueryBuilder
  .setProjectId(projectId)
- .setCredentials(credentials)
+ .setCredentials(!isNull(credentials) ? credentials : ServiceAccountCredentials.getApplicationDefault())
  .build()
  .getService();
  } catch (IOException e) {
  throw new RuntimeException(e);
  }
  }
 
+ public static boolean isUsingJsonCredentials(JsonNode config) {
+ return config.has(CONFIG_CREDS) && !config.get(CONFIG_CREDS).asText().isEmpty();
+ }
+
  /**
  * Strategy:
  * <p>

airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json

@@ -8,7 +8,7 @@
  "$schema": "http://json-schema.org/draft-07/schema#",
  "title": "BigQuery Destination Spec",
  "type": "object",
- "required": ["project_id", "dataset_id", "credentials_json"],
+ "required": ["project_id", "dataset_id"],
  "additionalProperties": false,
  "properties": {
  "project_id": {
@@ -58,7 +58,7 @@
  },
  "credentials_json": {
  "type": "string",
- "description": "The contents of the JSON service account key. Check out the <a href=\"https://docs.airbyte.io/integrations/destinations/bigquery\">docs</a> if you need help generating this key.",
+ "description": "The contents of the JSON service account key. Check out the <a href=\"https://docs.airbyte.io/integrations/destinations/bigquery\">docs</a> if you need help generating this key. Default credentials will be used if this field is left empty.",
  "title": "Credentials JSON",
  "airbyte_secret": true
  }