feat: Introduce DataFormatOptions to configure the output of BigQuery data types (#4010)

* feat: Create DataFormatOptions in BigQuery * feat: Add Builder class for DataFormatOptions * fix: Update existing references of useInt64Timestamp to use DataFormatOption's variant * chore: Fix lint issues * chore: Address PR feedback * chore: Add tests for useInt64Timestamp behavior * chore: Address failing tests and GCA * chore: Remove unused fromPb method

Author: lqiu96

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java

@@ -1731,7 +1731,10 @@ public TableDataList call() throws IOException {
  new PageImpl<>(
  new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap),
  cursor,
- transformTableData(result.getRows(), schema, serviceOptions.getUseInt64Timestamps())),
+ transformTableData(
+ result.getRows(),
+ schema,
+ serviceOptions.getDataFormatOptions().useInt64Timestamp())),
  result.getTotalRows());
  } catch (BigQueryRetryHelperException e) {
  throw BigQueryException.translateAndThrow(e);
@@ -2007,7 +2010,9 @@ public com.google.api.services.bigquery.model.QueryResponse call()
  new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)),
  cursor,
  transformTableData(
- results.getRows(), schema, getOptions().getUseInt64Timestamps())))
+ results.getRows(),
+ schema,
+ getOptions().getDataFormatOptions().useInt64Timestamp())))
  .setJobId(jobId)
  .setQueryId(results.getQueryId())
  .build();
@@ -2021,7 +2026,9 @@ public com.google.api.services.bigquery.model.QueryResponse call()
  new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
  null,
  transformTableData(
- results.getRows(), schema, getOptions().getUseInt64Timestamps())))
+ results.getRows(),
+ schema,
+ getOptions().getDataFormatOptions().useInt64Timestamp())))
  // Return the JobID of the successful job
  .setJobId(
  results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)
@@ -2066,10 +2073,9 @@ && getOptions().getOpenTelemetryTracer() != null) {
  }
  try (Scope queryScope = querySpan != null ? querySpan.makeCurrent() : null) {
  // If all parameters passed in configuration are supported by the query() method on the
- // backend,
- // put on fast path
+ // backend, put on fast path
  QueryRequestInfo requestInfo =
- new QueryRequestInfo(configuration, getOptions().getUseInt64Timestamps());
+ new QueryRequestInfo(configuration, getOptions().getDataFormatOptions());
  if (requestInfo.isFastQuerySupported(jobId)) {
  // Be careful when setting the projectID in JobId, if a projectID is specified in the JobId,
  // the job created by the query method will use that project. This may cause the query to

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryOptions.java

@@ -17,6 +17,7 @@
 package com.google.cloud.bigquery;
 
 import com.google.api.core.BetaApi;
+import com.google.api.core.ObsoleteApi;
 import com.google.api.gax.retrying.ResultRetryAlgorithm;
 import com.google.cloud.ServiceDefaults;
 import com.google.cloud.ServiceOptions;
@@ -26,6 +27,7 @@
 import com.google.cloud.bigquery.spi.BigQueryRpcFactory;
 import com.google.cloud.bigquery.spi.v2.HttpBigQueryRpc;
 import com.google.cloud.http.HttpTransportOptions;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import io.opentelemetry.api.trace.Tracer;
 import java.util.Set;
@@ -41,6 +43,7 @@ public class BigQueryOptions extends ServiceOptions<BigQuery, BigQueryOptions> {
  // set the option ThrowNotFound when you want to throw the exception when the value not found
  private boolean setThrowNotFound;
  private boolean useInt64Timestamps;
+ private DataFormatOptions dataFormatOptions;
  private JobCreationMode defaultJobCreationMode = JobCreationMode.JOB_CREATION_MODE_UNSPECIFIED;
  private boolean enableOpenTelemetryTracing;
  private Tracer openTelemetryTracer;
@@ -70,6 +73,7 @@ public static class Builder extends ServiceOptions.Builder<BigQuery, BigQueryOpt
 
  private String location;
  private boolean useInt64Timestamps;
+ private DataFormatOptions dataFormatOptions;
  private boolean enableOpenTelemetryTracing;
  private Tracer openTelemetryTracer;
  private ResultRetryAlgorithm<?> resultRetryAlgorithm;
@@ -94,11 +98,32 @@ public Builder setLocation(String location) {
  return this;
  }
 
+ /**
+ * This setter is marked as Obsolete. Prefer {@link #setDataFormatOptions(DataFormatOptions)} to
+ * set the int64timestamp configuration instead.
+ *
+ * <p>If useInt64Timestamps value is set in here and via DataFormatOptions, the
+ * DataFormatOptions configuration value is used.
+ *
+ * <p>{@code DataFormatOptions.newBuilder().setUseInt64Timestamp(...).build()}
+ */
+ @ObsoleteApi("Use setDataFormatOptions(DataFormatOptions) instead")
  public Builder setUseInt64Timestamps(boolean useInt64Timestamps) {
  this.useInt64Timestamps = useInt64Timestamps;
  return this;
  }
 
+ /**
+ * Set the format options for the BigQuery data types
+ *
+ * @param dataFormatOptions Configuration of the formatting options
+ */
+ public Builder setDataFormatOptions(DataFormatOptions dataFormatOptions) {
+ Preconditions.checkNotNull(dataFormatOptions, "DataFormatOptions cannot be null");
+ this.dataFormatOptions = dataFormatOptions;
+ return this;
+ }
+
  /**
  * Enables OpenTelemetry tracing functionality for this BigQuery instance
  *
@@ -143,6 +168,15 @@ private BigQueryOptions(Builder builder) {
  } else {
  this.resultRetryAlgorithm = BigQueryBaseService.DEFAULT_BIGQUERY_EXCEPTION_HANDLER;
  }
+
+ // If dataFormatOptions is not set, then create a new instance and set it with the
+ // useInt64Timestamps configured in BigQueryOptions
+ if (builder.dataFormatOptions == null) {
+ this.dataFormatOptions =
+ DataFormatOptions.newBuilder().useInt64Timestamp(builder.useInt64Timestamps).build();
+ } else {
+ this.dataFormatOptions = builder.dataFormatOptions;
+ }
  }
 
  private static class BigQueryDefaults implements ServiceDefaults<BigQuery, BigQueryOptions> {
@@ -191,8 +225,23 @@ public void setThrowNotFound(boolean setThrowNotFound) {
  this.setThrowNotFound = setThrowNotFound;
  }
 
+ /**
+ * This setter is marked as Obsolete. Prefer {@link
+ * Builder#setDataFormatOptions(DataFormatOptions)} to set the int64timestamp configuration
+ * instead.
+ *
+ * <p>If useInt64Timestamps is set via DataFormatOptions, then the value in DataFormatOptions will
+ * be used. Otherwise, this value will be passed to DataFormatOptions.
+ *
+ * <p>Alternative: {@code DataFormatOptions.newBuilder().setUseInt64Timestamp(...).build()}
+ */
+ @ObsoleteApi("Use Builder#setDataFormatOptions(DataFormatOptions) instead")
  public void setUseInt64Timestamps(boolean useInt64Timestamps) {
  this.useInt64Timestamps = useInt64Timestamps;
+ // Because this setter exists outside the Builder, DataFormatOptions needs be rebuilt to
+ // account for this setting.
+ this.dataFormatOptions =
+ dataFormatOptions.toBuilder().useInt64Timestamp(useInt64Timestamps).build();
  }
 
  @Deprecated
@@ -206,8 +255,22 @@ public boolean getThrowNotFound() {
  return setThrowNotFound;
  }
 
+ /**
+ * This getter is marked as Obsolete. Prefer {@link
+ * DataFormatOptions.Builder#useInt64Timestamp(boolean)} to set the int64timestamp configuration
+ * instead.
+ *
+ * <p>Warning: DataFormatOptions values have precedence. Use {@link
+ * DataFormatOptions#useInt64Timestamp()} to get `useInt64Timestamp` value used by the BigQuery
+ * client.
+ */
+ @ObsoleteApi("Use getDataFormatOptions().isUseInt64Timestamp() instead")
  public boolean getUseInt64Timestamps() {
- return useInt64Timestamps;
+ return dataFormatOptions.useInt64Timestamp();
+ }
+
+ public DataFormatOptions getDataFormatOptions() {
+ return dataFormatOptions;
  }
 
  public JobCreationMode getDefaultJobCreationMode() {

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DataFormatOptions.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigquery;
+
+import com.google.auto.value.AutoValue;
+import java.io.Serializable;
+
+/**
+ * Google BigQuery DataFormatOptions. Configures the output format for data types returned from
+ * BigQuery.
+ */
+@AutoValue
+public abstract class DataFormatOptions implements Serializable {
+ public enum TimestampFormatOptions {
+ TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED("TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED"),
+ FLOAT64("FLOAT64"),
+ INT64("INT64"),
+ ISO8601_STRING("ISO8601_STRING");
+
+ private final String format;
+
+ TimestampFormatOptions(String format) {
+ this.format = format;
+ }
+
+ @Override
+ public String toString() {
+ return format;
+ }
+ }
+
+ public abstract boolean useInt64Timestamp();
+
+ public abstract TimestampFormatOptions timestampFormatOptions();
+
+ public static Builder newBuilder() {
+ return new AutoValue_DataFormatOptions.Builder()
+ .useInt64Timestamp(false)
+ .timestampFormatOptions(TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED);
+ }
+
+ public abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ public abstract static class Builder {
+ public abstract Builder useInt64Timestamp(boolean useInt64Timestamp);
+
+ public abstract Builder timestampFormatOptions(TimestampFormatOptions timestampFormatOptions);
+
+ public abstract DataFormatOptions build();
+ }
+
+ com.google.api.services.bigquery.model.DataFormatOptions toPb() {
+ com.google.api.services.bigquery.model.DataFormatOptions request =
+ new com.google.api.services.bigquery.model.DataFormatOptions();
+ request.setUseInt64Timestamp(useInt64Timestamp());
+ request.setTimestampOutputFormat(timestampFormatOptions().toString());
+ return request;
+ }
+}

google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryRequestInfo.java

@@ -46,7 +46,8 @@ final class QueryRequestInfo {
  private final DataFormatOptions formatOptions;
  private final String reservation;
 
- QueryRequestInfo(QueryJobConfiguration config, Boolean useInt64Timestamps) {
+ QueryRequestInfo(
+ QueryJobConfiguration config, com.google.cloud.bigquery.DataFormatOptions dataFormatOptions) {
  this.config = config;
  this.connectionProperties = config.getConnectionProperties();
  this.defaultDataset = config.getDefaultDataset();
@@ -61,7 +62,7 @@ final class QueryRequestInfo {
  this.useLegacySql = config.useLegacySql();
  this.useQueryCache = config.useQueryCache();
  this.jobCreationMode = config.getJobCreationMode();
- this.formatOptions = new DataFormatOptions().setUseInt64Timestamp(useInt64Timestamps);
+ this.formatOptions = dataFormatOptions.toPb();
  this.reservation = config.getReservation();
  }
 

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryOptionsTest.java

@@ -16,6 +16,11 @@
 
 package com.google.cloud.bigquery;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 import com.google.cloud.TransportOptions;
 import org.junit.Assert;
 import org.junit.Test;
@@ -35,4 +40,55 @@ public void testInvalidTransport() {
  Assert.assertNotNull(expected.getMessage());
  }
  }
+
+ @Test
+ public void dataFormatOptions_createdByDefault() {
+ BigQueryOptions options = BigQueryOptions.newBuilder().setProjectId("project-id").build();
+
+ assertNotNull(options.getDataFormatOptions());
+ assertFalse(options.getDataFormatOptions().useInt64Timestamp());
+ assertEquals(
+ DataFormatOptions.TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED,
+ options.getDataFormatOptions().timestampFormatOptions());
+ }
+
+ @Test
+ public void nonBuilderSetUseInt64Timestamp_capturedInDataFormatOptions() {
+ BigQueryOptions options =
+ BigQueryOptions.newBuilder()
+ .setDataFormatOptions(DataFormatOptions.newBuilder().useInt64Timestamp(false).build())
+ .setProjectId("project-id")
+ .build();
+ options.setUseInt64Timestamps(true);
+
+ assertTrue(options.getDataFormatOptions().useInt64Timestamp());
+ }
+
+ @Test
+ public void nonBuilderSetUseInt64Timestamp_overridesEverything() {
+ BigQueryOptions options = BigQueryOptions.newBuilder().setProjectId("project-id").build();
+ options.setUseInt64Timestamps(true);
+
+ assertTrue(options.getDataFormatOptions().useInt64Timestamp());
+ }
+
+ @Test
+ public void noDataFormatOptions_capturesUseInt64TimestampSetInBuilder() {
+ BigQueryOptions options =
+ BigQueryOptions.newBuilder().setUseInt64Timestamps(true).setProjectId("project-id").build();
+
+ assertTrue(options.getDataFormatOptions().useInt64Timestamp());
+ }
+
+ @Test
+ public void dataFormatOptionsSetterHasPrecedence() {
+ BigQueryOptions options =
+ BigQueryOptions.newBuilder()
+ .setProjectId("project-id")
+ .setDataFormatOptions(DataFormatOptions.newBuilder().useInt64Timestamp(true).build())
+ .setUseInt64Timestamps(false)
+ .build();
+
+ assertTrue(options.getDataFormatOptions().useInt64Timestamp());
+ }
 }

google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryRequestInfoTest.java

@@ -140,7 +140,8 @@ public class QueryRequestInfoTest {
  .setJobCreationMode(jobCreationModeRequired)
  .setReservation(RESERVATION)
  .build();
- QueryRequestInfo REQUEST_INFO = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false);
+ QueryRequestInfo REQUEST_INFO =
+ new QueryRequestInfo(QUERY_JOB_CONFIGURATION, DataFormatOptions.newBuilder().build());
  private static final QueryJobConfiguration QUERY_JOB_CONFIGURATION_SUPPORTED =
  QueryJobConfiguration.newBuilder(QUERY)
  .setUseQueryCache(USE_QUERY_CACHE)
@@ -156,7 +157,8 @@ public class QueryRequestInfoTest {
  .setReservation(RESERVATION)
  .build();
  QueryRequestInfo REQUEST_INFO_SUPPORTED =
- new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED, false);
+ new QueryRequestInfo(
+ QUERY_JOB_CONFIGURATION_SUPPORTED, DataFormatOptions.newBuilder().build());
 
  @Test
  public void testIsFastQuerySupported() {
@@ -177,17 +179,25 @@ public void testToPb() {
  @Test
  public void equalTo() {
  compareQueryRequestInfo(
- new QueryRequestInfo(QUERY_JOB_CONFIGURATION_SUPPORTED, false), REQUEST_INFO_SUPPORTED);
- compareQueryRequestInfo(new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false), REQUEST_INFO);
+ new QueryRequestInfo(
+ QUERY_JOB_CONFIGURATION_SUPPORTED, DataFormatOptions.newBuilder().build()),
+ REQUEST_INFO_SUPPORTED);
+ compareQueryRequestInfo(
+ new QueryRequestInfo(QUERY_JOB_CONFIGURATION, DataFormatOptions.newBuilder().build()),
+ REQUEST_INFO);
  }
 
  @Test
  public void testInt64Timestamp() {
- QueryRequestInfo requestInfo = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, false);
+ QueryRequestInfo requestInfo =
+ new QueryRequestInfo(QUERY_JOB_CONFIGURATION, DataFormatOptions.newBuilder().build());
  QueryRequest requestPb = requestInfo.toPb();
  assertFalse(requestPb.getFormatOptions().getUseInt64Timestamp());
 
- QueryRequestInfo requestInfoLosslessTs = new QueryRequestInfo(QUERY_JOB_CONFIGURATION, true);
+ QueryRequestInfo requestInfoLosslessTs =
+ new QueryRequestInfo(
+ QUERY_JOB_CONFIGURATION,
+ DataFormatOptions.newBuilder().useInt64Timestamp(true).build());
  QueryRequest requestLosslessTsPb = requestInfoLosslessTs.toPb();
  assertTrue(requestLosslessTsPb.getFormatOptions().getUseInt64Timestamp());
  }