Skip to content

Commit 179193a

Browse files
egreco12Evan Grecogcf-owl-bot[bot]
authored
feat: Add integration tests with RetrySettings enabled. (#2275)
* Add integration tests with RetrySettings enabled. Initially, these tests are not run automatically as the running Service Account requires permissions on special GCP projects that inject instream errors into streams to test retries. There will be a following nightly build that will run these tests. --------- Co-authored-by: Evan Greco <egreco@google.com> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 664b550 commit 179193a

File tree

7 files changed

+398
-7
lines changed

7 files changed

+398
-7
lines changed

‎.kokoro/build.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ integration)
6565
-DtrimStackTrace=false \
6666
-Dclirr.skip=true \
6767
-Denforcer.skip=true \
68+
-Dit.test=!ITBigQueryWrite*RetryTest \
69+
-Dsurefire.failIfNoSpecifiedTests=false \
70+
-Dfailsafe.failIfNoSpecifiedTests=false \
6871
-fae \
6972
verify
7073
RETURN_CODE=$?

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies:
5050
If you are using Gradle 5.x or later, add this to your dependencies:
5151

5252
```Groovy
53-
implementation platform('com.google.cloud:libraries-bom:26.26.0')
53+
implementation platform('com.google.cloud:libraries-bom:26.27.0')
5454
5555
implementation 'com.google.cloud:google-cloud-bigquerystorage'
5656
```

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -985,14 +985,14 @@ private Boolean retryOnRetryableError(Code errorCode, AppendRequestAndResponse r
985985
Long offset =
986986
requestWrapper.message.hasOffset() ? requestWrapper.message.getOffset().getValue() : -1;
987987
if (isDefaultStreamName(streamName) || offset == -1) {
988-
log.fine(
988+
log.info(
989989
String.format(
990990
"Retrying default stream message in stream %s for in-stream error: %s, retry count:"
991991
+ " %s",
992992
streamName, errorCode, requestWrapper.retryCount));
993993
addMessageToFrontOfWaitingQueue(requestWrapper);
994994
} else {
995-
log.fine(
995+
log.info(
996996
String.format(
997997
"Retrying exclusive message in stream %s at offset %d for in-stream error: %s, retry"
998998
+ " count: %s",
@@ -1089,6 +1089,7 @@ private void requestCallback(AppendRowsResponse response) {
10891089
// Retries need to happen on the same thread as queue locking may occur
10901090
if (response.hasError()) {
10911091
if (retryOnRetryableError(Code.values()[response.getError().getCode()], requestWrapper)) {
1092+
log.info("Attempting to retry on error: " + response.getError().toString());
10921093
return;
10931094
}
10941095
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/ConnectionWorkerPoolTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -389,15 +389,15 @@ public void testCloseExternalClient()
389389
// Create some stream writers.
390390
List<StreamWriter> streamWriterList = new ArrayList<>();
391391
for (int i = 0; i < 4; i++) {
392-
StreamWriter sw =
392+
streamWriterList.add(
393393
StreamWriter.newBuilder(
394394
String.format("projects/p1/datasets/d1/tables/t%s/streams/_default", i),
395395
externalClient)
396+
.setEnableConnectionPool(true)
396397
.setWriterSchema(createProtoSchema())
397398
.setTraceId(TEST_TRACE_ID)
398-
.setEnableConnectionPool(true)
399-
.build();
400-
streamWriterList.add(sw);
399+
.setLocation("us")
400+
.build());
401401
}
402402

403403
for (long i = 0; i < appendCount; i++) {

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteNonQuotaRetryTest.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery.storage.v1.it;
18+
19+
import com.google.cloud.bigquery.BigQuery;
20+
import com.google.cloud.bigquery.DatasetInfo;
21+
import com.google.cloud.bigquery.Field;
22+
import com.google.cloud.bigquery.LegacySQLTypeName;
23+
import com.google.cloud.bigquery.Schema;
24+
import com.google.cloud.bigquery.StandardTableDefinition;
25+
import com.google.cloud.bigquery.TableId;
26+
import com.google.cloud.bigquery.TableInfo;
27+
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
28+
import com.google.cloud.bigquery.storage.v1.WriteStream;
29+
import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
30+
import com.google.protobuf.Descriptors.DescriptorValidationException;
31+
import java.io.IOException;
32+
import java.util.logging.Logger;
33+
import org.junit.AfterClass;
34+
import org.junit.BeforeClass;
35+
import org.junit.Test;
36+
37+
/** Integration tests for BigQuery Write API. */
38+
public class ITBigQueryWriteNonQuotaRetryTest {
39+
private static final Logger LOG = Logger.getLogger(ITBigQueryWriteQuotaRetryTest.class.getName());
40+
private static final String DATASET = RemoteBigQueryHelper.generateDatasetName();
41+
private static final String TABLE = "testtable";
42+
private static final String DESCRIPTION = "BigQuery Write Java manual client test dataset";
43+
// This project is configured on the server to inject INTERNAL in-stream errors every
44+
// 10 messages. This is done to verify in-stream message retries.
45+
private static final String NON_QUOTA_RETRY_PROJECT_ID = "bq-write-api-java-retry-test";
46+
private static BigQueryWriteClient client;
47+
private static BigQuery bigquery;
48+
49+
@BeforeClass
50+
public static void beforeClass() throws IOException {
51+
client = BigQueryWriteClient.create();
52+
53+
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
54+
bigquery = bigqueryHelper.getOptions().getService();
55+
DatasetInfo datasetInfo =
56+
DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build();
57+
bigquery.create(datasetInfo);
58+
LOG.info("Created test dataset: " + DATASET);
59+
TableInfo tableInfo =
60+
TableInfo.newBuilder(
61+
TableId.of(DATASET, TABLE),
62+
StandardTableDefinition.of(
63+
Schema.of(
64+
Field.newBuilder("foo", LegacySQLTypeName.STRING)
65+
.setMode(Field.Mode.NULLABLE)
66+
.build())))
67+
.build();
68+
bigquery.create(tableInfo);
69+
}
70+
71+
@AfterClass
72+
public static void afterClass() {
73+
if (client != null) {
74+
client.close();
75+
}
76+
77+
if (bigquery != null) {
78+
RemoteBigQueryHelper.forceDelete(bigquery, DATASET);
79+
LOG.info("Deleted test dataset: " + DATASET);
80+
}
81+
}
82+
83+
@Test
84+
public void testJsonStreamWriterCommittedStreamWithNonQuotaRetry()
85+
throws IOException, InterruptedException, DescriptorValidationException {
86+
WriteRetryTestUtil.runExclusiveRetryTest(
87+
bigquery,
88+
client,
89+
DATASET,
90+
NON_QUOTA_RETRY_PROJECT_ID,
91+
WriteStream.Type.COMMITTED,
92+
/* requestCount=*/ 901,
93+
/* rowBatchSize=*/ 1);
94+
}
95+
96+
@Test
97+
public void testJsonStreamWriterDefaultStreamWithNonQuotaRetry()
98+
throws IOException, InterruptedException, DescriptorValidationException {
99+
WriteRetryTestUtil.runDefaultRetryTest(
100+
bigquery,
101+
client,
102+
DATASET,
103+
NON_QUOTA_RETRY_PROJECT_ID,
104+
/* requestCount=*/ 901,
105+
/* rowBatchSize=*/ 1);
106+
}
107+
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryWriteQuotaRetryTest.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigquery.storage.v1.it;
18+
19+
import com.google.cloud.bigquery.BigQuery;
20+
import com.google.cloud.bigquery.DatasetInfo;
21+
import com.google.cloud.bigquery.Field;
22+
import com.google.cloud.bigquery.LegacySQLTypeName;
23+
import com.google.cloud.bigquery.Schema;
24+
import com.google.cloud.bigquery.StandardTableDefinition;
25+
import com.google.cloud.bigquery.TableId;
26+
import com.google.cloud.bigquery.TableInfo;
27+
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
28+
import com.google.cloud.bigquery.storage.v1.WriteStream;
29+
import com.google.cloud.bigquery.testing.RemoteBigQueryHelper;
30+
import com.google.protobuf.Descriptors.DescriptorValidationException;
31+
import java.io.IOException;
32+
import java.util.logging.Logger;
33+
import org.junit.AfterClass;
34+
import org.junit.BeforeClass;
35+
import org.junit.Test;
36+
37+
/** Integration tests for BigQuery Write API. */
38+
public class ITBigQueryWriteQuotaRetryTest {
39+
private static final Logger LOG = Logger.getLogger(ITBigQueryWriteQuotaRetryTest.class.getName());
40+
private static final String DATASET = RemoteBigQueryHelper.generateDatasetName();
41+
private static final String TABLE = "testtable";
42+
private static final String DESCRIPTION = "BigQuery Write Java manual client test dataset";
43+
// This project is configured on the server to inject RESOURCE_EXHAUSTED in-stream errors every
44+
// 10 messages. This is done to verify in-stream message retries.
45+
private static final String QUOTA_RETRY_PROJECT_ID = "bq-writeapi-java-quota-retry";
46+
private static BigQueryWriteClient client;
47+
private static BigQuery bigquery;
48+
49+
@BeforeClass
50+
public static void beforeClass() throws IOException {
51+
client = BigQueryWriteClient.create();
52+
53+
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
54+
bigquery = bigqueryHelper.getOptions().getService();
55+
DatasetInfo datasetInfo =
56+
DatasetInfo.newBuilder(/* datasetId = */ DATASET).setDescription(DESCRIPTION).build();
57+
bigquery.create(datasetInfo);
58+
LOG.info("Created test dataset: " + DATASET);
59+
TableInfo tableInfo =
60+
TableInfo.newBuilder(
61+
TableId.of(DATASET, TABLE),
62+
StandardTableDefinition.of(
63+
Schema.of(
64+
Field.newBuilder("foo", LegacySQLTypeName.STRING)
65+
.setMode(Field.Mode.NULLABLE)
66+
.build())))
67+
.build();
68+
bigquery.create(tableInfo);
69+
}
70+
71+
@AfterClass
72+
public static void afterClass() {
73+
if (client != null) {
74+
client.close();
75+
}
76+
77+
if (bigquery != null) {
78+
RemoteBigQueryHelper.forceDelete(bigquery, DATASET);
79+
LOG.info("Deleted test dataset: " + DATASET);
80+
}
81+
}
82+
83+
@Test
84+
public void testJsonStreamWriterCommittedStreamWithQuotaRetry()
85+
throws IOException, InterruptedException, DescriptorValidationException {
86+
WriteRetryTestUtil.runExclusiveRetryTest(
87+
bigquery,
88+
client,
89+
DATASET,
90+
QUOTA_RETRY_PROJECT_ID,
91+
WriteStream.Type.COMMITTED,
92+
/* requestCount=*/ 901,
93+
/* rowBatchSize=*/ 1);
94+
}
95+
96+
@Test
97+
public void testJsonStreamWriterDefaultStreamWithQuotaRetry()
98+
throws IOException, InterruptedException, DescriptorValidationException {
99+
WriteRetryTestUtil.runDefaultRetryTest(
100+
bigquery,
101+
client,
102+
DATASET,
103+
QUOTA_RETRY_PROJECT_ID,
104+
/* requestCount=*/ 901,
105+
/* rowBatchSize=*/ 1);
106+
}
107+
}

0 commit comments

Comments
 (0)