Skip to content

Commit 76bb401

Browse files
fix: fix compatibility of CloudBigtableIO with the emulator env var (#3940)
There are 2 issues that prevented CloudBigtableIO from working with the emulator: 1. useBulk macro setting would override the endpoint to batch-bigtable.googleapis.com regardless if the BIGTABLE_EMULATOR_ENV was set 2. the emulator doesnt return sample row key sizes when when the data is tiny, so split calculations broke
1 parent 548aa8f commit 76bb401

File tree

2 files changed

+6
-2
lines changed

2 files changed

+6
-2
lines changed

bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/wrappers/veneer/BigtableHBaseVeneerSettings.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,9 @@ private void configureConnection(StubSettings.Builder<?, ?> stubSettings, String
429429
endpointOverride =
430430
Optional.of(hostOverride.or(defaultHostname) + ":" + portOverride.or(defaultPort));
431431
} else if (endpointKey.equals(BIGTABLE_HOST_KEY)
432+
// only apply batch endpoint when the default endpoint hasnt changed (ie. when
433+
// BIGTABLE_EMULATOR_HOST is set)
434+
&& defaultEndpoint.equals("bigtable.googleapis.com:443")
432435
&& configuration.getBoolean(BIGTABLE_USE_BATCH, false)) {
433436
endpointOverride = Optional.of(BIGTABLE_BATCH_DATA_HOST_DEFAULT + ":443");
434437
}

bigtable-dataflow-parent/bigtable-hbase-beam/src/main/java/com/google/cloud/bigtable/beam/CloudBigtableIO.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ protected long calculateEstimatedSizeBytes(PipelineOptions options) throws IOExc
335335
protected List<SourceWithKeys> split(
336336
long regionSize, long desiredBundleSizeBytes, byte[] startKey, byte[] stopKey)
337337
throws IOException {
338-
Preconditions.checkState(desiredBundleSizeBytes > 0);
338+
Preconditions.checkState(desiredBundleSizeBytes >= 0);
339339
int splitCount = (int) Math.ceil((double) (regionSize) / (double) (desiredBundleSizeBytes));
340340

341341
if (splitCount < 2 || stopKey.length == 0 || Bytes.compareTo(startKey, stopKey) >= 0) {
@@ -477,7 +477,8 @@ protected SourceWithKeys(CloudBigtableScanConfiguration configuration, long esti
477477
"Source keys not in order: [%s, %s]",
478478
Bytes.toStringBinary(startRow), Bytes.toStringBinary(stopRow)));
479479
}
480-
Preconditions.checkState(estimatedSize > 0, "Source size must be positive", estimatedSize);
480+
Preconditions.checkState(
481+
estimatedSize >= 0, "Source size cannot be negative", estimatedSize);
481482
}
482483
this.estimatedSize = estimatedSize;
483484
SOURCE_LOG.debug("Source with split: {}.", this);

0 commit comments

Comments
 (0)