Skip to content

Commit 8a5e60e

Browse files
liubonanliubonan
authored
feat: allow restore backup to different instance (#515)
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [x] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/java-bigtable/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [x] Ensure the tests and linter pass - [x] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary) Fixes #789 kokoro:force-run
1 parent b4dfddd commit 8a5e60e

File tree

3 files changed

+183
-39
lines changed

3 files changed

+183
-39
lines changed

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/RestoreTableRequest.java‎

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,46 @@
2020
import com.google.common.base.Objects;
2121
import com.google.common.base.Preconditions;
2222
import javax.annotation.Nonnull;
23+
import javax.annotation.Nullable;
2324

2425
/** Fluent wrapper for {@link com.google.bigtable.admin.v2.RestoreTableRequest} */
2526
public final class RestoreTableRequest {
2627
private final com.google.bigtable.admin.v2.RestoreTableRequest.Builder requestBuilder =
2728
com.google.bigtable.admin.v2.RestoreTableRequest.newBuilder();
28-
private final String backupId;
29-
private final String clusterId;
29+
private final String sourceBackupId;
30+
private final String sourceClusterId;
31+
private final String sourceInstanceId;
3032

31-
public static RestoreTableRequest of(String clusterId, String backupId) {
32-
RestoreTableRequest request = new RestoreTableRequest(clusterId, backupId);
33+
/**
34+
* Create a {@link RestoreTableRequest} object. It assumes the source backup locates in the same
35+
* instance as the destination table. To restore a table from a backup in another instance, use
36+
* {@link #of(String, String, String) of} method.
37+
*/
38+
public static RestoreTableRequest of(String sourceClusterId, String sourceBackupId) {
39+
RestoreTableRequest request = new RestoreTableRequest(null, sourceClusterId, sourceBackupId);
3340
return request;
3441
}
3542

36-
private RestoreTableRequest(String clusterId, String backupId) {
37-
Preconditions.checkNotNull(clusterId);
38-
Preconditions.checkNotNull(backupId);
39-
this.backupId = backupId;
40-
this.clusterId = clusterId;
43+
/**
44+
* Create a {@link RestoreTableRequest} object. The source backup could locate in a the same or a
45+
* different instance.
46+
*/
47+
public static RestoreTableRequest of(
48+
String sourceInstanceId, String sourceClusterId, String sourceBackupId) {
49+
RestoreTableRequest request =
50+
new RestoreTableRequest(sourceInstanceId, sourceClusterId, sourceBackupId);
51+
return request;
52+
}
53+
54+
private RestoreTableRequest(
55+
@Nullable String sourceInstanceId,
56+
@Nonnull String sourceClusterId,
57+
@Nonnull String sourceBackupId) {
58+
Preconditions.checkNotNull(sourceClusterId);
59+
Preconditions.checkNotNull(sourceBackupId);
60+
this.sourceBackupId = sourceBackupId;
61+
this.sourceInstanceId = sourceInstanceId;
62+
this.sourceClusterId = sourceClusterId;
4163
}
4264

4365
public RestoreTableRequest setTableId(String tableId) {
@@ -56,13 +78,15 @@ public boolean equals(Object o) {
5678
}
5779
RestoreTableRequest that = (RestoreTableRequest) o;
5880
return Objects.equal(requestBuilder.getTableId(), that.requestBuilder.getTableId())
59-
&& Objects.equal(clusterId, that.clusterId)
60-
&& Objects.equal(backupId, that.backupId);
81+
&& Objects.equal(sourceInstanceId, that.sourceInstanceId)
82+
&& Objects.equal(sourceClusterId, that.sourceClusterId)
83+
&& Objects.equal(sourceBackupId, that.sourceBackupId);
6184
}
6285

6386
@Override
6487
public int hashCode() {
65-
return Objects.hashCode(requestBuilder.getTableId(), clusterId, backupId);
88+
return Objects.hashCode(
89+
requestBuilder.getTableId(), sourceInstanceId, sourceClusterId, sourceBackupId);
6690
}
6791

6892
@InternalApi
@@ -73,7 +97,12 @@ public com.google.bigtable.admin.v2.RestoreTableRequest toProto(
7397

7498
return requestBuilder
7599
.setParent(NameUtil.formatInstanceName(projectId, instanceId))
76-
.setBackup(NameUtil.formatBackupName(projectId, instanceId, clusterId, backupId))
100+
.setBackup(
101+
NameUtil.formatBackupName(
102+
projectId,
103+
sourceInstanceId == null ? instanceId : sourceInstanceId,
104+
sourceClusterId,
105+
sourceBackupId))
77106
.build();
78107
}
79108
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java‎

Lines changed: 90 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -103,26 +103,9 @@ public static void createClient()
103103
prefix = String.format("020%d", System.currentTimeMillis());
104104

105105
tableAdmin = testEnvRule.env().getTableAdminClientForInstance(targetInstance);
106-
107-
testTable =
108-
tableAdmin.createTable(
109-
CreateTableRequest.of(generateId(TEST_TABLE_SUFFIX)).addFamily("cf1"));
110-
111-
// Populate test data.
112106
dataClient = testEnvRule.env().getDataClientForInstance(targetInstance);
113-
byte[] rowBytes = new byte[1024];
114-
Random random = new Random();
115-
random.nextBytes(rowBytes);
116107

117-
List<ApiFuture<?>> futures = Lists.newArrayList();
118-
for (int i = 0; i < 10; i++) {
119-
ApiFuture<Void> future =
120-
dataClient.mutateRowAsync(
121-
RowMutation.create(testTable.getId(), "test-row-" + i)
122-
.setCell("cf1", ByteString.EMPTY, ByteString.copyFrom(rowBytes)));
123-
futures.add(future);
124-
}
125-
ApiFutures.allAsList(futures).get(3, TimeUnit.MINUTES);
108+
testTable = createAndPopulateTestTable(tableAdmin, dataClient);
126109
}
127110

128111
@AfterClass
@@ -266,32 +249,89 @@ public void deleteBackupTest() throws InterruptedException {
266249
@Test
267250
public void restoreTableTest() throws InterruptedException, ExecutionException {
268251
String backupId = generateId("restore-" + TEST_BACKUP_SUFFIX);
269-
String tableId = generateId("restored-table");
252+
String restoredTableId = generateId("restored-table");
270253
tableAdmin.createBackup(createBackupRequest(backupId));
271254

272255
// Wait 2 minutes so that the RestoreTable API will trigger an optimize restored
273256
// table operation.
274257
Thread.sleep(120 * 1000);
275258

276259
try {
277-
RestoreTableRequest req = RestoreTableRequest.of(targetCluster, backupId).setTableId(tableId);
260+
RestoreTableRequest req =
261+
RestoreTableRequest.of(targetCluster, backupId).setTableId(restoredTableId);
278262
RestoredTableResult result = tableAdmin.restoreTable(req);
279263
assertWithMessage("Incorrect restored table id")
280264
.that(result.getTable().getId())
281-
.isEqualTo(tableId);
265+
.isEqualTo(restoredTableId);
282266

283267
if (result.getOptimizeRestoredTableOperationToken() != null) {
284268
// The assertion might be missing if the test is running against a HDD cluster or an
285269
// optimization is not necessary.
286270
tableAdmin.awaitOptimizeRestoredTable(result.getOptimizeRestoredTableOperationToken());
287-
Table restoredTable = tableAdmin.getTable(tableId);
271+
Table restoredTable = tableAdmin.getTable(restoredTableId);
288272
assertWithMessage("Incorrect restored table id")
289273
.that(restoredTable.getId())
290-
.isEqualTo(tableId);
274+
.isEqualTo(restoredTableId);
291275
}
292276
} finally {
293277
tableAdmin.deleteBackup(targetCluster, backupId);
294-
tableAdmin.deleteTable(tableId);
278+
tableAdmin.deleteTable(restoredTableId);
279+
}
280+
}
281+
282+
@Test
283+
public void crossInstanceRestoreTest()
284+
throws InterruptedException, IOException, ExecutionException, TimeoutException {
285+
String backupId = generateId("cross-" + TEST_BACKUP_SUFFIX);
286+
String restoredTableId = generateId("restored-table-2");
287+
288+
// Set up a new instance to test cross-instance restore. The source backup is stored in this
289+
// instance.
290+
String sourceInstance =
291+
AbstractTestEnv.TEST_INSTANCE_PREFIX + "backup-" + Instant.now().getEpochSecond();
292+
String sourceCluster = AbstractTestEnv.TEST_CLUSTER_PREFIX + Instant.now().getEpochSecond();
293+
instanceAdmin.createInstance(
294+
CreateInstanceRequest.of(sourceInstance)
295+
.addCluster(sourceCluster, testEnvRule.env().getSecondaryZone(), 3, StorageType.SSD)
296+
.setDisplayName("backups-source-test-instance")
297+
.addLabel("state", "readytodelete")
298+
.setType(Type.PRODUCTION));
299+
BigtableTableAdminClient sourceTableAdmin =
300+
testEnvRule.env().getTableAdminClientForInstance(sourceInstance);
301+
Table sourceTable =
302+
createAndPopulateTestTable(
303+
sourceTableAdmin, testEnvRule.env().getDataClientForInstance(sourceInstance));
304+
sourceTableAdmin.createBackup(
305+
CreateBackupRequest.of(sourceCluster, backupId)
306+
.setSourceTableId(sourceTable.getId())
307+
.setExpireTime(Instant.now().plus(Duration.ofHours(6))));
308+
309+
// Wait 2 minutes so that the RestoreTable API will trigger an optimize restored
310+
// table operation.
311+
Thread.sleep(120 * 1000);
312+
313+
try {
314+
RestoreTableRequest req =
315+
RestoreTableRequest.of(sourceInstance, sourceCluster, backupId)
316+
.setTableId(restoredTableId);
317+
RestoredTableResult result = tableAdmin.restoreTable(req);
318+
assertWithMessage("Incorrect restored table id")
319+
.that(result.getTable().getId())
320+
.isEqualTo(restoredTableId);
321+
assertWithMessage("Incorrect instance id")
322+
.that(result.getTable().getInstanceId())
323+
.isEqualTo(targetInstance);
324+
325+
// The assertion might be missing if the test is running against a HDD cluster or an
326+
// optimization is not necessary.
327+
assertWithMessage("Empty OptimizeRestoredTable token")
328+
.that(result.getOptimizeRestoredTableOperationToken())
329+
.isNotNull();
330+
tableAdmin.awaitOptimizeRestoredTable(result.getOptimizeRestoredTableOperationToken());
331+
tableAdmin.getTable(restoredTableId);
332+
} finally {
333+
sourceTableAdmin.deleteBackup(sourceCluster, backupId);
334+
instanceAdmin.deleteInstance(sourceInstance);
295335
}
296336
}
297337

@@ -327,13 +367,37 @@ public void backupIamTest() throws InterruptedException {
327367
}
328368
}
329369

330-
private CreateBackupRequest createBackupRequest(String backupName) {
331-
return CreateBackupRequest.of(targetCluster, backupName)
370+
private CreateBackupRequest createBackupRequest(String backupId) {
371+
return CreateBackupRequest.of(targetCluster, backupId)
332372
.setSourceTableId(testTable.getId())
333373
.setExpireTime(Instant.now().plus(Duration.ofDays(15)));
334374
}
335375

336376
private static String generateId(String name) {
337377
return prefix + "-" + name;
338378
}
379+
380+
private static Table createAndPopulateTestTable(
381+
BigtableTableAdminClient tableAdmin, BigtableDataClient dataClient)
382+
throws InterruptedException, ExecutionException, TimeoutException {
383+
Table testTable =
384+
tableAdmin.createTable(
385+
CreateTableRequest.of(generateId(TEST_TABLE_SUFFIX)).addFamily("cf1"));
386+
387+
// Populate test data.
388+
byte[] rowBytes = new byte[1024];
389+
Random random = new Random();
390+
random.nextBytes(rowBytes);
391+
392+
List<ApiFuture<?>> futures = Lists.newArrayList();
393+
for (int i = 0; i < 10; i++) {
394+
ApiFuture<Void> future =
395+
dataClient.mutateRowAsync(
396+
RowMutation.create(testTable.getId(), "test-row-" + i)
397+
.setCell("cf1", ByteString.EMPTY, ByteString.copyFrom(rowBytes)));
398+
futures.add(future);
399+
}
400+
ApiFutures.allAsList(futures).get(3, TimeUnit.MINUTES);
401+
return testTable;
402+
}
339403
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/RestoreTableRequestTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public class RestoreTableRequestTest {
3030
private static final String PROJECT_ID = "my-project";
3131
private static final String INSTANCE_ID = "my-instance";
3232
private static final String CLUSTER_ID = "my-cluster";
33+
private static final String SOURCE_INSTANCE_ID = "source-instance-id";
3334

3435
@Test
3536
public void testToProto() {
@@ -45,6 +46,21 @@ public void testToProto() {
4546
assertThat(request.toProto(PROJECT_ID, INSTANCE_ID)).isEqualTo(requestProto);
4647
}
4748

49+
@Test
50+
public void testToProtoCrossInstance() {
51+
RestoreTableRequest request =
52+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID).setTableId(TABLE_ID);
53+
54+
com.google.bigtable.admin.v2.RestoreTableRequest requestProto =
55+
com.google.bigtable.admin.v2.RestoreTableRequest.newBuilder()
56+
.setParent(NameUtil.formatInstanceName(PROJECT_ID, INSTANCE_ID))
57+
.setBackup(
58+
NameUtil.formatBackupName(PROJECT_ID, SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID))
59+
.setTableId(TABLE_ID)
60+
.build();
61+
assertThat(request.toProto(PROJECT_ID, INSTANCE_ID)).isEqualTo(requestProto);
62+
}
63+
4864
@Test
4965
public void testEquality() {
5066
RestoreTableRequest request =
@@ -56,6 +72,22 @@ public void testEquality() {
5672
.isNotEqualTo(RestoreTableRequest.of(CLUSTER_ID, BACKUP_ID).setTableId("another-table"));
5773
}
5874

75+
@Test
76+
public void testEqualityCrossInstance() {
77+
RestoreTableRequest request =
78+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID).setTableId(TABLE_ID);
79+
80+
assertThat(request)
81+
.isEqualTo(
82+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID).setTableId(TABLE_ID));
83+
assertThat(request)
84+
.isNotEqualTo(RestoreTableRequest.of(CLUSTER_ID, BACKUP_ID).setTableId(TABLE_ID));
85+
assertThat(request)
86+
.isNotEqualTo(
87+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID)
88+
.setTableId("another-table"));
89+
}
90+
5991
@Test
6092
public void testHashCode() {
6193
RestoreTableRequest request =
@@ -66,4 +98,23 @@ public void testHashCode() {
6698
.isNotEqualTo(
6799
RestoreTableRequest.of(CLUSTER_ID, BACKUP_ID).setTableId("another-table").hashCode());
68100
}
101+
102+
@Test
103+
public void testHashCodeCrossInstance() {
104+
RestoreTableRequest request =
105+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID).setTableId(TABLE_ID);
106+
assertThat(request.hashCode())
107+
.isEqualTo(
108+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID)
109+
.setTableId(TABLE_ID)
110+
.hashCode());
111+
assertThat(request.hashCode())
112+
.isNotEqualTo(
113+
RestoreTableRequest.of(CLUSTER_ID, BACKUP_ID).setTableId(TABLE_ID).hashCode());
114+
assertThat(request.hashCode())
115+
.isNotEqualTo(
116+
RestoreTableRequest.of(SOURCE_INSTANCE_ID, CLUSTER_ID, BACKUP_ID)
117+
.setTableId("another-table")
118+
.hashCode());
119+
}
69120
}

0 commit comments

Comments
 (0)