feat: Add matchGlob parameter to BlobListOption (#1965)

This is a new feature (b/236167515) where List Objects allows filtering results based on a provided glob pattern. Currently only the JSON API supports the matchGlob parameter, so using the option with gRPC will throw an exception. gRPC support is planned for later. --------- Co-authored-by: BenWhitehead <BenWhitehead@users.noreply.github.com>

Author: briantruong777

google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java

@@ -20,6 +20,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Objects.requireNonNull;
 
+import com.google.api.core.BetaApi;
 import com.google.api.core.InternalExtensionOnly;
 import com.google.api.gax.paging.Page;
 import com.google.auth.ServiceAccountSigner;
@@ -1300,6 +1301,18 @@ public static BlobListOption endOffset(@NonNull String endOffset) {
  return new BlobListOption(UnifiedOpts.endOffset(endOffset));
  }
 
+ /**
+ * Returns an option to set a glob pattern to filter results to blobs that match the pattern.
+ *
+ * @see <a href="https://cloud.google.com/storage/docs/json_api/v1/objects/list">List
+ * Objects</a>
+ */
+ @BetaApi
+ @TransportCompatibility({Transport.HTTP})
+ public static BlobListOption matchGlob(@NonNull String glob) {
+ return new BlobListOption(UnifiedOpts.matchGlob(glob));
+ }
+
  /**
  * Returns an option to define the billing user project. This option is required by buckets with
  * `requester_pays` flag enabled to assign operation costs.

google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java

@@ -413,6 +413,11 @@ static KmsKeyName kmsKeyName(@NonNull String kmsKeyName) {
  return new KmsKeyName(kmsKeyName);
  }
 
+ static MatchGlob matchGlob(@NonNull String glob) {
+ requireNonNull(glob, "glob must be non null");
+ return new MatchGlob(glob);
+ }
+
  static Md5Match md5Match(@NonNull String md5) {
  requireNonNull(md5, "md5 must be non null");
  return new Md5Match(md5);
@@ -1070,6 +1075,20 @@ public Mapper<RewriteObjectRequest.Builder> rewriteObject() {
  }
  }
 
+ static final class MatchGlob extends RpcOptVal<String> implements ObjectListOpt {
+ private static final long serialVersionUID = 8819855597395473178L;
+
+ private MatchGlob(String val) {
+ super(StorageRpc.Option.MATCH_GLOB, val);
+ }
+
+ @Override
+ public Mapper<ListObjectsRequest.Builder> listObjects() {
+ return GrpcStorageImpl.throwHttpJsonOnly(
+ com.google.cloud.storage.Storage.BlobListOption.class, "matchGlob(String)");
+ }
+ }
+
  @Deprecated
  static final class Md5Match implements ObjectTargetOpt {
  private static final long serialVersionUID = 5237207911268363887L;

google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java

@@ -423,6 +423,7 @@ public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Opti
  .setDelimiter(Option.DELIMITER.getString(options))
  .setStartOffset(Option.START_OFF_SET.getString(options))
  .setEndOffset(Option.END_OFF_SET.getString(options))
+ .setMatchGlob(Option.MATCH_GLOB.getString(options))
  .setPrefix(Option.PREFIX.getString(options))
  .setMaxResults(Option.MAX_RESULTS.getLong(options))
  .setPageToken(Option.PAGE_TOKEN.getString(options))

google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java

@@ -61,6 +61,7 @@ enum Option {
  DELIMITER("delimiter"),
  START_OFF_SET("startOffset"),
  END_OFF_SET("endOffset"),
+ MATCH_GLOB("matchGlob"),
  VERSIONS("versions"),
  FIELDS("fields"),
  CUSTOMER_SUPPLIED_KEY("customerSuppliedKey"),

google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java

@@ -1308,6 +1308,27 @@ public void testListBlobsWithOffset() {
  assertArrayEquals(blobList.toArray(), Iterables.toArray(page.getValues(), Blob.class));
  }
 
+ @Test
+ public void testListBlobsMatchGlob() {
+ String cursor = "cursor";
+ String matchGlob = "foo*bar";
+ Map<StorageRpc.Option, ?> options = ImmutableMap.of(StorageRpc.Option.MATCH_GLOB, matchGlob);
+ ImmutableList<BlobInfo> blobInfoList = ImmutableList.of(BLOB_INFO1, BLOB_INFO2);
+ Tuple<String, Iterable<com.google.api.services.storage.model.StorageObject>> result =
+ Tuple.of(
+ cursor, Iterables.transform(blobInfoList, Conversions.apiary().blobInfo()::encode));
+ doReturn(result)
+ .doThrow(UNEXPECTED_CALL_EXCEPTION)
+ .when(storageRpcMock)
+ .list(BUCKET_NAME1, options);
+
+ initializeService();
+ ImmutableList<Blob> blobList = ImmutableList.of(expectedBlob1, expectedBlob2);
+ Page<Blob> page = storage.list(BUCKET_NAME1, Storage.BlobListOption.matchGlob(matchGlob));
+ assertEquals(cursor, page.getNextPageToken());
+ assertArrayEquals(blobList.toArray(), Iterables.toArray(page.getValues(), Blob.class));
+ }
+
  @Test
  public void testListBlobsWithException() {
  doThrow(STORAGE_FAILURE).when(storageRpcMock).list(BUCKET_NAME1, EMPTY_RPC_OPTIONS);

google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectTest.java

@@ -16,6 +16,7 @@
 
 package com.google.cloud.storage.it;
 
+import static com.google.cloud.storage.TestUtils.assertAll;
 import static com.google.common.truth.Truth.assertThat;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertArrayEquals;
@@ -65,6 +66,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
@@ -542,6 +544,60 @@ public void testListBlobsCurrentDirectoryIncludesBothObjectsAndSyntheticDirector
  assertThat(actual).contains(PackagePrivateMethodWorkarounds.noAcl(obj2Gen1));
  }
 
+ @Test
+ // When gRPC support is added for matchGlob, enable this test for gRPC.
+ @Exclude(transports = Transport.GRPC)
+ public void testListBlobsWithMatchGlob() throws Exception {
+ BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build();
+ try (TemporaryBucket tempBucket =
+ TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) {
+ BucketInfo bucket = tempBucket.getBucket();
+ assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foo/bar").build()));
+ assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foo/baz").build()));
+ assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foo/foobar").build()));
+ assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foobar").build()));
+
+ Page<Blob> page1 = storage.list(bucket.getName(), BlobListOption.matchGlob("foo*bar"));
+ Page<Blob> page2 = storage.list(bucket.getName(), BlobListOption.matchGlob("foo**bar"));
+ Page<Blob> page3 = storage.list(bucket.getName(), BlobListOption.matchGlob("**/foobar"));
+ Page<Blob> page4 = storage.list(bucket.getName(), BlobListOption.matchGlob("*/ba[rz]"));
+ Page<Blob> page5 = storage.list(bucket.getName(), BlobListOption.matchGlob("*/ba[!a-y]"));
+ Page<Blob> page6 =
+ storage.list(bucket.getName(), BlobListOption.matchGlob("**/{foobar,baz}"));
+ Page<Blob> page7 =
+ storage.list(bucket.getName(), BlobListOption.matchGlob("foo/{foo*,*baz}"));
+ assertAll(
+ () ->
+ assertThat(Iterables.transform(page1.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foobar")
+ .inOrder(),
+ () ->
+ assertThat(Iterables.transform(page2.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foo/bar", "foo/foobar", "foobar")
+ .inOrder(),
+ () ->
+ assertThat(Iterables.transform(page3.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foo/foobar", "foobar")
+ .inOrder(),
+ () ->
+ assertThat(Iterables.transform(page4.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foo/bar", "foo/baz")
+ .inOrder(),
+ () ->
+ assertThat(Iterables.transform(page5.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foo/baz")
+ .inOrder(),
+ () ->
+ assertThat(Iterables.transform(page6.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foo/baz", "foo/foobar", "foobar")
+ .inOrder(),
+ () ->
+ assertThat(Iterables.transform(page7.iterateAll(), blob -> blob.getName()))
+ .containsExactly("foo/baz", "foo/foobar")
+ .inOrder());
+ }
+ }
+
  @Test
  public void testListBlobsMultiplePages() {
  String basePath = generator.randomObjectName();