feat: add support for partial success in ListBuckets for json (#3415)

Author: nidhiii-27

google-cloud-storage/clirr-ignored-differences.xml

@@ -167,6 +167,12 @@
  <method>com.google.cloud.storage.BucketInfo$Builder setGoogleManagedEncryptionEnforcementConfig(com.google.cloud.storage.BucketInfo$GoogleManagedEncryptionEnforcementConfig)</method>
  </difference>
 
+ <difference>
+ <differenceType>7013</differenceType>
+ <className>com/google/cloud/storage/BucketInfo$Builder</className>
+ <method>com.google.cloud.storage.BucketInfo$Builder setIsUnreachable(java.lang.Boolean)</method>
+ </difference>
+
  <!-- make beta api constructors private, they still retain their factory methods. -->
  <difference>
  <differenceType>7004</differenceType>

google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java

@@ -789,6 +789,12 @@ public Builder setCustomerSuppliedEncryptionEnforcementConfig(
  return this;
  }
 
+ @Override
+ public Builder setIsUnreachable(Boolean isUnreachable) {
+ infoBuilder.setIsUnreachable(isUnreachable);
+ return this;
+ }
+
  @Override
  public Bucket build() {
  return new Bucket(storage, infoBuilder);
@@ -997,6 +1003,12 @@ public Builder clearCustomerSuppliedEncryptionEnforcementConfig() {
  infoBuilder.clearCustomerSuppliedEncryptionEnforcementConfig();
  return this;
  }
+
+ @Override
+ Builder clearIsUnreachable() {
+ infoBuilder.clearIsUnreachable();
+ return this;
+ }
  }
 
  Bucket(Storage storage, BucketInfo.BuilderImpl infoBuilder) {

google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java

@@ -131,6 +131,7 @@ public class BucketInfo implements Serializable {
  customerManagedEncryptionEnforcementConfig;
  private final @Nullable CustomerSuppliedEncryptionEnforcementConfig
  customerSuppliedEncryptionEnforcementConfig;
+ private final Boolean isUnreachable;
 
  private final transient ImmutableSet<NamedField> modifiedFields;
 
@@ -2638,6 +2639,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {
  */
  public abstract Builder setIpFilter(IpFilter ipFilter);
 
+ public abstract Builder setIsUnreachable(Boolean isUnreachable);
+
  /** Creates a {@code BucketInfo} object. */
  public abstract BucketInfo build();
 
@@ -2708,6 +2711,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {
  abstract Builder clearCustomerManagedEncryptionEnforcementConfig();
 
  abstract Builder clearCustomerSuppliedEncryptionEnforcementConfig();
+
+ abstract Builder clearIsUnreachable();
  }
 
  static final class BuilderImpl extends Builder {
@@ -2751,6 +2756,7 @@ static final class BuilderImpl extends Builder {
  private GoogleManagedEncryptionEnforcementConfig googleManagedEncryptionEnforcementConfig;
  private CustomerManagedEncryptionEnforcementConfig customerManagedEncryptionEnforcementConfig;
  private CustomerSuppliedEncryptionEnforcementConfig customerSuppliedEncryptionEnforcementConfig;
+ private Boolean isUnreachable;
  private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();
 
  BuilderImpl(String name) {
@@ -2799,6 +2805,7 @@ static final class BuilderImpl extends Builder {
  bucketInfo.customerManagedEncryptionEnforcementConfig;
  customerSuppliedEncryptionEnforcementConfig =
  bucketInfo.customerSuppliedEncryptionEnforcementConfig;
+ isUnreachable = bucketInfo.isUnreachable;
  }
 
  @Override
@@ -3250,6 +3257,13 @@ public Builder setIpFilter(IpFilter ipFilter) {
  return this;
  }
 
+ @Override
+ public Builder setIsUnreachable(Boolean isUnreachable) {
+ Boolean tmp = firstNonNull(isUnreachable, Data.<Boolean>nullOf(Boolean.class));
+ this.isUnreachable = tmp;
+ return this;
+ }
+
  @Override
  public BucketInfo build() {
  checkNotNull(name);
@@ -3460,6 +3474,12 @@ BuilderImpl clearCustomerSuppliedEncryptionEnforcementConfig() {
  return this;
  }
 
+ @Override
+ BuilderImpl clearIsUnreachable() {
+ this.isUnreachable = null;
+ return this;
+ }
+
  private Builder clearDeleteLifecycleRules() {
  if (lifecycleRules != null && !lifecycleRules.isEmpty()) {
  ImmutableList<LifecycleRule> nonDeleteRules =
@@ -3513,6 +3533,7 @@ private Builder clearDeleteLifecycleRules() {
  customerManagedEncryptionEnforcementConfig = builder.customerManagedEncryptionEnforcementConfig;
  customerSuppliedEncryptionEnforcementConfig =
  builder.customerSuppliedEncryptionEnforcementConfig;
+ isUnreachable = builder.isUnreachable;
  modifiedFields = builder.modifiedFields.build();
  }
 
@@ -3886,6 +3907,16 @@ public HierarchicalNamespace getHierarchicalNamespace() {
  return customerSuppliedEncryptionEnforcementConfig;
  }
 
+ /**
+ * Returns a {@code Boolean} with {@code true} if the bucket is unreachable, else {@code null}
+ *
+ * <p>A bucket may be unreachable if the region in which it resides is experiencing an outage or
+ * if there are other temporary access issues.
+ */
+ public Boolean isUnreachable() {
+ return Data.isNull(isUnreachable) ? null : isUnreachable;
+ }
+
  /** Returns a builder for the current bucket. */
  public Builder toBuilder() {
  return new BuilderImpl(this);
@@ -3931,7 +3962,8 @@ public int hashCode() {
  ipFilter,
  googleManagedEncryptionEnforcementConfig,
  customerManagedEncryptionEnforcementConfig,
- customerSuppliedEncryptionEnforcementConfig);
+ customerSuppliedEncryptionEnforcementConfig,
+ isUnreachable);
  }
 
  @Override
@@ -3985,7 +4017,8 @@ public boolean equals(Object o) {
  that.customerManagedEncryptionEnforcementConfig)
  && Objects.equals(
  customerSuppliedEncryptionEnforcementConfig,
- that.customerSuppliedEncryptionEnforcementConfig);
+ that.customerSuppliedEncryptionEnforcementConfig)
+ && Objects.equals(isUnreachable, that.isUnreachable);
  }
 
  @Override

google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java

@@ -18,6 +18,7 @@
 
 import static com.google.cloud.storage.Storage.BucketField.IP_FILTER;
 import static com.google.cloud.storage.Storage.BucketField.SOFT_DELETE_POLICY;
+import static com.google.cloud.storage.Utils.bucketNameCodec;
 import static com.google.cloud.storage.Utils.dateTimeCodec;
 import static com.google.cloud.storage.Utils.durationSecondsCodec;
 import static com.google.cloud.storage.Utils.ifNonNull;
@@ -609,7 +610,7 @@ private Bucket bucketInfoEncode(BucketInfo from) {
 
  @SuppressWarnings("deprecation")
  private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket from) {
- BucketInfo.Builder to = new BucketInfo.BuilderImpl(from.getName());
+ BucketInfo.Builder to = new BucketInfo.BuilderImpl(bucketNameCodec.decode(from.getName()));
  ifNonNull(from.getProjectNumber(), to::setProject);
  ifNonNull(from.getAcl(), toListOf(bucketAcl()::decode), to::setAcl);
  ifNonNull(from.getCors(), toListOf(cors()::decode), to::setCors);
@@ -674,6 +675,9 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket
  ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention);
  ifNonNull(from.getSoftDeletePolicy(), this::softDeletePolicyDecode, to::setSoftDeletePolicy);
  ifNonNull(from.getIpFilter(), ipFilterCodec::decode, to::setIpFilter);
+ if (from.containsKey("isUnreachable")) {
+ to.setIsUnreachable(Boolean.TRUE);
+ }
  return to.build();
  }
 

google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java

@@ -2476,6 +2476,11 @@ public static BucketListOption pageToken(@NonNull String pageToken) {
  return new BucketListOption(UnifiedOpts.pageToken(pageToken));
  }
 
+ @TransportCompatibility({Transport.HTTP})
+ public static BucketListOption returnPartialSuccess(boolean returnPartialSuccess) {
+ return new BucketListOption(UnifiedOpts.returnPartialSuccess(returnPartialSuccess));
+ }
+
  /**
  * Returns an option to set a prefix to filter results to buckets whose names begin with this
  * prefix.

google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java

@@ -463,6 +463,10 @@ static PageToken pageToken(@NonNull String pageToken) {
  return new PageToken(pageToken);
  }
 
+ static ReturnPartialSuccess returnPartialSuccess(boolean returnPartialSuccess) {
+ return new ReturnPartialSuccess(returnPartialSuccess);
+ }
+
  static PredefinedAcl predefinedAcl(Storage.@NonNull PredefinedAcl predefinedAcl) {
  requireNonNull(predefinedAcl, "predefinedAcl must be non null");
  return new PredefinedAcl(predefinedAcl.getEntry());
@@ -1639,6 +1643,19 @@ public Mapper<ListObjectsRequest.Builder> listObjects() {
  }
  }
 
+ static final class ReturnPartialSuccess extends RpcOptVal<Boolean> implements BucketListOpt {
+ private static final long serialVersionUID = -1370658416509499277L;
+
+ private ReturnPartialSuccess(boolean val) {
+ super(StorageRpc.Option.RETURN_PARTIAL_SUCCESS, val);
+ }
+
+ @Override
+ public Mapper<ListBucketsRequest.Builder> listBuckets() {
+ return b -> b.setReturnPartialSuccess(val);
+ }
+ }
+
  static final class PredefinedAcl extends RpcOptVal<String>
  implements BucketTargetOpt, ObjectTargetOpt {
  private static final long serialVersionUID = -1743736785228368741L;

google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java

@@ -461,11 +461,18 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) {
  .setPrefix(Option.PREFIX.getString(options))
  .setMaxResults(Option.MAX_RESULTS.getLong(options))
  .setPageToken(Option.PAGE_TOKEN.getString(options))
+ .setReturnPartialSuccess(Option.RETURN_PARTIAL_SUCCESS.getBoolean(options))
  .setFields(Option.FIELDS.getString(options))
  .setUserProject(Option.USER_PROJECT.getString(options));
  setExtraHeaders(list, options);
- com.google.api.services.storage.model.Buckets buckets = list.execute();
- return Tuple.<String, Iterable<Bucket>>of(buckets.getNextPageToken(), buckets.getItems());
+ com.google.api.services.storage.model.Buckets bucketList = list.execute();
+ Iterable<Bucket> buckets =
+ Iterables.concat(
+ firstNonNull(bucketList.getItems(), ImmutableList.<Bucket>of()),
+ bucketList.getUnreachable() != null
+ ? Lists.transform(bucketList.getUnreachable(), createUnreachableBucket())
+ : ImmutableList.<Bucket>of());
+ return Tuple.<String, Iterable<Bucket>>of(bucketList.getNextPageToken(), buckets);
  } catch (IOException ex) {
  span.setStatus(Status.UNKNOWN.withDescription(ex.getMessage()));
  throw translate(ex);
@@ -530,6 +537,10 @@ private static String detectContentType(StorageObject object, Map<Option, ?> opt
  return firstNonNull(contentType, "application/octet-stream");
  }
 
+ private static Function<String, Bucket> createUnreachableBucket() {
+ return bucketName -> new Bucket().setName(bucketName).set("isUnreachable", "true");
+ }
+
  private static Function<String, StorageObject> objectFromPrefix(final String bucket) {
  return new Function<String, StorageObject>() {
  @Override

google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java

@@ -58,6 +58,7 @@ enum Option {
  PROJECTION("projection"),
  MAX_RESULTS("maxResults"),
  PAGE_TOKEN("pageToken"),
+ RETURN_PARTIAL_SUCCESS("returnPartialSuccess"),
  DELIMITER("delimiter"),
  START_OFF_SET("startOffset"),
  END_OFF_SET("endOffset"),

google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java

@@ -200,6 +200,8 @@ public class StorageImplMockitoTest {
  Storage.BucketListOption.fields();
  private static final Map<StorageRpc.Option, ?> BUCKET_LIST_OPTIONS =
  ImmutableMap.of(StorageRpc.Option.MAX_RESULTS, PAGE_SIZE, StorageRpc.Option.PREFIX, "prefix");
+ private static final Map<StorageRpc.Option, ?> BUCKET_LIST_PARTIAL_SUCCESS_OPTION =
+ ImmutableMap.of(StorageRpc.Option.RETURN_PARTIAL_SUCCESS, true);
 
  // Blob list options
  private static final Storage.BlobListOption BLOB_LIST_PAGE_SIZE =

google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITListBucketTest.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage.it;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.api.gax.paging.Page;
+import com.google.cloud.storage.Bucket;
+import com.google.cloud.storage.BucketInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BucketListOption;
+import com.google.cloud.storage.TransportCompatibility.Transport;
+import com.google.cloud.storage.it.runner.StorageITRunner;
+import com.google.cloud.storage.it.runner.annotations.Backend;
+import com.google.cloud.storage.it.runner.annotations.BucketFixture;
+import com.google.cloud.storage.it.runner.annotations.BucketType;
+import com.google.cloud.storage.it.runner.annotations.CrossRun;
+import com.google.cloud.storage.it.runner.annotations.Inject;
+import com.google.cloud.storage.it.runner.registry.Generator;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(StorageITRunner.class)
+@CrossRun(
+ backends = {Backend.TEST_BENCH},
+ transports = {Transport.HTTP})
+public class ITListBucketTest {
+ @Inject public Storage storage;
+
+ @Inject public BucketInfo defaultBucket;
+
+ @Inject
+ @BucketFixture(BucketType.HNS)
+ public BucketInfo hnsBucket;
+
+ @Inject public Generator generator;
+
+ private static final String UNREACHABLE_BUCKET_SUFFIX = ".unreachable";
+
+ @Test
+ public void testListBucketWithPartialSuccess() throws Exception {
+ doTest(Reachability.Unreachable, BucketListOption.returnPartialSuccess(true));
+ }
+
+ @Test
+ public void testListBucketWithoutPartialSuccess() throws Exception {
+ doTest(Reachability.Reachable);
+ }
+
+ private void doTest(
+ Reachability expectedReachabilityOfUnreachableBucket, BucketListOption... bucketListOption)
+ throws Exception {
+ // TESTBENCH considers a bucket to be unreachable if the bucket name contains "unreachable"
+ String name = generator.randomBucketName() + UNREACHABLE_BUCKET_SUFFIX;
+ BucketInfo info = BucketInfo.of(name);
+ try (TemporaryBucket tmpBucket =
+ TemporaryBucket.newBuilder().setBucketInfo(info).setStorage(storage).build()) {
+ Map<String, Reachability> expected =
+ ImmutableMap.of(
+ defaultBucket.getName(), Reachability.Reachable,
+ hnsBucket.getName(), Reachability.Reachable,
+ tmpBucket.getBucket().getName(), expectedReachabilityOfUnreachableBucket);
+
+ Page<Bucket> page = storage.list(bucketListOption);
+
+ Map<String, Reachability> actual =
+ page.streamAll().collect(Collectors.toMap(BucketInfo::getName, Reachability::forBucket));
+
+ assertThat(actual).containsAtLeastEntriesIn(expected);
+ }
+ }
+
+ private enum Reachability {
+ Reachable,
+ Unreachable;
+
+ static Reachability forBucket(BucketInfo b) {
+ if (b.isUnreachable() != null && b.isUnreachable()) {
+ return Unreachable;
+ } else {
+ return Reachable;
+ }
+ }
+ }
+}