feat: ReEncryptInstructionFile Implementation (#470)

--------- Co-authored-by: Anirav Kareddy <aniravk@amazon.com>

Author: akareddy04

src/examples/java/software/amazon/encryption/s3/examples/ReEncryptInstructionFileExample.java

@@ -46,20 +46,30 @@
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
+import software.amazon.encryption.s3.internal.ContentMetadata;
+import software.amazon.encryption.s3.internal.ContentMetadataDecodingStrategy;
+import software.amazon.encryption.s3.internal.ContentMetadataEncodingStrategy;
 import software.amazon.encryption.s3.internal.ConvertSDKRequests;
 import software.amazon.encryption.s3.internal.GetEncryptedObjectPipeline;
 import software.amazon.encryption.s3.internal.InstructionFileConfig;
 import software.amazon.encryption.s3.internal.MultiFileOutputStream;
 import software.amazon.encryption.s3.internal.MultipartUploadObjectPipeline;
 import software.amazon.encryption.s3.internal.PutEncryptedObjectPipeline;
+import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest;
+import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse;
 import software.amazon.encryption.s3.internal.UploadObjectObserver;
 import software.amazon.encryption.s3.materials.AesKeyring;
 import software.amazon.encryption.s3.materials.CryptographicMaterialsManager;
+import software.amazon.encryption.s3.materials.DecryptMaterialsRequest;
+import software.amazon.encryption.s3.materials.DecryptionMaterials;
 import software.amazon.encryption.s3.materials.DefaultCryptoMaterialsManager;
+import software.amazon.encryption.s3.materials.EncryptedDataKey;
+import software.amazon.encryption.s3.materials.EncryptionMaterials;
 import software.amazon.encryption.s3.materials.Keyring;
 import software.amazon.encryption.s3.materials.KmsKeyring;
 import software.amazon.encryption.s3.materials.MultipartConfiguration;
 import software.amazon.encryption.s3.materials.PartialRsaKeyPair;
+import software.amazon.encryption.s3.materials.RawKeyring;
 import software.amazon.encryption.s3.materials.RsaKeyring;
 
 import javax.crypto.SecretKey;
@@ -69,6 +79,7 @@
 import java.security.Provider;
 import java.security.SecureRandom;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -81,7 +92,8 @@
 import java.util.function.Consumer;
 
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_BUFFER_SIZE_BYTES;
-import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX;
+
+import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_INSTRUCTION_FILE_SUFFIX;
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MAX_ALLOWED_BUFFER_SIZE_BYTES;
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MIN_ALLOWED_BUFFER_SIZE_BYTES;
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.instructionFileKeysToDelete;
@@ -97,6 +109,9 @@ public class S3EncryptionClient extends DelegatingS3Client {
  public static final ExecutionAttribute<Map<String, String>> ENCRYPTION_CONTEXT = new ExecutionAttribute<>("EncryptionContext");
  public static final ExecutionAttribute<MultipartConfiguration> CONFIGURATION = new ExecutionAttribute<>("MultipartConfiguration");
 
+ //Used for specifying custom instruction file suffix on a per-request basis
+ public static final ExecutionAttribute<String> CUSTOM_INSTRUCTION_FILE_SUFFIX = new ExecutionAttribute<>("CustomInstructionFileSuffix");
+
  private final S3Client _wrappedClient;
  private final S3AsyncClient _wrappedAsyncClient;
  private final CryptographicMaterialsManager _cryptoMaterialsManager;
@@ -143,6 +158,18 @@ public static Consumer<AwsRequestOverrideConfiguration.Builder> withAdditionalCo
  builder.putExecutionAttribute(S3EncryptionClient.ENCRYPTION_CONTEXT, encryptionContext);
  }
 
+ /**
+ * Attaches a custom instruction file suffix to a request. Must be used as a parameter to
+ * {@link S3Request#overrideConfiguration()} in the request.
+ * This allows specifying a custom suffix for the instruction file on a per-request basis.
+ * @param customInstructionFileSuffix the custom suffix to use for the instruction file.
+ * @return Consumer for use in overrideConfiguration()
+ */
+ public static Consumer<AwsRequestOverrideConfiguration.Builder> withCustomInstructionFileSuffix(String customInstructionFileSuffix) {
+ return builder ->
+ builder.putExecutionAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX, customInstructionFileSuffix);
+ }
+
  /**
  * Attaches multipart configuration to a request. Must be used as a parameter to
  * {@link S3Request#overrideConfiguration()} in the request.
@@ -154,7 +181,6 @@ public static Consumer<AwsRequestOverrideConfiguration.Builder> withAdditionalCo
  builder.putExecutionAttribute(S3EncryptionClient.CONFIGURATION, multipartConfiguration);
  }
 
-
  /**
  * Attaches encryption context and multipart configuration to a request.
  * * Must be used as a parameter to
@@ -172,6 +198,77 @@ public static Consumer<AwsRequestOverrideConfiguration.Builder> withAdditionalCo
  .putExecutionAttribute(S3EncryptionClient.CONFIGURATION, multipartConfiguration);
  }
 
+ /**
+ * Re-encrypts an instruction file with a new keyring while preserving the original encrypted object in S3.
+ * This enables:
+ * 1. Key rotation by updating instruction file metadata without re-encrypting object content
+ * 2. Sharing encrypted objects with partners by creating new instruction files with a custom suffix using their public keys
+ * <p>
+ * Key rotation scenarios:
+ * - Legacy to V3: Can rotate same wrapping key from legacy wrapping algorithms to fully supported wrapping algorithms
+ * - Within V3: When rotating the wrapping key, the new keyring must be different from the current keyring
+ *
+ * @param reEncryptInstructionFileRequest the request containing bucket, object key, new keyring, and optional instruction file suffix
+ * @return ReEncryptInstructionFileResponse containing the bucket, object key, and instruction file suffix used
+ * @throws S3EncryptionClientException if the new keyring has the same materials description as the current one
+ */
+ public ReEncryptInstructionFileResponse reEncryptInstructionFile(ReEncryptInstructionFileRequest reEncryptInstructionFileRequest) {
+ //Build request to retrieve the encrypted object and its associated instruction file
+ final GetObjectRequest request = GetObjectRequest.builder()
+ .bucket(reEncryptInstructionFileRequest.bucket())
+ .key(reEncryptInstructionFileRequest.key())
+ .build();
+
+ ResponseInputStream<GetObjectResponse> response = this.getObject(request);
+ ContentMetadataDecodingStrategy decodingStrategy = new ContentMetadataDecodingStrategy(_instructionFileConfig);
+ ContentMetadata contentMetadata = decodingStrategy.decode(request, response.response());
+
+ //Extract cryptographic parameters from the current instruction file that MUST be preserved during re-encryption
+ final AlgorithmSuite algorithmSuite = contentMetadata.algorithmSuite();
+ final EncryptedDataKey originalEncryptedDataKey = contentMetadata.encryptedDataKey();
+ final Map<String, String> currentKeyringMaterialsDescription = contentMetadata.encryptedDataKeyMatDescOrContext();
+ final byte[] iv = contentMetadata.contentIv();
+
+ //Decrypt the data key using the current keyring
+ DecryptionMaterials decryptedMaterials = this._cryptoMaterialsManager.decryptMaterials(
+ DecryptMaterialsRequest.builder()
+ .algorithmSuite(algorithmSuite)
+ .encryptedDataKeys(Collections.singletonList(originalEncryptedDataKey))
+ .s3Request(request)
+ .build()
+ );
+
+ final byte[] plaintextDataKey = decryptedMaterials.plaintextDataKey();
+
+ //Prepare encryption materials with the decrypted data key
+ EncryptionMaterials encryptionMaterials = EncryptionMaterials.builder()
+ .algorithmSuite(algorithmSuite)
+ .plaintextDataKey(plaintextDataKey)
+ .s3Request(request)
+ .build();
+
+ //Re-encrypt the data key with the new keyring while preserving other cryptographic parameters
+ RawKeyring newKeyring = reEncryptInstructionFileRequest.newKeyring();
+ EncryptionMaterials encryptedMaterials = newKeyring.onEncrypt(encryptionMaterials);
+
+ final Map<String, String> newMaterialsDescription = encryptedMaterials.materialsDescription().getMaterialsDescription();
+ //Validate that the new keyring has different materials description than the old keyring
+ if (newMaterialsDescription.equals(currentKeyringMaterialsDescription)) {
+ throw new S3EncryptionClientException("New keyring must have new materials description!");
+ }
+
+ //Create or update instruction file with the re-encrypted metadata while preserving IV
+ ContentMetadataEncodingStrategy encodeStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig);
+ encodeStrategy.encodeMetadata(encryptedMaterials, iv, PutObjectRequest.builder()
+ .bucket(reEncryptInstructionFileRequest.bucket())
+ .key(reEncryptInstructionFileRequest.key())
+ .build(), reEncryptInstructionFileRequest.instructionFileSuffix());
+
+ return new ReEncryptInstructionFileResponse(reEncryptInstructionFileRequest.bucket(),
+ reEncryptInstructionFileRequest.key(), reEncryptInstructionFileRequest.instructionFileSuffix());
+
+ }
+
  /**
  * See {@link S3EncryptionClient#putObject(PutObjectRequest, RequestBody)}.
  * <p>
@@ -380,7 +477,7 @@ public DeleteObjectResponse deleteObject(DeleteObjectRequest deleteObjectRequest
  // Delete the object
  DeleteObjectResponse deleteObjectResponse = _wrappedAsyncClient.deleteObject(actualRequest).join();
  // If Instruction file exists, delete the instruction file as well.
- String instructionObjectKey = deleteObjectRequest.key() + INSTRUCTION_FILE_SUFFIX;
+ String instructionObjectKey = deleteObjectRequest.key() + DEFAULT_INSTRUCTION_FILE_SUFFIX;
  _wrappedAsyncClient.deleteObject(builder -> builder
  .overrideConfiguration(API_NAME_INTERCEPTOR)
  .bucket(deleteObjectRequest.bucket())

src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java

@@ -15,7 +15,7 @@
  */
 public class S3EncryptionClientUtilities {
 
- public static final String INSTRUCTION_FILE_SUFFIX = ".instruction";
+ public static final String DEFAULT_INSTRUCTION_FILE_SUFFIX = ".instruction";
  public static final long MIN_ALLOWED_BUFFER_SIZE_BYTES = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF.cipherBlockSizeBytes();
  public static final long MAX_ALLOWED_BUFFER_SIZE_BYTES = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF.cipherMaxContentLengthBytes();
 
@@ -32,7 +32,7 @@ public class S3EncryptionClientUtilities {
  */
  static List<ObjectIdentifier> instructionFileKeysToDelete(final DeleteObjectsRequest request) {
  return request.delete().objects().stream()
- .map(o -> o.toBuilder().key(o.key() + INSTRUCTION_FILE_SUFFIX).build())
+ .map(o -> o.toBuilder().key(o.key() + DEFAULT_INSTRUCTION_FILE_SUFFIX).build())
  .collect(Collectors.toList());
  }
 }

src/main/java/software/amazon/encryption/s3/S3EncryptionClientUtilities.java

@@ -64,7 +64,7 @@ public String encryptedDataKeyAlgorithm() {
  */
  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "False positive; underlying"
  + " implementation is immutable")
- public Map<String, String> encryptedDataKeyContext() {
+ public Map<String, String> encryptedDataKeyMatDescOrContext() {
  return _encryptionContextOrMatDesc;
  }
 

src/main/java/software/amazon/encryption/s3/internal/ContentMetadata.java

@@ -9,6 +9,7 @@
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.encryption.s3.S3EncryptionClient;
 import software.amazon.encryption.s3.S3EncryptionClientException;
 import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
 import software.amazon.encryption.s3.materials.EncryptedDataKey;
@@ -24,7 +25,7 @@
 import java.util.Map;
 import java.util.concurrent.CompletionException;
 
-import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX;
+import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_INSTRUCTION_FILE_SUFFIX;
 
 public class ContentMetadataDecodingStrategy {
 
@@ -224,9 +225,13 @@ private ContentMetadata decodeFromObjectMetadata(GetObjectRequest request, GetOb
  }
 
  private ContentMetadata decodeFromInstructionFile(GetObjectRequest request, GetObjectResponse response) {
+ String instructionFileSuffix = request.overrideConfiguration()
+ .flatMap(config -> config.executionAttributes().getOptionalAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX))
+ .orElse(DEFAULT_INSTRUCTION_FILE_SUFFIX);
+
  GetObjectRequest instructionGetObjectRequest = GetObjectRequest.builder()
  .bucket(request.bucket())
- .key(request.key() + INSTRUCTION_FILE_SUFFIX)
+ .key(request.key() + instructionFileSuffix)
  .build();
 
  ResponseInputStream<GetObjectResponse> instruction;

src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java

@@ -14,6 +14,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_INSTRUCTION_FILE_SUFFIX;
+
 public class ContentMetadataEncodingStrategy {
 
  private static final Base64.Encoder ENCODER = Base64.getEncoder();
@@ -24,16 +26,20 @@ public ContentMetadataEncodingStrategy(InstructionFileConfig instructionFileConf
  }
 
  public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) {
+ return encodeMetadata(materials, iv, putObjectRequest, DEFAULT_INSTRUCTION_FILE_SUFFIX);
+ }
+
+ public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest, String instructionFileSuffix) {
  if (_instructionFileConfig.isInstructionFilePutEnabled()) {
  final String metadataString = metadataToString(materials, iv);
- _instructionFileConfig.putInstructionFile(putObjectRequest, metadataString);
+ _instructionFileConfig.putInstructionFile(putObjectRequest, metadataString, instructionFileSuffix);
  // the original request object is returned as-is
  return putObjectRequest;
  } else {
  Map<String, String> newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv);
  return putObjectRequest.toBuilder()
-  .metadata(newMetadata)
-  .build();
+ .metadata(newMetadata)
+ .build();
  }
  }
 
@@ -51,6 +57,7 @@ public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials
  .build();
  }
  }
+
  private String metadataToString(EncryptionMaterials materials, byte[] iv) {
  // this is just the metadata map serialized as JSON
  // so first get the Map

src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java

@@ -85,7 +85,7 @@ private DecryptionMaterials prepareMaterialsFromRequest(final GetObjectRequest g
  .s3Request(getObjectRequest)
  .algorithmSuite(algorithmSuite)
  .encryptedDataKeys(encryptedDataKeys)
- .encryptionContext(contentMetadata.encryptedDataKeyContext())
+ .encryptionContext(contentMetadata.encryptedDataKeyMatDescOrContext())
  .ciphertextLength(getObjectResponse.contentLength())
  .contentRange(getObjectRequest.range())
  .build();

src/main/java/software/amazon/encryption/s3/internal/GetEncryptedObjectPipeline.java

@@ -15,7 +15,8 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX;
+
+import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_INSTRUCTION_FILE_SUFFIX;
 import static software.amazon.encryption.s3.internal.MetadataKeyConstants.INSTRUCTION_FILE;
 
 /**
@@ -49,6 +50,10 @@ boolean isInstructionFilePutEnabled() {
  }
 
  PutObjectResponse putInstructionFile(PutObjectRequest request, String instructionFileContent) {
+ return putInstructionFile(request, instructionFileContent, DEFAULT_INSTRUCTION_FILE_SUFFIX);
+ }
+
+ PutObjectResponse putInstructionFile(PutObjectRequest request, String instructionFileContent, String instructionFileSuffix) {
  // This shouldn't happen in practice because the metadata strategy will evaluate
  // if instruction file Puts are enabled before calling this method; check again anyway for robustness
  if (!_enableInstructionFilePut) {
@@ -60,12 +65,11 @@ PutObjectResponse putInstructionFile(PutObjectRequest request, String instructio
  // It contains a key with no value identifying it as an instruction file
  instFileMetadata.put(INSTRUCTION_FILE, "");
 
- // In a future release, non-default suffixes will be supported.
  // Use toBuilder to keep all other fields the same as the actual request
  final PutObjectRequest instPutRequest = request.toBuilder()
-  .key(request.key() + INSTRUCTION_FILE_SUFFIX)
-  .metadata(instFileMetadata)
-  .build();
+ .key(request.key() + instructionFileSuffix)
+ .metadata(instFileMetadata)
+ .build();
  switch (_clientType) {
  case SYNCHRONOUS:
  return _s3Client.putObject(instPutRequest, RequestBody.fromString(instructionFileContent));