add s3 checksum algorithm

Signed-off-by: jorgee <jorge.ejarque@seqera.io>

Author: jorgee

docs/reference/config.md

@@ -101,6 +101,7 @@ aws {
  connectionTimeout = 10000
  uploadStorageClass = 'INTELLIGENT_TIERING'
  storageEncryption = 'AES256'
+ checksumAlgorithm = 'SHA256'
  }
  batch {
  cliPath = '/home/ec2-user/miniconda/bin/aws'
@@ -194,6 +195,9 @@ The following settings are available:
 `aws.client.anonymous`
 : Allow the access of public S3 buckets without the need to provide AWS credentials (default: `false`). Any service that does not accept unsigned requests will return a service access error.
 
+`aws.client.checksumAlgorithm`
+: The S3 checksum algorithm to be used when saving objects on S3. Can be one of `CRC32`, `CRC32C`, `SHA1`, `SHA256` or `CRC64NVME`.
+
 `aws.client.s3Acl`
 : Allow the setting of predefined bucket permissions, also known as *canned ACL*. Permitted values are `Private`, `PublicRead`, `PublicReadWrite`, `AuthenticatedRead`, `LogDeliveryWrite`, `BucketOwnerRead`, `BucketOwnerFullControl`, and `AwsExecRead` (default: none). See [Amazon docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) for details.
 

modules/nf-lang/src/main/java/nextflow/config/scopes/AwsClientConfig.java

@@ -29,6 +29,12 @@ public class AwsClientConfig implements ConfigScope {
  """)
  public boolean anonymous;
 
+ @ConfigOption
+ @Description("""
+ The S3 checksum algorithm to be used when saving objects on S3. Can be one of `CRC32`, `CRC32C`, `SHA1`, `SHA256` or `CRC64NVME`.
+ """)
+ public String checksumAlgorithm;
+
  @ConfigOption
  @Description("""
  Specify predefined bucket permissions, also known as *canned ACL*. Can be one of `Private`, `PublicRead`, `PublicReadWrite`, `AuthenticatedRead`, `LogDeliveryWrite`, `BucketOwnerRead`, `BucketOwnerFullControl`, or `AwsExecRead`.

plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy

@@ -370,13 +370,17 @@ class AwsBatchExecutor extends Executor implements ExtensionPoint, TaskArrayExec
  final debug = opts.debug ? ' --debug' : ''
  final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : ''
  final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : ''
+ final checksum = opts.checksumAlgorithm ? " --checksum-algorithm $opts.checksumAlgorithm" : ''
  final requesterPays = opts.requesterPays ? ' --request-payer requester' : ''
- final aws = "$cli s3 cp --only-show-errors${sse}${kms}${debug}${requesterPays}"
+ final aws = "$cli s3 cp --only-show-errors${sse}${kms}${checksum}${debug}${requesterPays}"
  final cmd = "trap \"{ ret=\$?; $aws ${TaskRun.CMD_LOG} ${workDir}/${TaskRun.CMD_LOG}||true; exit \$ret; }\" EXIT; $aws ${workDir}/${TaskRun.CMD_RUN} - | bash 2>&1 | tee ${TaskRun.CMD_LOG}"
  return cmd
  }
 
  static String s5Cmd(String workDir, AwsOptions opts) {
+ if( opts.checksumAlgorithm ){
+ log.warn1("Checksum Algorithm is not supported by `s5cmd` command. This option will be ignored in command line operations.")
+ }
  final cli = opts.getS5cmdPath()
  final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : ''
  final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : ''

plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy

@@ -116,6 +116,10 @@ class AwsOptions implements CloudTransferOptions {
  return awsConfig.s3Config.getStorageClass()
  }
 
+ String getChecksumAlgorithm(){
+ return awsConfig.s3Config.getChecksumAlgorithm()
+ }
+
  String getStorageEncryption() {
  return awsConfig.s3Config.getStorageEncryption()
  }

plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy

@@ -17,6 +17,8 @@
 
 package nextflow.cloud.aws.config
 
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
+
 import static nextflow.cloud.aws.util.AwsHelper.*
 
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL
@@ -41,6 +43,8 @@ class AwsS3Config {
 
  private String storageKmsKeyId
 
+ private String checksumAlgorithm
+
  private Boolean debug
 
  private ObjectCannedACL s3Acl
@@ -59,6 +63,7 @@ class AwsS3Config {
  this.storageClass = parseStorageClass((opts.storageClass ?: opts.uploadStorageClass) as String) // 'uploadStorageClass' is kept for legacy purposes
  this.storageEncryption = parseStorageEncryption(opts.storageEncryption as String)
  this.storageKmsKeyId = opts.storageKmsKeyId
+ this.checksumAlgorithm = parseChecksumAlgorithm(opts.checksumAlgorithm as String)
  this.pathStyleAccess = opts.s3PathStyleAccess as Boolean
  this.anonymous = opts.anonymous as Boolean
  this.s3Acl = parseS3Acl(opts.s3Acl as String)
@@ -85,6 +90,15 @@ class AwsS3Config {
  return null
  }
 
+ private String parseChecksumAlgorithm(String value) {
+ if (value) {
+ if( value in ChecksumAlgorithm.knownValues()*.toString() )
+ return value
+ log.warn "Unsupported AWS checksum algorithm: $value"
+ }
+ return null
+ }
+
  // ==== getters =====
  String getEndpoint() {
  return endpoint
@@ -102,6 +116,10 @@ class AwsS3Config {
  return storageKmsKeyId
  }
 
+ String getChecksumAlgorithm() {
+ return checksumAlgorithm
+ }
+
  Boolean getDebug() {
  return debug
  }

plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java

@@ -76,6 +76,8 @@ public class S3Client {
 
 private boolean global;
 
+ private ChecksumAlgorithm checksumAlgorithm;
+
 public S3Client(AwsClientFactory factory, Properties props, boolean global) {
 S3SyncClientConfiguration clientConfig = S3SyncClientConfiguration.create(props);
 this.factory = factory;
@@ -155,6 +157,9 @@ private PutObjectRequest preparePutObjectRequest(PutObjectRequest.Builder reqBui
 if( storageEncryption!=null ) {
 reqBuilder.serverSideEncryption(storageEncryption);
 }
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
 if( contentType!=null ) {
 reqBuilder.contentType(contentType);
 }
@@ -183,6 +188,9 @@ public PutObjectResponse putObject(String bucket, String keyName, InputStream in
 if( storageEncryption!=null ) {
 reqBuilder.serverSideEncryption(storageEncryption);
 }
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
 if( contentType!=null ) {
 reqBuilder.contentType(contentType);
 }
@@ -217,6 +225,9 @@ public void copyObject(CopyObjectRequest.Builder reqBuilder, List<Tag> tags, Str
 if( kmsKeyId !=null ) {
 reqBuilder.ssekmsKeyId(kmsKeyId);
 }
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
 if( contentType!=null ) {
 reqBuilder.metadataDirective(MetadataDirective.REPLACE);
 reqBuilder.contentType(contentType);
@@ -281,6 +292,13 @@ public void setTransferManagerThreads(String value) {
 }
 }
 
+ public void setChecksumAlgorithm(String alg){
+ if( alg == null )
+return;
+this.checksumAlgorithm = ChecksumAlgorithm.fromValue(alg);
+log.debug("Setting S3 ChecksumAlgorithm={}", alg);
+ }
+
 public ObjectCannedACL getCannedAcl() {
 return cannedAcl;
 }
@@ -340,6 +358,9 @@ public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSiz
 if( kmsKeyId != null ) {
 reqBuilder.ssekmsKeyId(kmsKeyId);
 }
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
 
 if( tags != null && tags.size()>0 ) {
 reqBuilder.tagging( Tagging.builder().tagSet(tags).build() );
@@ -548,6 +569,9 @@ private PutObjectRequest.Builder updateBuilder(PutObjectRequest.Builder porBuild
 porBuilder.serverSideEncryption(storageEncryption);
 if( kmsKeyId != null )
 porBuilder.ssekmsKeyId(kmsKeyId);
+ if( checksumAlgorithm != null ) {
+ porBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
 if( tags != null && !tags.isEmpty() )
 porBuilder.tagging(Tagging.builder().tagSet(tags).build());
 return porBuilder;
@@ -593,6 +617,9 @@ public void copyFile(CopyObjectRequest.Builder reqBuilder, List<Tag> tags, Strin
 if( kmsKeyId !=null ) {
  reqBuilder.ssekmsKeyId(kmsKeyId);
 }
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
 if( contentType!=null ) {
 reqBuilder.metadataDirective(MetadataDirective.REPLACE);
  reqBuilder.contentType(contentType);

plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java

@@ -340,6 +340,7 @@ private S3OutputStream createUploaderOutputStream( S3Path fileToUpload ) {
 .setCannedAcl(s3.getCannedAcl())
 .setStorageClass(storageClass)
 .setStorageEncryption(props.getProperty("storage_encryption"))
+ .setChecksumAlgorithm(props.getProperty("checksum_algorithm"))
 .setKmsKeyId(props.getProperty("storage_kms_key_id"))
 .setContentType(fileToUpload.getContentType())
 .setTags(fileToUpload.getTagsList());
@@ -742,6 +743,7 @@ protected S3FileSystem createFileSystem(URI uri, AwsConfig awsConfig) {
 // set the client acl
 client.setCannedAcl(getProp(props, "s_3_acl", "s3_acl", "s3acl", "s3Acl"));
 client.setStorageEncryption(props.getProperty("storage_encryption"));
+ client.setChecksumAlgorithm(props.getProperty("checksum_algorithm"));
 client.setKmsKeyId(props.getProperty("storage_kms_key_id"));
 client.setTransferManagerThreads(props.getProperty("transfer_manager_threads"));
  client.setRequesterPaysEnabled(props.getProperty("requester_pays"));

plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java

@@ -87,6 +87,8 @@ public final class S3OutputStream extends OutputStream {
 
  private String contentType;
 
+ private ChecksumAlgorithm checksumAlgorithm;
+
  /**
  * Indicates if the stream has been closed.
  */
@@ -201,6 +203,12 @@ public S3OutputStream setKmsKeyId(String kmsKeyId) {
  return this;
  }
 
+ public S3OutputStream setChecksumAlgorithm(String checksumAlgorithm){
+ if( checksumAlgorithm !=null )
+ this.checksumAlgorithm = ChecksumAlgorithm.fromValue(checksumAlgorithm);
+ return this;
+ }
+
  public S3OutputStream setContentType(String type) {
  this.contentType = type;
  return this;
@@ -428,6 +436,10 @@ private CreateMultipartUploadResponse initiateMultipartUpload() throws IOExcepti
  reqBuilder.serverSideEncryption(storageEncryption);
  }
 
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
+
  if( contentType != null ) {
  reqBuilder.contentType(contentType);
  }
@@ -614,6 +626,10 @@ private void putObject(final InputStream content, final long contentLength, byte
  reqBuilder.serverSideEncryption( storageEncryption );
  }
 
+ if( checksumAlgorithm != null ) {
+ reqBuilder.checksumAlgorithm(checksumAlgorithm);
+ }
+
  if( contentType != null ) {
  reqBuilder.contentType(contentType);
  }

plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy

@@ -38,6 +38,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
  private String s5cmdPath
  private String acl = ''
  private String requesterPays = ''
+ private String checksumAlgorithm = ''
 
  S3BashLib withCliPath(String cliPath) {
  if( cliPath )
@@ -61,6 +62,11 @@ class S3BashLib extends BashFunLib<S3BashLib> {
  this.storageClass = value
  return this
  }
+ S3BashLib withChecksumAlgorithm(String value) {
+ if( value )
+ this.checksumAlgorithm = value ? "--checksum-algorithm $value " : ''
+ return this
+ }
 
  S3BashLib withStorageEncryption(String value) {
  if( value )
@@ -112,11 +118,11 @@ class S3BashLib extends BashFunLib<S3BashLib> {
  local name=\$1
  local s3path=\$2
  if [[ "\$name" == - ]]; then
- $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass - "\$s3path"
+ $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass - "\$s3path"
  elif [[ -d "\$name" ]]; then
- $cli s3 cp --only-show-errors --recursive ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
+ $cli s3 cp --only-show-errors --recursive ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
  else
- $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
+ $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
  fi
  }
  
@@ -187,6 +193,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
  .withMaxTransferAttempts( opts.maxTransferAttempts )
  .withCliPath( opts.awsCli )
  .withStorageClass(opts.storageClass )
+ .withChecksumAlgorithm( opts.checksumAlgorithm )
  .withStorageEncryption( opts.storageEncryption )
  .withStorageKmsKeyId( opts.storageKmsKeyId )
  .withRetryMode( opts.retryMode )

plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy

@@ -126,6 +126,7 @@ class AwsBatchFileCopyStrategyTest extends Specification {
  1 * opts.getAwsCli() >> 'aws'
  1 * opts.getStorageClass() >> null
  1 * opts.getStorageEncryption() >> null
+ 1 * opts.getChecksumAlgorithm() >> null
 
  script == '''\
  # bash helper functions
@@ -217,6 +218,7 @@ class AwsBatchFileCopyStrategyTest extends Specification {
  1 * opts.getAwsCli() >> '/foo/aws'
  1 * opts.getStorageClass() >> 'STANDARD_IA'
  1 * opts.getStorageEncryption() >> 'AES256'
+ 1 * opts.getChecksumAlgorithm() >> 'SHA256'
 
  script == '''\
  # bash helper functions
@@ -281,11 +283,11 @@ class AwsBatchFileCopyStrategyTest extends Specification {
  local name=$1
  local s3path=$2
  if [[ "$name" == - ]]; then
- /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA - "$s3path"
+ /foo/aws s3 cp --only-show-errors --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA - "$s3path"
  elif [[ -d "$name" ]]; then
- /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name"
+ /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA "$name" "$s3path/$name"
  else
- /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name"
+ /foo/aws s3 cp --only-show-errors --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA "$name" "$s3path/$name"
  fi
  }