feat: Add EMR Serverless runtimeConf and recent EMR release versions (#313)

* Add EMR Serverless runtime and recent EMR release versions --------- Co-authored-by: lmouhib <mouhib.lotfi@gmail.com>

Author: dacort

framework/API.md

@@ -6,6 +6,10 @@
  */
 
 export enum EmrRuntimeVersion {
+ V7_0 = 'emr-7.0.0',
+ V6_15 = 'emr-6.15.0',
+ V6_14 = 'emr-6.14.0',
+ V6_13 = 'emr-6.13.0',
  V6_12 = 'emr-6.12.0',
  V6_11_1 = 'emr-6.11.1',
  V6_11 = 'emr-6.11.0',
@@ -29,6 +33,10 @@ const EMR_EKS_IMAGE_URL = 'public.ecr.aws/emr-on-eks/spark/';
  * The list of supported Spark images to use in the SparkCICDPipeline.
  */
 export enum SparkImage {
+ EMR_7_0 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V7_0 + ':latest',
+ EMR_6_15 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V6_15 + ':latest',
+ EMR_6_14 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V6_14 + ':latest',
+ EMR_6_13 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V6_13 + ':latest',
  EMR_6_12 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V6_12 + ':latest',
  EMR_6_11 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V6_11 + ':latest',
  EMR_6_10 = EMR_EKS_IMAGE_URL + EmrRuntimeVersion.V6_10 + ':latest',

framework/src/processing/lib/emr-releases.ts

@@ -50,6 +50,10 @@ export interface SparkEmrServerlessRuntimeProps {
  * The image configuration.
  */
  readonly imageConfiguration?: IResolvable | aws_emrserverless.CfnApplication.ImageConfigurationInputProperty;
+ /**
+ * The runtime and monitoring configurations to used as defaults for all of the job runs of this application.
+ */
+ readonly runtimeConfiguration?: IResolvable | aws_emrserverless.CfnApplication.ConfigurationObjectProperty[];
  /**
  * The configuration for an application to automatically stop after a certain amount of time being idle.
  */

framework/src/processing/lib/spark-runtime/emr-serverless/spark-emr-runtime-serverless-props.ts

@@ -14,6 +14,30 @@ import { Match, Template } from 'aws-cdk-lib/assertions';
 import { AccountRootPrincipal, PolicyDocument, PolicyStatement, Role } from 'aws-cdk-lib/aws-iam';
 import { EmrRuntimeVersion, SparkEmrServerlessRuntime } from '../../../src/processing';
 
+describe('Create an EMR Serverless Application with runtime configuration', () => {
+
+ const app = new App();
+ const stack = new Stack(app, 'Stack');
+
+ new SparkEmrServerlessRuntime(stack, 'SparkRuntimeServerlessStack', {
+ releaseLabel: EmrRuntimeVersion.V6_15,
+ name: 'spark-serverless-demo',
+ runtimeConfiguration: [{ classification: 'spark-defaults', properties: { 'spark.driver.cores': '4' } }],
+ });
+
+ const template = Template.fromStack(stack);
+
+ test('EMR Serverless application created with default config', () => {
+ template.hasResource('AWS::EMRServerless::Application',
+ Match.objectLike({
+ Properties: {
+ RuntimeConfiguration: [{ Classification: 'spark-defaults', Properties: { 'spark.driver.cores': '4' } }],
+ },
+ }),
+ );
+ });
+});
+
 
 describe('Create an EMR Serverless Application for Spark and grant access', () => {