publishing v2.2.8

Author: daboncanplay

ASP 0 - Course Agenda.py

@@ -10,6 +10,8 @@
 # MAGIC %md
 # MAGIC # Apache Spark Programming with Databricks
 # MAGIC ## Course Agenda
+# MAGIC 
+# MAGIC Taught over 4 half-days or 2 full-days.
 
 # COMMAND ----------
 
@@ -26,12 +28,15 @@
 # MAGIC * [ASP 2.2 - Reader & Writer]($./ASP 2 - Spark Core/ASP 2.2 - Reader & Writer)
 # MAGIC * [ASP 2.2L - Ingesting Data Lab]($./ASP 2 - Spark Core/ASP 2.2L - Ingesting Data Lab)
 # MAGIC * [ASP 2.3 - DataFrame & Column]($./ASP 2 - Spark Core/ASP 2.3 - DataFrame & Column)
-# MAGIC * [ASP 2.3L - Purchase Revenues Lab]($./ASP 2 - Spark Core/ASP 2.3L - Purchase Revenues Lab)
 
 # COMMAND ----------
 
 # MAGIC %md
 # MAGIC # Day 2
+# MAGIC 
+# MAGIC ## Spark Core (Continued)
+# MAGIC * [ASP 2.3L - Purchase Revenues Lab]($./ASP 2 - Spark Core/ASP 2.3L - Purchase Revenues Lab)
+# MAGIC 
 # MAGIC ## Functions
 # MAGIC * [ASP 3.1 - Aggregation]($./ASP 3 - Functions/ASP 3.1 - Aggregation)
 # MAGIC * [ASP 3.1L - Revenue by Traffic Lab]($./ASP 3 - Functions/ASP 3.1L - Revenue by Traffic Lab)
@@ -40,15 +45,17 @@
 # MAGIC * [ASP 3.3 - Complex Types]($./ASP 3 - Functions/ASP 3.3 - Complex Types)
 # MAGIC * [ASP 3.3L - Users]($./ASP 3 - Functions/ASP 3.3L - Users)
 # MAGIC * [ASP 3.4 - Additional Functions]($./ASP 3 - Functions/ASP 3.4 - Additional Functions)
-# MAGIC * [ASP 3.4L - Abandoned Carts Lab]($./ASP 3 - Functions/ASP 3.4L - Abandoned Carts Lab)
-# MAGIC * [ASP 3.5 - UDFs]($./ASP 3 - Functions/ASP 3.5 - UDFs)
-# MAGIC * [ASP 3.5L - Sort Day Lab]($./ASP 3 - Functions/ASP 3.5L - Sort Day Lab)
 # MAGIC 
 
 # COMMAND ----------
 
 # MAGIC %md
 # MAGIC # Day 3
+# MAGIC ## Functions (Continued)
+# MAGIC * [ASP 3.4L - Abandoned Carts Lab]($./ASP 3 - Functions/ASP 3.4L - Abandoned Carts Lab)
+# MAGIC * [ASP 3.5 - UDFs]($./ASP 3 - Functions/ASP 3.5 - UDFs)
+# MAGIC * [ASP 3.5L - Sort Day Lab]($./ASP 3 - Functions/ASP 3.5L - Sort Day Lab)
+# MAGIC 
 # MAGIC ## Performance
 # MAGIC * [ASP 4.1 - Query Optimization]($./ASP 4 - Performance/ASP 4.1 - Query Optimization)
 # MAGIC * [ASP 4.2 - Partitioning]($./ASP 4 - Performance/ASP 4.2 - Partitioning)

ASP 1 - Introductions/ASP 1.1 - Databricks Platform.py

@@ -130,7 +130,7 @@
 # MAGIC | Horizontal Rule | `---` |
 # MAGIC | Code | ``` `code` ``` |
 # MAGIC | Link | `[text](https://www.example.com)` |
-# MAGIC | Image | `[alt text](image.jpg)`|
+# MAGIC | Image | `![alt text](image.jpg)`|
 # MAGIC | Ordered List | `1. First items` <br> `2. Second Item` <br> `3. Third Item` |
 # MAGIC | Unordered List | `- First items` <br> `- Second Item` <br> `- Third Item` |
 # MAGIC | Code Block | ```` ``` ```` <br> `code block` <br> ```` ``` ````|
@@ -265,7 +265,7 @@
 
 # COMMAND ----------
 
-print(f"Database Name: {DA.db_name}")
+print(f"Database Name: {DA.schema_name}")
 
 # COMMAND ----------
 
@@ -274,7 +274,7 @@
 # COMMAND ----------
 
 # MAGIC %sql
-# MAGIC SHOW TABLES IN ${DA.db_name}
+# MAGIC SHOW TABLES IN ${DA.schema_name}
 
 # COMMAND ----------
 

ASP 2 - Spark Core/ASP 2.2 - Reader & Writer.py

@@ -307,7 +307,7 @@
 
 # COMMAND ----------
 
-print(f"Database Name: {DA.db_name}")
+print(f"Database Name: {DA.schema_name}")
 
 # COMMAND ----------
 
@@ -316,7 +316,7 @@
 # COMMAND ----------
 
 # MAGIC %sql
-# MAGIC SHOW TABLES IN ${DA.db_name}
+# MAGIC SHOW TABLES IN ${DA.schema_name}
 
 # COMMAND ----------
 

Includes/Classroom-Setup-5.1a.py

@@ -3,14 +3,14 @@
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_1_1(self, df):
  suite = DA.tests.new("5.1a-1.1")
 
- suite.test_true(df.isStreaming, description="The query is streaming")
+ suite.test_true(actual_value=lambda: df.isStreaming, description="The query is streaming")
 
  columns = ['order_id', 'email', 'transaction_timestamp', 'total_item_quantity', 'purchase_revenue_in_usd', 'unique_items', 'items']
- suite.test_sequence(actual_value=df.columns, 
+ suite.test_sequence(actual_value=lambda: df.columns, 
  expected_value=columns,
  test_column_order=False,
  description=f"DataFrame contains all {len(columns)} columns",
@@ -21,44 +21,53 @@ def validate_1_1(self, df):
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
-def validate_2_1(self, schema:StructType):
- 
+@ValidationHelper.monkey_patch
+def validate_2_1(self, schema: StructType):
+
  suite = DA.tests.new("5.1a-2.1")
- 
- suite.test_equals(actual_value=type(schema), expected_value=StructType, description="Schema is of type StructType", hint="Found [[ACTUAL_VALUE]]")
- 
- suite.test_length(schema.fieldNames(), 7, description="Schema contians seven fields", hint="Found [[LEN_ACTUAL_VALUE]]: [[ACTUAL_VALUE]]")
-
- suite.test_struct_field(schema, "order_id", "LongType", None)
- suite.test_struct_field(schema, "email", "StringType", None) 
- suite.test_struct_field(schema, "transaction_timestamp", "LongType", None)
- suite.test_struct_field(schema, "total_item_quantity", "LongType", None)
- suite.test_struct_field(schema, "purchase_revenue_in_usd", "DoubleType", None)
- suite.test_struct_field(schema, "unique_items", "LongType", None)
- suite.test_struct_field(schema, "items", "StructType", None)
- 
+
+ suite.test_equals(
+ actual_value=lambda: type(schema),
+ expected_value=StructType,
+ description="Schema is of type StructType",
+ hint="Found [[ACTUAL_VALUE]]",
+ )
+
+ suite.test_length(
+ lambda: schema.fieldNames(),
+ expected_length=7,
+ description="Schema contians seven fields",
+ hint="Found [[LEN_ACTUAL_VALUE]]: [[ACTUAL_VALUE]]",
+ )
+
+ suite.test_schema_field(lambda: schema, "order_id", "LongType", None)
+ suite.test_schema_field(lambda: schema, "email", "StringType", None)
+ suite.test_schema_field(lambda: schema, "transaction_timestamp", "LongType", None)
+ suite.test_schema_field(lambda: schema, "total_item_quantity", "LongType", None)
+ suite.test_schema_field(lambda: schema, "purchase_revenue_in_usd", "DoubleType", None)
+ suite.test_schema_field(lambda: schema, "unique_items", "LongType", None)
+ suite.test_schema_field(lambda: schema, "items", "StructType", None)
+
  suite.display_results()
  assert suite.passed, "One or more tests failed."
 
-
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_3_1(self, query):
  suite = DA.tests.new("5.1a-3.1")
 
- suite.test_true(query.isActive, description="The query is active")
+ suite.test_true(actual_value=lambda: query.isActive, description="The query is active")
 
- suite.test_equals(coupon_sales_query.lastProgress["name"], "coupon_sales",
+ suite.test_equals(lambda: coupon_sales_query.lastProgress["name"], "coupon_sales",
  description="The query name is \"coupon_sales\".")
 
  coupons_output_path = f"{DA.paths.working_dir}/coupon-sales/output"
- suite.test(actual_value=None, test_function=lambda: len(dbutils.fs.ls(coupons_output_path)) > 0,
+ suite.test(actual_value=lambda: None, test_function=lambda: len(dbutils.fs.ls(coupons_output_path)) > 0,
  description=f"Found at least one file in .../coupon-sales/output")
 
  coupons_checkpoint_path = f"{DA.paths.checkpoints}/coupon-sales"
- suite.test(actual_value=None, test_function=lambda: len(dbutils.fs.ls(coupons_checkpoint_path)) > 0,
+ suite.test(actual_value=lambda: None, test_function=lambda: len(dbutils.fs.ls(coupons_checkpoint_path)) > 0,
  description=f"Found at least one file in .../coupon-sales")
 
  suite.display_results()
@@ -67,43 +76,43 @@ def validate_3_1(self, query):
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_4_1(self, query_id, query_status):
  suite = DA.tests.new("5.1a-4.1")
 
- suite.test_sequence(actual_value=query_status.keys(),
+ suite.test_sequence(actual_value=lambda: query_status.keys(),
  expected_value=["message", "isDataAvailable", "isTriggerActive"],
  test_column_order=False,
  description="Valid status value.")
 
- suite.test_equals(type(query_id), str, description="Valid query_id value.")
+ suite.test_equals(lambda: type(query_id), str, description="Valid query_id value.")
 
  suite.display_results()
  assert suite.passed, "One or more tests failed."
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_5_1(self, query):
  suite = DA.tests.new("5.1a-5.1")
 
- suite.test_false(query.isActive, description="The query is not active")
+ suite.test_false(actual_value=lambda: query.isActive, description="The query is not active")
 
  suite.display_results()
  assert suite.passed, "One or more tests failed."
 
 
 # COMMAND ----------
 
-DA = DBAcademyHelper(**helper_arguments) # Create the DA object
-DA.reset_environment() # Reset by removing databases and files from other lessons
-DA.init(install_datasets=True, # Initialize, install and validate the datasets
- create_db=True) # Continue initialization, create the user-db
+DA = DBAcademyHelper(course_config, lesson_config)
+DA.reset_lesson()
+DA.init()
+DA.conclude_setup()
 
 DA.paths.sales = f"{DA.paths.datasets}/ecommerce/sales/sales.delta"
 DA.paths.users = f"{DA.paths.datasets}/ecommerce/users/users.delta"
 DA.paths.events = f"{DA.paths.datasets}/ecommerce/events/events.delta"
 DA.paths.products = f"{DA.paths.datasets}/products/products.delta"
 
-DA.conclude_setup() # Conclude setup by advertising environmental changes
+DA.conclude_setup()
 

Includes/Classroom-Setup-5.1b.py

@@ -3,71 +3,71 @@
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_1_1(self, schema):
  suite = DA.tests.new("5.1b-1.1")
 
- suite.test_equals(actual_value=type(schema), expected_value=StructType, description="Schema is of type StructType", hint="Found [[ACTUAL_VALUE]]")
+ suite.test_equals(lambda: type(schema), expected_value=StructType, description="Schema is of type StructType", hint="Found [[ACTUAL_VALUE]]")
 
- suite.test_length(schema.fieldNames(), 12, description="Schema contians 12 field", hint="Found [[LEN_ACTUAL_VALUE]]: [[ACTUAL_VALUE]]")
+ suite.test_length(lambda: schema.fieldNames(), 12, description="Schema contians 12 field", hint="Found [[LEN_ACTUAL_VALUE]]: [[ACTUAL_VALUE]]")
 
- suite.test_struct_field(schema, "device", "StringType", None)
- suite.test_struct_field(schema, "ecommerce", "StructType", None)
- suite.test_struct_field(schema, "event_name", "StringType", None)
- suite.test_struct_field(schema, "event_previous_timestamp", "LongType", None)
- suite.test_struct_field(schema, "event_timestamp", "LongType", None)
- suite.test_struct_field(schema, "geo", "StructType", None)
- suite.test_struct_field(schema, "items", "ArrayType", None)
- suite.test_struct_field(schema, "traffic_source", "StringType", None)
- suite.test_struct_field(schema, "user_first_touch_timestamp", "LongType", None)
- suite.test_struct_field(schema, "user_id", "StringType", None)
- suite.test_struct_field(schema, "hour", "IntegerType", None)
- suite.test_struct_field(schema, "createdAt", "TimestampType", None)
+ suite.test_schema_field(lambda: schema, "device", "StringType", None)
+ suite.test_schema_field(lambda: schema, "ecommerce", "StructType", None)
+ suite.test_schema_field(lambda: schema, "event_name", "StringType", None)
+ suite.test_schema_field(lambda: schema, "event_previous_timestamp", "LongType", None)
+ suite.test_schema_field(lambda: schema, "event_timestamp", "LongType", None)
+ suite.test_schema_field(lambda: schema, "geo", "StructType", None)
+ suite.test_schema_field(lambda: schema, "items", "ArrayType", None)
+ suite.test_schema_field(lambda: schema, "traffic_source", "StringType", None)
+ suite.test_schema_field(lambda: schema, "user_first_touch_timestamp", "LongType", None)
+ suite.test_schema_field(lambda: schema, "user_id", "StringType", None)
+ suite.test_schema_field(lambda: schema, "hour", "IntegerType", None)
+ suite.test_schema_field(lambda: schema, "createdAt", "TimestampType", None)
 
  suite.display_results()
  assert suite.passed, "One or more tests failed."
 
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_2_1(self, schema):
  suite = DA.tests.new("5.1b-2.1")
 
- suite.test_equals(actual_value=type(schema), expected_value=StructType, description="Schema is of type StructType", hint="Found [[ACTUAL_VALUE]]")
+ suite.test_equals(lambda: type(schema), expected_value=StructType, description="Schema is of type StructType", hint="Found [[ACTUAL_VALUE]]")
 
- suite.test_length(schema.fieldNames(), 3, description="Schema contians three field", hint="Found [[LEN_ACTUAL_VALUE]]: [[ACTUAL_VALUE]]")
+ suite.test_length(lambda: schema.fieldNames(), 3, description="Schema contians three field", hint="Found [[LEN_ACTUAL_VALUE]]: [[ACTUAL_VALUE]]")
 
- suite.test_struct_field(schema, "traffic_source", "StringType", None)
- suite.test_struct_field(schema, "active_users", "LongType", None)
- suite.test_struct_field(schema, "hour", "IntegerType", None)
+ suite.test_schema_field(lambda: schema, "traffic_source", "StringType", None)
+ suite.test_schema_field(lambda: schema, "active_users", "LongType", None)
+ suite.test_schema_field(lambda: schema, "hour", "IntegerType", None)
 
  suite.display_results()
  assert suite.passed, "One or more tests failed."
 
 
 # COMMAND ----------
 
-@TestHelper.monkey_patch
+@ValidationHelper.monkey_patch
 def validate_4_1(self):
  suite = DA.tests.new("5.1b-4.1")
 
- suite.test_length(spark.streams.active, 0, description="All queries have stopped streaming")
+ suite.test_length(lambda: spark.streams.active, 0, description="All queries have stopped streaming")
 
  suite.display_results()
  assert suite.passed, "One or more tests failed."
 
 # COMMAND ----------
 
-DA = DBAcademyHelper(**helper_arguments) # Create the DA object
-DA.reset_environment() # Reset by removing databases and files from other lessons
-DA.init(install_datasets=True, # Initialize, install and validate the datasets
- create_db=True) # Continue initialization, create the user-db
+DA = DBAcademyHelper(course_config, lesson_config)
+DA.reset_lesson()
+DA.init()
+DA.conclude_setup()
 
 DA.paths.sales = f"{DA.paths.datasets}/ecommerce/sales/sales.delta"
 DA.paths.users = f"{DA.paths.datasets}/ecommerce/users/users.delta"
 DA.paths.events = f"{DA.paths.datasets}/ecommerce/events/events.delta"
 DA.paths.products = f"{DA.paths.datasets}/products/products.delta"
 
-DA.conclude_setup() # Conclude setup by advertising environmental changes
+DA.conclude_setup()