Publishing v2.1.1

Author: SireInsectus

Apache-Spark-Programming-with-Databricks/ASP 0 - Course Agenda.py

@@ -38,6 +38,7 @@
 # MAGIC * [ASP 3.2 - Datetimes]($./ASP 3 - Functions/ASP 3.2 - Datetimes)
 # MAGIC * [ASP 3.2L - Active Users Lab]($./ASP 3 - Functions/ASP 3.2L - Active Users Lab)
 # MAGIC * [ASP 3.3 - Complex Types]($./ASP 3 - Functions/ASP 3.3 - Complex Types)
+# MAGIC * [ASP 3.3L - Users]($./ASP 3 - Functions/ASP 3.3L - Users)
 # MAGIC * [ASP 3.4 - Additional Functions]($./ASP 3 - Functions/ASP 3.4 - Additional Functions)
 # MAGIC * [ASP 3.4L - Abandoned Carts Lab]($./ASP 3 - Functions/ASP 3.4L - Abandoned Carts Lab)
 # MAGIC * [ASP 3.5 - UDFs]($./ASP 3 - Functions/ASP 3.5 - UDFs)

Apache-Spark-Programming-with-Databricks/ASP 1 - Introductions/ASP 1.1 - Databricks Platform.py

@@ -161,7 +161,7 @@
 
 # COMMAND ----------
 
-# MAGIC %md `%fs` is shorthand for the <a href="https://docs.databricks.com/dev-tools/databricks-utils.html" target="_blank">DBUtils</a> module: **`dbutils.fs`**
+# MAGIC %md **`%fs`** is shorthand for the <a href="https://docs.databricks.com/dev-tools/databricks-utils.html" target="_blank">DBUtils</a> module: **`dbutils.fs`**
 
 # COMMAND ----------
 
@@ -219,7 +219,9 @@
 # COMMAND ----------
 
 # MAGIC %sql
-# MAGIC CREATE TABLE IF NOT EXISTS events USING DELTA OPTIONS (path "${c.events_path}");
+# MAGIC CREATE TABLE IF NOT EXISTS events
+# MAGIC USING DELTA
+# MAGIC OPTIONS (path = "${c.events_path}");
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 2 - Spark Core/ASP 2.2 - Reader & Writer.py

@@ -320,7 +320,7 @@
 # MAGIC 
 # MAGIC #### Delta Lake's Key Features
 # MAGIC - ACID transactions
-# MAGIC - Scalable metadata handline
+# MAGIC - Scalable metadata handling
 # MAGIC - Unified streaming and batch processing
 # MAGIC - Time travel (data versioning)
 # MAGIC - Schema enforcement and evolution

Apache-Spark-Programming-with-Databricks/ASP 2 - Spark Core/ASP 2.3 - DataFrame & Column.py

@@ -228,7 +228,7 @@
 # MAGIC %md #### **`dropDuplicates()`**
 # MAGIC Returns a new DataFrame with duplicate rows removed, optionally considering only a subset of columns.
 # MAGIC 
-# MAGIC ##### Alias: `distinct`
+# MAGIC ##### Alias: **`distinct`**
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 3 - Functions/ASP 3.1 - Aggregation.py

@@ -100,7 +100,7 @@
 # MAGIC %md ## Built-In Functions
 # MAGIC In addition to DataFrame and Column transformation methods, there are a ton of helpful functions in Spark's built-in <a href="https://docs.databricks.com/spark/latest/spark-sql/language-manual/sql-ref-functions-builtin.html" target="_blank">SQL functions</a> module.
 # MAGIC 
-# MAGIC In Scala, this is <a href="https://spark.apache.org/docs/latest/api/scala/org/apache/spark/sql/functions$.html" target="_bank">**`org.apache.spark.sql.functions`**</a>, and <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql.html#functions" target="_blank">**`pyspark.sql.functions`**</a> in Python. Functions from this module must be imported into your code.
+# MAGIC In Scala, this is <a href="https://spark.apache.org/docs/latest/api/scala/org/apache/spark/sql/functions$.html" target="_blank">**`org.apache.spark.sql.functions`**</a>, and <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql.html#functions" target="_blank">**`pyspark.sql.functions`**</a> in Python. Functions from this module must be imported into your code.
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 3 - Functions/ASP 3.2 - Datetimes.py

@@ -122,7 +122,7 @@
 # MAGIC %md #### **`year`**
 # MAGIC Extracts the year as an integer from a given date/timestamp/string.
 # MAGIC 
-# MAGIC ##### Similar methods: `month`, `dayofweek`, `minute`, `second`, etc.
+# MAGIC ##### Similar methods: **`month`**, **`dayofweek`**, **`minute`**, **`second`**, etc.
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 3 - Functions/ASP 3.3L - Users.py

@@ -20,15 +20,6 @@
 
 # COMMAND ----------
 
-details_df = (df
- .withColumn("items", explode("items"))
- .select("email", "items.item_name")
- .withColumn("details", split(col("item_name"), " "))
- )
-display(details_df)
-
-# COMMAND ----------
-
 # MAGIC %md ### 1. Extract item details from purchases
 # MAGIC 
 # MAGIC - Explode the **`items`** field in **`df`** with the results replacing the existing **`items`** field

Apache-Spark-Programming-with-Databricks/ASP 3 - Functions/ASP 3.4 - Additional Functions.py

@@ -81,21 +81,21 @@
 # COMMAND ----------
 
 # MAGIC %md ### Joining DataFrames
-# MAGIC The DataFrame <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame.join.html?highlight=join#pyspark.sql.DataFrame.join" target="_blank">**`join`**</a> method joins two DataFrames based on a given join expression. Several different types of joins are supported. For example:
+# MAGIC The DataFrame <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame.join.html?highlight=join#pyspark.sql.DataFrame.join" target="_blank">**`join`**</a> method joins two DataFrames based on a given join expression. 
 # MAGIC 
-# MAGIC ```
-# MAGIC # Inner join based on equal values of a shared column called "name" (i.e., an equi join)
-# MAGIC df1.join(df2, "name")
+# MAGIC Several different types of joins are supported:
 # MAGIC 
-# MAGIC # Inner join based on equal values of the shared columns called "name" and "age"
-# MAGIC df1.join(df2, ["name", "age"])
+# MAGIC Inner join based on equal values of a shared column called "name" (i.e., an equi join)<br/>
+# MAGIC **`df1.join(df2, "name")`**
 # MAGIC 
-# MAGIC # Full outer join based on equal values of a shared column called "name"
-# MAGIC df1.join(df2, "name", "outer")
+# MAGIC Inner join based on equal values of the shared columns called "name" and "age"<br/>
+# MAGIC **`df1.join(df2, ["name", "age"])`**
 # MAGIC 
-# MAGIC # Left outer join based on an explicit column expression
-# MAGIC df1.join(df2, df1["customer_name"] == df2["account_name"], "left_outer")
-# MAGIC ```
+# MAGIC Full outer join based on equal values of a shared column called "name"<br/>
+# MAGIC **`df1.join(df2, "name", "outer")`**
+# MAGIC 
+# MAGIC Left outer join based on an explicit column expression<br/>
+# MAGIC **`df1.join(df2, df1["customer_name"] == df2["account_name"], "left_outer")`**
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 4 - Performance/ASP 4.2 - Partitioning.py

@@ -62,7 +62,7 @@
 
 # COMMAND ----------
 
-# MAGIC %md #### `repartition`
+# MAGIC %md #### **`repartition`**
 # MAGIC Returns a new DataFrame that has exactly **`n`** partitions.
 # MAGIC 
 # MAGIC - Wide transformation
@@ -79,7 +79,7 @@
 
 # COMMAND ----------
 
-# MAGIC %md #### `coalesce`
+# MAGIC %md #### **`coalesce`**
 # MAGIC Returns a new DataFrame that has exactly **`n`** partitions, when fewer partitions are requested.
 # MAGIC 
 # MAGIC If a larger number of partitions is requested, it will stay at the current number of partitions.

Apache-Spark-Programming-with-Databricks/ASP 4 - Performance/ASP 4.2L - De-Duping Data Lab.py

@@ -51,7 +51,7 @@
 # COMMAND ----------
 
 # MAGIC %md
-# MAGIC It's helpful to look at the file first, so you can check the format with `dbutils.fs.head()`.
+# MAGIC It's helpful to look at the file first, so you can check the format with **`dbutils.fs.head()`**.
 
 # COMMAND ----------