Publishing v2.2.3

Author: SireInsectus

Apache-Spark-Programming-with-Databricks/ASP 0 - Course Agenda.py renamed to ASP 0 - Course Agenda.py

@@ -142,26 +142,34 @@
 # MAGIC The <a href="https://docs.databricks.com/data/databricks-file-system.html" target="_blank">Databricks File System</a> (DBFS) is a virtual file system that allows you to treat cloud object storage as though it were local files and directories on the cluster.
 # MAGIC 
 # MAGIC Run file system commands on DBFS using the magic command: **`%fs`**
+# MAGIC 
+# MAGIC <br/>
+# MAGIC <img src="https://files.training.databricks.com/images/icon_hint_24.png"/>
+# MAGIC Replace the instances of <strong>FILL_IN</strong> in the cells below with your email address:
+
+# COMMAND ----------
+
+# MAGIC %fs mounts
 
 # COMMAND ----------
 
 # MAGIC %fs ls
 
 # COMMAND ----------
 
-# MAGIC %fs mounts
+# MAGIC %fs ls dbfs:/tmp
 
 # COMMAND ----------
 
-# MAGIC %fs ls /databricks-datasets
+# MAGIC %fs put dbfs:/tmp/FILL_IN.txt "This is a test of the emergency broadcast system, this is only a test" --overwrite=true
 
 # COMMAND ----------
 
-# MAGIC %fs head /databricks-datasets/README.md
+# MAGIC %fs head dbfs:/tmp/FILL_IN.txt
 
 # COMMAND ----------
 
-# MAGIC %fs mounts
+# MAGIC %fs ls dbfs:/tmp
 
 # COMMAND ----------
 
@@ -178,28 +186,41 @@
 
 # COMMAND ----------
 
-dbutils.fs.ls("/databricks-datasets")
+dbutils.fs.ls("dbfs:/tmp")
 
 # COMMAND ----------
 
 # MAGIC %md Visualize results in a table using the Databricks <a href="https://docs.databricks.com/notebooks/visualizations/index.html#display-function-1" target="_blank">display</a> function
 
 # COMMAND ----------
 
-files = dbutils.fs.ls("/databricks-datasets")
+files = dbutils.fs.ls("dbfs:/tmp")
 display(files)
 
 # COMMAND ----------
 
+# MAGIC %md Let's take one more look at our temp file...
+
+# COMMAND ----------
+
+file_name = "dbfs:/tmp/FILL_IN.txt"
+contents = dbutils.fs.head(file_name)
+
+print("-"*80)
+print(contents)
+print("-"*80)
+
+# COMMAND ----------
+
 # MAGIC %md ## Our First Table
 # MAGIC 
-# MAGIC Is located in the path identfied by **`events_path`** (a variable we created for you).
+# MAGIC Is located in the path identfied by **`DA.paths.events`** (a variable we created for you).
 # MAGIC 
 # MAGIC We can see those files by running the following cell
 
 # COMMAND ----------
 
-files = dbutils.fs.ls(events_path)
+files = dbutils.fs.ls(DA.paths.events)
 display(files)
 
 # COMMAND ----------
@@ -213,7 +234,16 @@
 
 # COMMAND ----------
 
-spark.sql(f"SET c.events_path = {events_path}")
+spark.conf.set("whatever.events", DA.paths.events)
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC <img src="https://files.training.databricks.com/images/icon_note_24.png"> In the above example we use **`whatever.`** to give our variable a "namespace".
+# MAGIC 
+# MAGIC This is so that we don't accidently step over other configuration parameters.
+# MAGIC 
+# MAGIC You will see throughout this course our usage of the "DA" namesapce as in **`DA.paths.some_file`**
 
 # COMMAND ----------
 
@@ -225,15 +255,26 @@
 # MAGIC %sql
 # MAGIC CREATE TABLE IF NOT EXISTS events
 # MAGIC USING DELTA
-# MAGIC OPTIONS (path = "${c.events_path}");
+# MAGIC OPTIONS (path = "${whatever.events}");
+
+# COMMAND ----------
+
+# MAGIC %md This table was saved in the database created for you in classroom setup.
+# MAGIC 
+# MAGIC See database name printed below.
 
 # COMMAND ----------
 
-# MAGIC %md This table was saved in the database created for you in the classroom setup. See the database name printed below.
+print(f"Database Name: {DA.db_name}")
 
 # COMMAND ----------
 
-print(database_name)
+# MAGIC %md ... or even the tables in that database:
+
+# COMMAND ----------
+
+# MAGIC %sql
+# MAGIC SHOW TABLES IN ${DA.db_name}
 
 # COMMAND ----------
 
@@ -299,7 +340,7 @@
 # COMMAND ----------
 
 dbutils.widgets.text("name", "Brickster", "Name")
-dbutils.widgets.multiselect("colors", "orange", ["red", "orange", "black", "blue"], "Traffic Sources")
+dbutils.widgets.multiselect("colors", "orange", ["red", "orange", "black", "blue"], "Favorite Color?")
 
 # COMMAND ----------
 
@@ -331,7 +372,7 @@
 
 # COMMAND ----------
 
-classroom_cleanup()
+DA.cleanup()
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 1 - Introductions/ASP 1.1 - Databricks Platform.py renamed to ASP 1 - Introductions/ASP 1.1 - Databricks Platform.py

@@ -29,9 +29,9 @@
 # COMMAND ----------
 
 # MAGIC %md ### 1. List data files in DBFS using magic commands
-# MAGIC Use a magic command to display files located in the DBFS directory: **`dbfs:/databricks-datasets`**
+# MAGIC Use a magic command to display files located in the DBFS directory: **`dbfs:/user`**
 # MAGIC 
-# MAGIC <img src="https://files.training.databricks.com/images/icon_hint_32.png" alt="Hint"> You should see several datasets that come pre-installed in Databricks such as: **`COVID`**, **`adult`**, and **`airlines`**.
+# MAGIC <img src="https://files.training.databricks.com/images/icon_hint_32.png" alt="Hint"> You should see several user directories including your own. Depending on your permissions, you may see only your user directory.
 
 # COMMAND ----------
 
@@ -41,10 +41,10 @@
 # COMMAND ----------
 
 # MAGIC %md ### 2. List data files in DBFS using dbutils
-# MAGIC - Use **`dbutils`** to get the files at the directory above and save it to the variable **`files`**
+# MAGIC - Use **`dbutils`** to get the files at the directory above and assign it to the variable **`files`**
 # MAGIC - Use the Databricks display() function to display the contents in **`files`**
 # MAGIC 
-# MAGIC <img src="https://files.training.databricks.com/images/icon_hint_32.png" alt="Hint"> You should see several datasets that come pre-installed in Databricks such as: **`COVID`**, **`adult`**, and **`airlines`**.
+# MAGIC <img src="https://files.training.databricks.com/images/icon_hint_32.png" alt="Hint"> Just as before, you should see several user directories including your own.
 
 # COMMAND ----------
 
@@ -55,10 +55,10 @@
 # COMMAND ----------
 
 # MAGIC %md ### 3. Create tables below from files in DBFS
-# MAGIC - Create the **`users`** table using the spark-context variable **`c.users_path`**
-# MAGIC - Create the **`sales`** table using the spark-context variable **`c.sales_path`**
-# MAGIC - Create the **`products`** table using the spark-context variable **`c.products_path`**
-# MAGIC - Create the **`events`** table using the spark-context variable **`c.events_path`**
+# MAGIC - Create the **`users`** table using the spark-context variable **`DA.paths.users`**
+# MAGIC - Create the **`sales`** table using the spark-context variable **`DA.paths.sales`**
+# MAGIC - Create the **`products`** table using the spark-context variable **`DA.paths.products`**
+# MAGIC - Create the **`events`** table using the spark-context variable **`DA.paths.events`**
 # MAGIC 
 # MAGIC <img src="https://files.training.databricks.com/images/icon_hint_32.png"> Hint: We created the **`events`** table in the previous notebook but in a different database.
 
@@ -164,7 +164,7 @@
 
 # COMMAND ----------
 
-classroom_cleanup()
+DA.cleanup()
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 1 - Introductions/ASP 1.1L - Explore Datasets Lab.py renamed to ASP 1 - Introductions/ASP 1.1L - Explore Datasets Lab.py

@@ -20,8 +20,8 @@
 # MAGIC 1. Convert between DataFrames and SQL
 # MAGIC 
 # MAGIC ##### Methods
-# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql.html#spark-session-apis" target="_blank">SparkSession</a>: **`sql`**, **`table`**
-# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame.html" target="_blank">DataFrame</a>:
+# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/spark_session.html" target="_blank">SparkSession</a>: **`sql`**, **`table`**
+# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/dataframe.html" target="_blank">DataFrame</a>:
 # MAGIC - Transformations: **`select`**, **`where`**, **`orderBy`**
 # MAGIC - Actions: **`show`**, **`count`**, **`take`**
 # MAGIC - Other methods: **`printSchema`**, **`schema`**, **`createOrReplaceTempView`**
@@ -282,7 +282,7 @@
 
 # COMMAND ----------
 
-classroom_cleanup()
+DA.cleanup()
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/Solutions/ASP 2 - Spark Core/ASP 2.1 - Spark SQL.py renamed to ASP 2 - Spark Core/ASP 2.1 - Spark SQL.py

@@ -18,10 +18,10 @@
 # MAGIC 1. Create the same DataFrame using a SQL query
 # MAGIC 
 # MAGIC ##### Methods
-# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.SparkSession.html?highlight=sparksession" target="_blank">SparkSession</a>: **`sql`**, **`table`**
-# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame.html" target="_blank">DataFrame</a> transformations: **`select`**, **`where`**, **`orderBy`**
+# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/spark_session.html" target="_blank">SparkSession</a>: **`sql`**, **`table`**
+# MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/dataframe.html" target="_blank">DataFrame</a> transformations: **`select`**, **`where`**, **`orderBy`**
 # MAGIC - <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame.html" target="_blank">DataFrame</a> actions: **`select`**, **`count`**, **`take`**
-# MAGIC - Other <a href="https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame.html" target="_blank">DataFrame</a> methods: **`printSchema`**, **`schema`**, **`createOrReplaceTempView`**
+# MAGIC - Other <a href="https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/dataframe.html" target="_blank">DataFrame</a> methods: **`printSchema`**, **`schema`**, **`createOrReplaceTempView`**
 
 # COMMAND ----------
 
@@ -118,7 +118,7 @@
 
 # COMMAND ----------
 
-classroom_cleanup()
+DA.cleanup()
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/ASP 2 - Spark Core/ASP 2.1L - Spark SQL Lab.py renamed to ASP 2 - Spark Core/ASP 2.1L - Spark SQL Lab.py

@@ -45,7 +45,7 @@
 
 # COMMAND ----------
 
-users_csv_path = f"{datasets_dir}/users/users-500k.csv"
+users_csv_path = f"{DA.paths.datasets}/ecommerce/users/users-500k.csv"
 
 users_df = (spark
  .read
@@ -123,7 +123,7 @@
 
 # COMMAND ----------
 
-events_json_path = f"{datasets_dir}/events/events-500k.json"
+events_json_path = f"{DA.paths.datasets}/ecommerce/events/events-500k.json"
 
 events_df = (spark
  .read
@@ -187,7 +187,7 @@
 # COMMAND ----------
 
 # Step 1 - use this trick to transfer a value (the dataset path) between Python and Scala using the shared spark-config
-spark.conf.set("com.whatever.your_scope.events_path", events_json_path)
+spark.conf.set("whatever_your_scope.events", events_json_path)
 
 # COMMAND ----------
 
@@ -198,7 +198,7 @@
 
 # MAGIC %scala
 # MAGIC // Step 2 - pull the value from the config (or copy & paste it)
-# MAGIC val eventsJsonPath = spark.conf.get("com.whatever.your_scope.events_path")
+# MAGIC val eventsJsonPath = spark.conf.get("whatever_your_scope.events")
 # MAGIC 
 # MAGIC // Step 3 - Read in the JSON, but let it infer the schema
 # MAGIC val eventsSchema = spark.read
@@ -260,7 +260,7 @@
 
 # COMMAND ----------
 
-users_output_dir = working_dir + "/users.parquet"
+users_output_dir = f"{DA.paths.working_dir}/users.parquet"
 
 (users_df
  .write
@@ -301,11 +301,22 @@
 
 # COMMAND ----------
 
-# MAGIC %md This table was saved in the database created for you in classroom setup. See database name printed below.
+# MAGIC %md This table was saved in the database created for you in classroom setup.
+# MAGIC 
+# MAGIC See database name printed below.
+
+# COMMAND ----------
+
+print(f"Database Name: {DA.db_name}")
+
+# COMMAND ----------
+
+# MAGIC %md ... or even the tables in that database:
 
 # COMMAND ----------
 
-print(database_name)
+# MAGIC %sql
+# MAGIC SHOW TABLES IN ${DA.db_name}
 
 # COMMAND ----------
 
@@ -337,7 +348,7 @@
 
 # COMMAND ----------
 
-events_output_path = working_dir + "/delta/events"
+events_output_path = f"{DA.paths.working_dir}/delta/events"
 
 (events_df
  .write
@@ -352,7 +363,7 @@
 
 # COMMAND ----------
 
-classroom_cleanup()
+DA.cleanup()
 
 # COMMAND ----------
 

Apache-Spark-Programming-with-Databricks/Solutions/ASP 2 - Spark Core/ASP 2.2 - Reader & Writer.py renamed to ASP 2 - Spark Core/ASP 2.2 - Reader & Writer.py

@@ -25,17 +25,17 @@
 # COMMAND ----------
 
 # MAGIC %md ### 1. Read with infer schema
-# MAGIC - View the first CSV file using DBUtils method **`fs.head`** with the filepath provided in the variable **`single_product_cs_fil_path`**
+# MAGIC - View the first CSV file using DBUtils method **`fs.head`** with the filepath provided in the variable **`single_product_cs_file_path`**
 # MAGIC - Create **`products_df`** by reading from CSV files located in the filepath provided in the variable **`products_csv_path`**
 # MAGIC - Configure options to use first line as header and infer schema
 
 # COMMAND ----------
 
 # TODO
-single_product_csv_file_path = f"{datasets_dir}/products/products.csv/part-00000-tid-1663954264736839188-daf30e86-5967-4173-b9ae-d1481d3506db-2367-1-c000.csv"
+single_product_csv_file_path = f"{DA.paths.datasets}/products/products.csv/part-00000-tid-1663954264736839188-daf30e86-5967-4173-b9ae-d1481d3506db-2367-1-c000.csv"
 print(FILL_IN)
 
-products_csv_path = f"{datasets_dir}/products/products.csv"
+products_csv_path = f"{DA.paths.datasets}/products/products.csv"
 products_df = FILL_IN
 
 products_df.printSchema()
@@ -108,7 +108,7 @@
 # COMMAND ----------
 
 # TODO
-products_output_path = working_dir + "/delta/products"
+products_output_path = f"{DA.paths.working_dir}/delta/products"
 products_df.FILL_IN
 
 # COMMAND ----------
@@ -137,7 +137,7 @@
 
 # COMMAND ----------
 
-classroom_cleanup()
+DA.cleanup()
 
 # COMMAND ----------