add course files

Author: cassiobolba

.DS_Store

@@ -0,0 +1,154 @@
+------------------------ Prep para carregar dados ------------------------------
+
+CREATE OR REPLACE TABLE EVENTS
+(
+ GLOBALEVENTID integer
+ ,SQLDATE date
+ ,MONTHYEAR string
+ ,Actor1Name string
+ ,NumMentions integer
+ ,SOURCEURL string
+); 
+
+list @GDELT_EVENTS;
+
+
+CREATE OR REPLACE FILE FORMAT CSV_TAB_FMT
+FIELD_DELIMITER = '\t'
+	TYPE = CSV;
+
+show file formats in database GDELT;
+ 
+------------------------ Carregando os dados ------------------------------
+
+COPY INTO EVENTS FROM (
+SELECT 
+ $1::int GLOBALEVENTID
+ ,TO_DATE($2,'YYYYMMDD') AS SQLDATE
+ ,$3::string MONTHYEAR
+ ,$7::string Actor1Name
+ ,$34::int NumMentions
+ ,$58::string SOURCEURL
+FROM @MANAGE_DB.EXTERNAL_STAGES.GDELT_EVENTS/events/20190917.export.csv
+(file_format => MANAGE_DB.FILE_FORMATS.CSV_TAB_FMT) );
+--(file_format => MANAGE_DB.FILE_FORMATS.CSV_TAB_FMT)
+);
+
+
+--mostrar que nao tem mais nada
+select * from EVENTS limit 10;
+
+--change warehouse size from small to large (4x)
+alter warehouse compute_wh set warehouse_size='large';
+
+--load data with large warehouse
+show warehouses;
+
+--Rodar novamente comando copy e mostar que foi mais rapido
+
+
+
+------------------------ Cache results e Clone ------------------------------
+
+select * from EVENTS limit 20;
+
+SELECT 
+ Actor1Name as actor_name
+ ,sum(NumMentions) mentions_actor
+ ,count(GLOBALEVENTID) events_actor
+FROM EVENTS
+where Actor1Name is not null
+group by 1
+order by 2 desc;
+
+
+create table trips_dev clone trips;
+
+
+------------------------ Psemi-structured data ------------------------------
+
+create database weather;
+
+use role sysadmin;
+use warehouse compute_wh;
+use database weather;
+use schema public;
+
+create table json_weather_data (v variant);
+
+create stage nyc_weather
+url = 's3://snowflake-workshop-lab/weather-nyc';
+
+list @nyc_weather;
+
+copy into json_weather_data 
+from @nyc_weather 
+file_format = (type=json);
+
+select * from json_weather_data limit 10;
+
+create view json_weather_data_view as
+select
+ v:time::timestamp as observation_time,
+ v:city.id::int as city_id,
+ v:city.name::string as city_name,
+ v:city.country::string as country,
+ v:city.coord.lat::float as city_lat,
+ v:city.coord.lon::float as city_lon,
+ v:clouds.all::int as clouds,
+ (v:main.temp::float)-273.15 as temp_avg,
+ (v:main.temp_min::float)-273.15 as temp_min,
+ (v:main.temp_max::float)-273.15 as temp_max,
+ v:weather[0].main::string as weather,
+ v:weather[0].description::string as weather_desc,
+ v:weather[0].icon::string as weather_icon,
+ v:wind.deg::float as wind_dir,
+ v:wind.speed::float as wind_speed
+from json_weather_data
+where city_id = 5128638;
+
+
+------------------------ time travel ------------------------------
+
+drop table json_weather_data;=
+
+Select * from json_weather_data limit 10;
+
+undrop table json_weather_data;
+
+use role sysadmin;
+use warehouse compute_wh;
+use database citibike;
+use schema public;
+
+update trips set start_station_name = 'oops';
+
+select 
+start_station_name as "station",
+count(*) as "rides"
+from trips
+group by 1
+order by 2 desc
+limit 20;
+
+set query_id = 
+(select query_id from 
+table(information_schema.query_history_by_session (result_limit=>5)) 
+where query_text like 'update%' order by start_time limit 1);
+
+create or replace table trips as
+(select * from trips before (statement => $query_id));
+ 
+select 
+start_station_name as "station",
+count(*) as "rides"
+from trips
+group by 1
+order by 2 desc
+limit 20;
+
+
+
+
+
+

Snowflake/.DS_Store

@@ -0,0 +1,141 @@
+SELECT 
+$1
+ ,$2
+FROM @MANAGE_DB.EXTERNAL_STAGES.GDELT_EVENTS/events/20190916.export.csv ;
+
+CREATE DATABASE GDELT;
+
+CREATE SCHEMA EVENTS;
+
+CREATE TABLE IF NOT EXISTS GDELT.EVENTS.EVENTS_FULL (
+
+ GLOBALEVENTID INT
+
+ ,SQLDATE varchar
+
+ ,MonthYear varchar
+
+ ,Year varchar
+
+ ,FractionDate varchar
+
+ ,Actor1Code varchar
+
+ ,Actor1Name varchar
+
+ ,Actor1CountryCode varchar
+
+ ,Actor1KnownGroupCode varchar
+
+ ,Actor1EthnicCode varchar
+
+ ,Actor1Religion1Code varchar
+
+ ,Actor1Religion2Code varchar
+
+ ,Actor1Type1Code varchar
+
+ ,Actor1Type2Code varchar
+
+ ,Actor1Type3Code varchar
+
+ ,Actor2Code varchar
+
+ ,Actor2Name varchar
+
+ ,Actor2CountryCode varchar
+
+ ,Actor2KnownGroupCode varchar
+
+ ,Actor2EthnicCode varchar
+
+ ,Actor2Religion1Code varchar
+
+ ,Actor2Religion2Code varchar
+
+ ,Actor2Type1Code varchar
+
+ ,Actor2Type2Code varchar
+
+ ,Actor2Type3Code varchar
+
+ ,IsRootEvent varchar
+
+ ,EventCode varchar
+
+ ,EventBaseCode varchar
+
+ ,EventRootCode varchar
+
+ ,QuadClass varchar
+
+ ,GoldsteinScale varchar
+
+ ,NumMentions varchar
+
+ ,NumSources varchar
+
+ ,NumArticles varchar
+
+ ,AvgTone varchar
+
+ ,Actor1Geo_Type varchar
+
+ ,Actor1Geo_FullName varchar
+
+ ,Actor1Geo_CountryCode varchar
+
+ ,Actor1Geo_ADM1Code varchar
+
+ ,Actor1Geo_Lat varchar
+
+ ,Actor1Geo_Long varchar
+
+ ,Actor1Geo_FeatureID varchar
+
+ ,Actor2Geo_Type varchar
+
+ ,Actor2Geo_FullName varchar
+
+ ,Actor2Geo_CountryCode varchar
+
+ ,Actor2Geo_ADM1Code varchar
+
+ ,Actor2Geo_Lat varchar
+
+ ,Actor2Geo_Long varchar
+
+ ,Actor2Geo_FeatureID varchar
+
+ ,ActionGeo_Type varchar
+
+ ,ActionGeo_FullName varchar
+
+ ,ActionGeo_CountryCode varchar
+
+ ,ActionGeo_ADM1Code varchar
+
+ ,ActionGeo_Lat varchar
+
+ ,ActionGeo_Long varchar
+
+ ,ActionGeo_FeatureID varchar
+
+ ,DATEADDED varchar
+
+ ,SOURCEURL varchar
+
+ );
+
+
+COPY INTO GDELT.EVENTS.EVENTS_FULL
+FROM @MANAGE_DB.EXTERNAL_STAGES.GDELT_EVENTS/events
+ file_format = ( type = 'csv' field_delimiter = '\t')
+ pattern = '.*2019091.*'
+ ;
+
+select * from GDELT.EVENTS.EVENTS_FULL;
+
+
+
+ 

Snowflake/live-sf-getting-started.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE SCHEMA FILE_FORMATS;
+
+CREATE OR REPLACE FILE FORMAT MANAGE_DB.FILE_FORMATS.CSV_TAB_FMT
+FIELD_DELIMITER = '\t'
+ TYPE = CSV
+ --SKIP_HEADER=1
+ ;
+
+DESC FILE FORMAT FILE_FORMATS.CSV_TAB_FMT;
+
+LIST @MANAGE_DB.EXTERNAL_STAGES.GDELT_EVENTS/events/ ;
+
+COPY INTO GDELT.EVENTS.EVENTS_FULL
+FROM @MANAGE_DB.EXTERNAL_STAGES.GDELT_EVENTS/events/20130922.export.csv
+ file_format = MANAGE_DB.FILE_FORMATS.CSV_TAB_FMT
+;

Snowflake/the-snowflake-series-course/COPY.sql

@@ -0,0 +1,19 @@
+SELECT DISTINCT
+RAW_FILE:id id
+ ,array_size(RAW_FILE:multiMedia) size
+from medias.youtube.statistics_raw ;
+
+SELECT distinct
+ RAW_FILE:id::int id
+,RAW_FILE:createdAt createdAt
+ ,RAW_FILE:description::string description
+ ,RAW_FILE:likeDislike.dislikes dislikes
+,RAW_FILE:likeDislike.likes likes
+ ,RAW_FILE:likeDislike.userAction user_action
+ ,f.value:id multimedia_id
+FROM MEDIAS.YOUTUBE.STATISTICS_RAW ,
+table(flatten(RAW_FILE:multiMedia)) f
+--where RAW_FILE:id::int = 2114 
+
+
+;

Snowflake/the-snowflake-series-course/FILE FORMAT.sql

@@ -0,0 +1,27 @@
+
+CREATE OR REPLACE TABLE MEDIAS.YOUTUBE.STATISTICS AS
+SELECT distinct
+ RAW_FILE:id::int id
+,TO_TIMESTAMP(RAW_FILE:createdAt) createdAt
+ ,RAW_FILE:description::string description
+ ,RAW_FILE:likeDislike.dislikes::INT dislikes
+,RAW_FILE:likeDislike.likes::INT likes
+ ,RAW_FILE:likeDislike.userAction::INT user_action
+ ,f.value:id::INT multimedia_id
+FROM MEDIAS.YOUTUBE.STATISTICS_RAW ,
+table(flatten(RAW_FILE:multiMedia)) f 
+;
+
+SELECT COUNT(*) FROM MEDIAS.YOUTUBE.STATISTICS;
+
+INSERT INTO MEDIAS.YOUTUBE.STATISTICS
+SELECT distinct
+ RAW_FILE:id::int id
+,TO_TIMESTAMP(RAW_FILE:createdAt) createdAt
+ ,RAW_FILE:description::string description
+ ,RAW_FILE:likeDislike.dislikes::INT dislikes
+,RAW_FILE:likeDislike.likes::INT likes
+ ,RAW_FILE:likeDislike.userAction::INT user_action
+ ,f.value:id::INT multimedia_id
+FROM MEDIAS.YOUTUBE.STATISTICS_RAW ,
+table(flatten(RAW_FILE:multiMedia)) f ;

Snowflake/the-snowflake-series-course/FLATTEN.sql

@@ -0,0 +1,24 @@
+-- CRIAR STAGE > CARREGAR DADOS BRUTOS > ANALISE E PARSE > CRIAR O COMANDO COPY 
+
+CREATE OR REPLACE STAGE MANAGE_DB.EXTERNAL_STAGES.SNOWFLAKE_SERIES
+URL= 's3://snowflake-series/'
+ STORAGE_INTEGRATION = S3_INT;
+
+LIST @MANAGE_DB.EXTERNAL_STAGES.SNOWFLAKE_SERIES;
+
+CREATE OR REPLACE FILE FORMAT MANAGE_DB.FILE_FORMATS.JSON_FMT
+ TYPE = JSON;
+
+CREATE DATABASE MEDIAS;
+
+CREATE SCHEMA MEDIAS.YOUTUBE;
+
+CREATE OR REPLACE table MEDIAS.YOUTUBE.STATISTICS_RAW (
+ raw_file variant);
+
+COPY INTO MEDIAS.YOUTUBE.STATISTICS_RAW
+ FROM @MANAGE_DB.EXTERNAL_STAGES.SNOWFLAKE_SERIES
+ file_format= MANAGE_DB.FILE_FORMATS.JSON_FMT
+ files = ('youtube_data.json');
+
+SELECT * FROM MEDIAS.YOUTUBE.STATISTICS_RAW ;