Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
working version
  • Loading branch information
Sung Won Chung committed Nov 6, 2023
commit 2b12c848c89bc889fd5ef2c7da8f1feb1695d026
6 changes: 4 additions & 2 deletions data_diff/dbt.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from contextlib import nullcontext
import json
import time
import os
import re
import time
Expand Down Expand Up @@ -114,6 +115,7 @@ def dbt_diff(

with log_status_handler.status if log_status_handler else nullcontext():
for model in models:
start_time = time.monotonic()
if log_status_handler:
log_status_handler.set_prefix(f"Diffing {model.alias} \n")

Expand Down Expand Up @@ -164,12 +166,12 @@ def dbt_diff(
_diff_output_base(".".join(diff_vars.dev_path), ".".join(diff_vars.prod_path))
+ "Skipped due to unknown primary key. Add uniqueness tests, meta, or tags.\n"
)
log_status_handler.stop_counter(model_name=model.alias, start_time=start_time)

# wait for all threads
if diff_threads:
for thread in diff_threads:
thread.join()

_extension_notification()


Expand Down Expand Up @@ -508,7 +510,7 @@ def _cloud_diff(
diff_url = f"{api.host}/datadiffs/{diff_id}/overview"
rich.print(f"{diff_url} \n")
logger.error(error)

log_status_handler.stop_counter(model_name=model.alias, start_time=start_time)

def _diff_output_base(dev_path: str, prod_path: str) -> str:
return f"\n[green]{prod_path} <> {dev_path}[/] \n"
Expand Down
8 changes: 8 additions & 0 deletions data_diff/utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
import time
import logging
import math
import re
Expand Down Expand Up @@ -485,6 +486,7 @@ def __init__(self):
super().__init__()
self.status = Status("")
self.prefix = ""
self.diff_time = ""
self.cloud_diff_status = {}

def emit(self, record):
Expand All @@ -497,6 +499,12 @@ def emit(self, record):
def set_prefix(self, prefix_string):
self.prefix = prefix_string

def stop_counter(self, model_name, start_time):
end_time = time.monotonic()
duration = end_time - start_time
self.diff_time = f"[{model_name}] diff time: {duration:.2f}s"
print(self.diff_time)

def cloud_diff_started(self, model_name):
self.cloud_diff_status[model_name] = "[yellow]In Progress[/]"
self._update_cloud_status()
Expand Down