Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.

Make local dbt data diffs concurrent #776

Merged
merged 27 commits into from
Dec 5, 2023
Merged

Make local dbt data diffs concurrent #776

Changes from 1 commit
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
e6fac1e
v0 of concurrency
Nov 13, 2023
6529027
concurrent logging
Nov 13, 2023
1dbc44b
Merge branch 'master' of https://github.com/datafold/data-diff into f…
Nov 14, 2023
7ef9370
remove todo
Nov 15, 2023
fb5d8ec
remove todo
Nov 15, 2023
a56b2be
better var name
Nov 15, 2023
9aa13e3
add node name to logger
Nov 15, 2023
05c7921
format string logs
Nov 15, 2023
2b3184f
Merge branch 'master' of https://github.com/datafold/data-diff into f…
Nov 15, 2023
be3f631
add optional logger param
Nov 15, 2023
88592ad
avoid extra threads
Nov 17, 2023
e5ee4df
use thread pools
Nov 18, 2023
76d1302
not multithreaded at the connection level anymore
Nov 18, 2023
5812a16
show errors as they happen
Nov 20, 2023
8f6eb1b
show full stacktrace on error
Nov 20, 2023
30deafa
rearrange trace
Nov 20, 2023
6fc9900
more logs for debugging
Nov 20, 2023
8fe43de
update for threads mocking
Nov 20, 2023
12bebcc
clear log params
Nov 20, 2023
57a9eec
remove extra space
Nov 21, 2023
d64a168
Merge branch 'master' of https://github.com/datafold/data-diff into f…
Nov 21, 2023
c640bf7
Merge branch 'master' of https://github.com/datafold/data-diff into f…
Dec 5, 2023
5cd12ff
remove long traceback
Dec 5, 2023
6d41f2a
Ensure log_message is optional
sungchun12 Dec 5, 2023
02e147f
map threaded result to proper model id
Dec 5, 2023
852dc3e
explicit type and optional
Dec 5, 2023
81d78dd
rm submodules again
Dec 5, 2023
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add optional logger param
  • Loading branch information
Sung Won Chung committed Nov 15, 2023
commit be3f631d4e7801bc243324f5397db5e22afbd9fd
6 changes: 3 additions & 3 deletions tests/test_dbt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ def test_diff_is_not_cloud(self, mock_print, mock_dbt_parser, mock_cloud_diff, m
mock_dbt_parser_inst.get_models.assert_called_once()
mock_dbt_parser_inst.set_connection.assert_called_once()
mock_cloud_diff.assert_not_called()
mock_local_diff.assert_called_once_with(diff_vars, False)
mock_local_diff.assert_called_once_with(diff_vars, False, None)
mock_print.assert_not_called()

@patch("data_diff.dbt._get_diff_vars")
Expand Down Expand Up @@ -481,7 +481,7 @@ def test_diff_only_prod_db(self, mock_print, mock_dbt_parser, mock_cloud_diff, m
mock_dbt_parser_inst.get_models.assert_called_once()
mock_dbt_parser_inst.set_connection.assert_called_once()
mock_cloud_diff.assert_not_called()
mock_local_diff.assert_called_once_with(diff_vars, False)
mock_local_diff.assert_called_once_with(diff_vars, False, None)
mock_print.assert_not_called()

@patch("data_diff.dbt._get_diff_vars")
Expand Down Expand Up @@ -518,7 +518,7 @@ def test_diff_only_prod_schema(
mock_dbt_parser_inst.get_models.assert_called_once()
mock_dbt_parser_inst.set_connection.assert_called_once()
mock_cloud_diff.assert_not_called()
mock_local_diff.assert_called_once_with(diff_vars, False)
mock_local_diff.assert_called_once_with(diff_vars, False, None)
mock_print.assert_not_called()

@patch("data_diff.dbt._initialize_api")
Expand Down