Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.

Tests now cover oracle, Redshift, snowflake and bigquery; Various fixes to said drivers. #170

Merged
merged 11 commits into from
Jul 22, 2022
Merged

Tests now cover oracle, Redshift, snowflake and bigquery; Various fixes to said drivers. #170

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
92241df
Added Oracle tests (WIP)
erezsh Jul 21, 2022
44523b6
Fixes issue #122
erezsh Jul 21, 2022
7aa7594
Various fixes for Oracle
erezsh Jul 21, 2022
a27f64c
Added 'timestamp_value' fix for Oracle
erezsh Jul 21, 2022
83a3f23
Fix for redshift
erezsh Jul 21, 2022
b5dade5
Add Snowflake to tests
erezsh Jul 21, 2022
f72c29b
Fix tests to work with BigQuery
erezsh Jul 21, 2022
6a54ef0
Tiny refactor
erezsh Jul 21, 2022
1455b9b
Ran black
erezsh Jul 21, 2022
6e8fbfb
Bump Preql version
erezsh Jul 21, 2022
2020859
Update poetry.lock
erezsh Jul 21, 2022
File filter

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Various fixes for Oracle
  • Loading branch information
@erezsh
erezsh committed Jul 21, 2022
commit 7aa7594d65723fe80c06c3153ec7dc3b3fd22c3c
7 changes: 4 additions & 3 deletions data_diff/databases/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,10 @@ def _parse_type(

elif issubclass(cls, Decimal):
if numeric_scale is None:
raise ValueError(
f"{self.name}: Unexpected numeric_scale is NULL, for column {'.'.join(table_path)}.{col_name} of type {type_repr}."
)
numeric_scale = 0 # Needed for Oracle.
# raise ValueError(
# f"{self.name}: Unexpected numeric_scale is NULL, for column {'.'.join(table_path)}.{col_name} of type {type_repr}."
# )
return cls(precision=numeric_scale)

elif issubclass(cls, Float):
Expand Down
1 change: 1 addition & 0 deletions data_diff/databases/oracle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ def _parse_type(
regexps = {
r"TIMESTAMP\((\d)\) WITH LOCAL TIME ZONE": Timestamp,
r"TIMESTAMP\((\d)\) WITH TIME ZONE": TimestampTZ,
r"TIMESTAMP\((\d)\)": Timestamp,
}
for regexp, t_cls in regexps.items():
m = re.match(regexp + "$", type_repr)
Expand Down
2 changes: 1 addition & 1 deletion data_diff/sql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ class Time(Sql):
column: Optional[SqlOrStr] = None

def compile(self, c: Compiler):
return "'%s'" % self.time.isoformat()
return "timestamp '%s'" % self.time.isoformat(' ')


@dataclass
Expand Down
102 changes: 54 additions & 48 deletions tests/test_diff_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,26 +192,28 @@ def test_properties_on_empty_table(self):

def test_get_values(self):
time = "2022-01-01 00:00:00.000000"
res = self.preql(
f"""
new {self.table_src}(1, 1, 9, '{time}')
"""
)
self.preql.commit()
time_str = f"timestamp '{time}'"

cols = 'userid movieid rating timestamp'.split()
_insert_row(self.connection, self.table_src, cols, [1,1,9,time_str])
self.connection.query("COMMIT", None)
id_ = self.connection.query(f'select id from {self.table_src}', int)

table = self.table.with_schema()

self.assertEqual(1, table.count())
concatted = str(res["id"]) + time
concatted = str(id_) + time
self.assertEqual(str_to_checksum(concatted), table.count_and_checksum()[1])

def test_diff_small_tables(self):
time = "timestamp '2022-01-01 00:00:00'"
time = '2022-01-01 00:00:00'
time_str = f"timestamp '{time}'"


cols = 'userid movieid rating timestamp'.split()
_insert_row(self.connection, self.table_src, cols, [1,1,9,time])
_insert_row(self.connection, self.table_src, cols, [2,2,9,time])
_insert_row(self.connection, self.table_dst, cols, [1,1,9,time])
_insert_row(self.connection, self.table_src, cols, [1,1,9,time_str])
_insert_row(self.connection, self.table_src, cols, [2,2,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [1,1,9,time_str])
self.connection.query("COMMIT", None)
diff = list(self.differ.diff_tables(self.table, self.table2))
expected = [("-", ("2", time + ".000000"))]
Expand All @@ -221,21 +223,21 @@ def test_diff_small_tables(self):

def test_diff_table_above_bisection_threshold(self):
time = "2022-01-01 00:00:00"
self.preql(
f"""
new {self.table_src}(userid: 1, movieid: 1, rating: 9, timestamp: '{time}')
new {self.table_src}(userid: 2, movieid: 2, rating: 9, timestamp: '{time}')
new {self.table_src}(userid: 3, movieid: 3, rating: 9, timestamp: '{time}')
new {self.table_src}(userid: 4, movieid: 4, rating: 9, timestamp: '{time}')
new {self.table_src}(userid: 5, movieid: 5, rating: 9, timestamp: '{time}')

new {self.table_dst}(userid: 1, movieid: 1, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 2, movieid: 2, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 3, movieid: 3, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 4, movieid: 4, rating: 9, timestamp: '{time}')
"""
)
self.preql.commit()
time_str = f"timestamp '{time}'"

cols = 'userid movieid rating timestamp'.split()
_insert_row(self.connection, self.table_src, cols, [1,1,9,time_str])
_insert_row(self.connection, self.table_src, cols, [2,2,9,time_str])
_insert_row(self.connection, self.table_src, cols, [3,3,9,time_str])
_insert_row(self.connection, self.table_src, cols, [4,4,9,time_str])
_insert_row(self.connection, self.table_src, cols, [5,5,9,time_str])

_insert_row(self.connection, self.table_dst, cols, [1,1,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [2,2,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [3,3,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [4,4,9,time_str])
self.connection.query("COMMIT", None)

diff = list(self.differ.diff_tables(self.table, self.table2))
expected = [("-", ("5", time + ".000000"))]
self.assertEqual(expected, diff)
Expand All @@ -244,35 +246,39 @@ def test_diff_table_above_bisection_threshold(self):

def test_return_empty_array_when_same(self):
time = "2022-01-01 00:00:00"
self.preql(
f"""
new {self.table_src}(userid: 1, movieid: 1, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 1, movieid: 1, rating: 9, timestamp: '{time}')
"""
)
time_str = f"timestamp '{time}'"

cols = 'userid movieid rating timestamp'.split()

_insert_row(self.connection, self.table_src, cols, [1,1,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [1,1,9,time_str])

self.preql.commit()
diff = list(self.differ.diff_tables(self.table, self.table2))
self.assertEqual([], diff)

def test_diff_sorted_by_key(self):
time = "2022-01-01 00:00:00"
time2 = "2021-01-01 00:00:00"
self.preql(
f"""
new {self.table_src}(userid: 1, movieid: 1, rating: 9, timestamp: '{time}')
new {self.table_src}(userid: 2, movieid: 2, rating: 9, timestamp: '{time2}')
new {self.table_src}(userid: 3, movieid: 3, rating: 9, timestamp: '{time}')
new {self.table_src}(userid: 4, movieid: 4, rating: 9, timestamp: '{time2}')
new {self.table_src}(userid: 5, movieid: 5, rating: 9, timestamp: '{time}')

new {self.table_dst}(userid: 1, movieid: 1, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 2, movieid: 2, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 3, movieid: 3, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 4, movieid: 4, rating: 9, timestamp: '{time}')
new {self.table_dst}(userid: 5, movieid: 5, rating: 9, timestamp: '{time}')
"""
)
self.preql.commit()

time_str = f"timestamp '{time}'"
time_str2 = f"timestamp '{time2}'"

cols = 'userid movieid rating timestamp'.split()

_insert_row(self.connection, self.table_src, cols, [1,1,9,time_str])
_insert_row(self.connection, self.table_src, cols, [2,2,9,time_str2])
_insert_row(self.connection, self.table_src, cols, [3,3,9,time_str])
_insert_row(self.connection, self.table_src, cols, [4,4,9,time_str2])
_insert_row(self.connection, self.table_src, cols, [5,5,9,time_str])

_insert_row(self.connection, self.table_dst, cols, [1,1,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [2,2,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [3,3,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [4,4,9,time_str])
_insert_row(self.connection, self.table_dst, cols, [5,5,9,time_str])
self.connection.query("COMMIT", None)

differ = TableDiffer()
diff = list(differ.diff_tables(self.table, self.table2))
expected = [
Expand Down