Updated docs; Ran black

Author: erezsh

README.md

@@ -252,6 +252,7 @@ Options:
 
 Same-DB diff only:
  - `-m`, `--materialize` - Materialize the diff results into a new table in the database.
+ If a table exists by that name, it will be replaced.
  Use `%t` in the name to place a timestamp.
  Example: `-m test_mat_%t`
  - `--assume-unique-key` - Skip validating the uniqueness of the key column during joindiff, which is costly in non-cloud dbs.

data_diff/__main__.py

@@ -80,7 +80,9 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
 @click.argument("table1", required=False)
 @click.argument("database2", required=False)
 @click.argument("table2", required=False)
-@click.option("-k", "--key-columns", default=[], multiple=True, help="Names of primary key columns. Default='id'.", metavar="NAME")
+@click.option(
+ "-k", "--key-columns", default=[], multiple=True, help="Names of primary key columns. Default='id'.", metavar="NAME"
+)
 @click.option("-t", "--update-column", default=None, help="Name of updated_at/last_updated column", metavar="NAME")
 @click.option(
  "-c",
@@ -110,7 +112,7 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
  "--materialize",
  default=None,
  metavar="TABLE_NAME",
- help="Materialize the diff results into a new table in the database. (joindiff only)",
+ help="(joindiff only) Materialize the diff results into a new table in the database. If a table exists by that name, it will be replaced.",
 )
 @click.option(
  "--min-age",
@@ -345,7 +347,7 @@ def _main(
  *key_columns,
  update_column,
  *columns,
- )
+ ),
  )
 
  logging.info(f"Diffing using columns: key={key_columns} update={update_column} extra={columns}")
@@ -385,7 +387,7 @@ def _main(
  if differ.stats:
  print("Extra-Info:")
  for k, v in differ.stats.items():
- print(f' {k} = {v}')
+ print(f" {k} = {v}")
  else:
  for op, values in diff_iter:
  color = COLOR_SCHEME[op]

data_diff/config.py

@@ -26,13 +26,13 @@ def _apply_config(config: Dict[str, Any], run_name: str, kw: Dict[str, Any]):
  else:
  run_name = "default"
 
- if 'database1' in kw:
- for attr in ('table1', 'database2', 'table2'):
+ if "database1" in kw:
+ for attr in ("table1", "database2", "table2"):
  if kw[attr] is None:
  raise ValueError(f"Specified database1 but not {attr}. Must specify all 4 arguments, or niether.")
 
  for index in "12":
- run_args[index] = {attr: kw.pop(f"{attr}{index}") for attr in ('database', 'table')}
+ run_args[index] = {attr: kw.pop(f"{attr}{index}") for attr in ("database", "table")}
 
  # Process databases + tables
  for index in "12":

data_diff/databases/base.py

@@ -129,8 +129,8 @@ def query(self, sql_ast: Union[Expr, Generator], res_type: type = list):
  if getattr(self, "_interactive", False) and isinstance(sql_ast, Select):
  explained_sql = compiler.compile(Explain(sql_ast))
  explain = self._query(explained_sql)
- for row, in explain:
- logger.debug(f'EXPLAIN: {row}')
+ for (row,) in explain:
+ logger.debug(f"EXPLAIN: {row}")
  answer = input("Continue? [y/n] ")
  if not answer.lower() in ["y", "yes"]:
  sys.exit(1)

data_diff/diff_tables.py

@@ -157,8 +157,8 @@ def _bisect_and_diff_tables(self, table1, table2):
  raise NotImplementedError("Composite key not supported yet!")
  if len(table2.key_columns) > 1:
  raise NotImplementedError("Composite key not supported yet!")
- key1 ,= table1.key_columns
- key2 ,= table2.key_columns
+ (key1,) = table1.key_columns
+ (key2,) = table2.key_columns
 
  key_type = table1._schema[key1]
  key_type2 = table2._schema[key2]
@@ -214,7 +214,7 @@ def _bisect_and_diff_segments(
  assert table1.is_bounded and table2.is_bounded
 
  # Choose evenly spaced checkpoints (according to min_key and max_key)
- biggest_table = max(table1, table2, key=methodcaller('approximate_size'))
+ biggest_table = max(table1, table2, key=methodcaller("approximate_size"))
  checkpoints = biggest_table.choose_checkpoints(self.bisection_factor - 1)
 
  # Create new instances of TableSegment between each checkpoint

data_diff/joindiff_tables.py

@@ -92,8 +92,7 @@ def append_to_table_oracle(path: DbPath, expr: Expr):
 
 
 def append_to_table(path: DbPath, expr: Expr):
- """Append to table
- """
+ """Append to table"""
  assert expr.schema, expr
  t = table(path, schema=expr.schema)
  yield t.create(if_not_exists=True) # uses expr.schema

data_diff/queries/api.py

@@ -32,7 +32,7 @@ def cte(expr: Expr, *, name: Optional[str] = None, params: Sequence[str] = None)
 
 def table(*path: str, schema: Schema = None) -> TablePath:
  if len(path) == 1 and isinstance(path[0], tuple):
- path ,= path
+ (path,) = path
  if not all(isinstance(i, str) for i in path):
  raise TypeError(f"All elements of table path must be of type 'str'. Got: {path}")
  return TablePath(path, schema)

data_diff/queries/ast_classes.py

@@ -257,7 +257,7 @@ def type(self):
  types = {get_type(i) for i in self.args}
  if len(types) > 1:
  raise TypeError(f"Expected all args to have the same type, got {types}")
- t ,= types
+ (t,) = types
  return t
 
 
@@ -607,6 +607,7 @@ class Explain(ExprNode):
  def compile(self, c: Compiler) -> str:
  return c.database.explain_as_text(c.compile(self.select))
 
+
 # DDL
 
 

data_diff/table_segment.py

@@ -81,11 +81,11 @@ def with_schema(self) -> "TableSegment":
  def _make_key_range(self):
  if self.min_key is not None:
  assert len(self.key_columns) == 1
- k ,= self.key_columns
+ (k,) = self.key_columns
  yield self.min_key <= this[k]
  if self.max_key is not None:
  assert len(self.key_columns) == 1
- k ,= self.key_columns
+ (k,) = self.key_columns
  yield this[k] < self.max_key
 
  def _make_update_range(self):
@@ -178,7 +178,7 @@ def query_key_range(self) -> Tuple[int, int]:
  """Query database for minimum and maximum key. This is used for setting the initial bounds."""
  # Normalizes the result (needed for UUIDs) after the min/max computation
  # TODO better error if there is no schema
- k ,= self.key_columns
+ (k,) = self.key_columns
  select = self._make_select().select(
  ApplyFuncAndNormalizeAsString(this[k], min_),
  ApplyFuncAndNormalizeAsString(this[k], max_),

tests/test_diff_tables.py

@@ -44,6 +44,7 @@ def _class_per_db_dec(filter_name=None):
  ]
  return parameterized_class(("name", "db_name"), names)
 
+
 def _table_segment(database, table_path, key_columns, *args, **kw):
  if isinstance(key_columns, str):
  key_columns = (key_columns,)
@@ -195,16 +196,24 @@ def test_basic(self):
  def test_offset(self):
  differ = HashDiffer(bisection_factor=2, bisection_threshold=10)
  sec1 = self.now.shift(seconds=-1).datetime
- a = _table_segment(self.connection, self.table_src_path, "id", "datetime", max_update=sec1, case_sensitive=False)
- b = _table_segment(self.connection, self.table_dst_path, "id", "datetime", max_update=sec1, case_sensitive=False)
+ a = _table_segment(
+ self.connection, self.table_src_path, "id", "datetime", max_update=sec1, case_sensitive=False
+ )
+ b = _table_segment(
+ self.connection, self.table_dst_path, "id", "datetime", max_update=sec1, case_sensitive=False
+ )
  assert a.count() == 4
  assert b.count() == 3
 
  assert not list(differ.diff_tables(a, a))
  self.assertEqual(len(list(differ.diff_tables(a, b))), 1)
 
- a = _table_segment(self.connection, self.table_src_path, "id", "datetime", min_update=sec1, case_sensitive=False)
- b = _table_segment(self.connection, self.table_dst_path, "id", "datetime", min_update=sec1, case_sensitive=False)
+ a = _table_segment(
+ self.connection, self.table_src_path, "id", "datetime", min_update=sec1, case_sensitive=False
+ )
+ b = _table_segment(
+ self.connection, self.table_dst_path, "id", "datetime", min_update=sec1, case_sensitive=False
+ )
  assert a.count() == 2
  assert b.count() == 2
  assert not list(differ.diff_tables(a, b))