Merge branch 'master' into pass-inline-key-to-snowflake

Author: vvkh

.github/workflows/formatter.yml

@@ -7,17 +7,16 @@ on:
 
 jobs:
  linter_name:
- name: runner / black
+ name: runner / ruff
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v2
- - name: Check files using the black formatter
- uses: rickstaa/action-black@v1
- id: black_formatter
+ - uses: actions/checkout@v3
+ - name: Check files using the ruff formatter
+ uses: chartboost/ruff-action@v1
+ id: ruff_formatter
  with:
- black_args: ". -l 120"
- - name: Annotate diff changes using reviewdog
- if: steps.black_formatter.outputs.is_formatted == 'true'
- uses: reviewdog/action-suggester@v1
+ args: format
+ - name: Auto commit ruff formatting
+ uses: stefanzweifel/git-auto-commit-action@v5
  with:
- tool_name: blackfmt
+ commit_message: 'style fixes by ruff'

.gitignore

@@ -147,3 +147,6 @@ benchmark_*.png
 
 # VSCode
 .vscode
+
+# History
+.history

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+default_language_version:
+ python: python3
+
+repos:
+# Run the Ruff linter.
+- repo: https://github.com/astral-sh/ruff-pre-commit
+ # Ruff version.
+ rev: v0.1.2
+ hooks:
+ # Run the Ruff formatter.
+ - id: ruff-format

CONTRIBUTING.md

@@ -54,6 +54,8 @@ Once inside, you can install the dependencies.
 
 - Option 2: Run `pip install -e .` to install them, and data-diff, in the global context.
 
+- Run `pre-commit install` to automatically format your code before committing.
+
 At the bare minimum, you need MySQL to run the tests.
 
 You can create a local MySQL instance using `docker-compose up mysql`. The URI for it will be `mysql://mysql:Password1@localhost/mysql`. If you're using a different server, make sure to update `TEST_MYSQL_CONN_STRING` in `tests/common.py`. For your convenience, we recommend creating `tests/local_settings.py`, and to override the value there.

README.md

@@ -72,7 +72,7 @@ More information about the algorithm and performance considerations can be found
 # Get started
 
 ## Validating dbt model changes between dev and prod
-⚡ Looking to use `data-diff` in dbt development? Head over to [our `data-diff` + `dbt` documentation](https://docs.datafold.com/development_testing/open_source/) to get started!
+⚡ Looking to use `data-diff` in dbt development? Head over to [our `data-diff` + `dbt` documentation](https://docs.datafold.com/development_testing/how_it_works) to get started!
 
 ## Compare data tables between databases
 🔀 To compare data between databases, install `data-diff` with specific database adapters, e.g.:

data_diff/__main__.py

@@ -267,6 +267,20 @@ def write_usage(self, prog: str, args: str = "", prefix: Optional[str] = None) -
  metavar="PATH",
  help="Specify manifest to utilize for 'prod' comparison paths instead of using configuration.",
 )
+@click.option(
+ "-pd",
+ "--prod-database",
+ "prod_database",
+ default=None,
+ help="Override the dbt production database configuration within dbt_project.yml",
+)
+@click.option(
+ "-ps",
+ "--prod-schema",
+ "prod_schema",
+ default=None,
+ help="Override the dbt production schema configuration within dbt_project.yml",
+)
 def main(conf, run, **kw):
  log_handlers = _get_log_handlers(kw["dbt"])
  if kw["table2"] is None and kw["database2"]:
@@ -323,6 +337,8 @@ def main(conf, run, **kw):
  where_flag=kw["where"],
  stats_flag=kw["stats"],
  columns_flag=kw["columns"],
+ production_database_flag=kw["prod_database"],
+ production_schema_flag=kw["prod_schema"],
  )
  else:
  return _data_diff(
@@ -366,6 +382,8 @@ def _data_diff(
  cloud,
  dbt_profiles_dir,
  dbt_project_dir,
+ prod_database,
+ prod_schema,
  select,
  state,
  threads1=None,
@@ -519,7 +537,7 @@ def _data_diff(
 
  else:
  for op, values in diff_iter:
- color = COLOR_SCHEME[op]
+ color = COLOR_SCHEME.get(op, "grey62")
 
  if json_output:
  jsonl = json.dumps([op, list(values)])

data_diff/abcs/database_types.py

@@ -1,6 +1,6 @@
 import decimal
 from abc import ABC, abstractmethod
-from typing import Tuple, Union
+from typing import List, Optional, Tuple, Type, TypeVar, Union
 from datetime import datetime
 
 import attrs
@@ -12,9 +12,24 @@
 DbKey = Union[int, str, bytes, ArithUUID, ArithAlphanumeric]
 DbTime = datetime
 
+N = TypeVar("N")
 
-@attrs.define(frozen=True)
+
+@attrs.define(frozen=True, kw_only=True)
 class ColType:
+ # Arbitrary metadata added and fetched at runtime.
+ _notes: List[N] = attrs.field(factory=list, init=False, hash=False, eq=False)
+
+ def add_note(self, note: N) -> None:
+ self._notes.append(note)
+
+ def get_note(self, cls: Type[N]) -> Optional[N]:
+ """Get the latest added note of type ``cls`` or its descendants."""
+ for note in reversed(self._notes):
+ if isinstance(note, cls):
+ return note
+ return None
+
  @property
  def supported(self) -> bool:
  return True

data_diff/abcs/mixins.py

@@ -1,4 +1,5 @@
 from data_diff.databases.base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, QueryError, ConnectError, BaseDialect, Database
+from data_diff.databases.base import CHECKSUM_OFFSET
 from data_diff.databases._connect import connect as connect
 from data_diff.databases._connect import Connect as Connect
 from data_diff.databases.postgresql import PostgreSQL as PostgreSQL