fix float value precision calculation

Author: pik94

data_diff/databases/databricks.py

@@ -1,3 +1,4 @@
+import math
 from typing import Dict, Sequence
 import logging
 
@@ -61,11 +62,14 @@ def normalize_timestamp(self, value: str, coltype: TemporalType) -> str:
  return f"date_format({value}, 'yyyy-MM-dd HH:mm:ss.{precision_format}')"
 
  def normalize_number(self, value: str, coltype: NumericType) -> str:
- return self.to_string(f"cast({value} as decimal(38, {coltype.precision}))")
+ value = f"cast({value} as decimal(38, {coltype.precision}))"
+ if coltype.precision > 0:
+ value = f"format_number({value}, {coltype.precision})"
+ return f"replace({self.to_string(value)}, ',', '')"
 
  def _convert_db_precision_to_digits(self, p: int) -> int:
- # Subtracting 1 due to wierd precision issues
- return max(super()._convert_db_precision_to_digits(p) - 1, 0)
+ # Subtracting 2 due to wierd precision issues
+ return max(super()._convert_db_precision_to_digits(p) - 2, 0)
 
 
 class Databricks(ThreadedDatabase):
@@ -103,8 +107,6 @@ def query_table_schema(self, path: DbPath) -> Dict[str, tuple]:
  cursor.columns(catalog_name=self._args['catalog'], schema_name=schema, table_name=table)
  try:
  rows = cursor.fetchall()
- except:
- rows = None
  finally:
  conn.close()
  if not rows:
@@ -129,7 +131,7 @@ def _process_table_schema(
  row = (row[0], row_type, None, None, 0)
 
  elif issubclass(type_cls, Float):
- numeric_precision = self._convert_db_precision_to_digits(row[2])
+ numeric_precision = math.ceil(row[2] / math.log(2, 10))
  row = (row[0], row_type, None, numeric_precision, None)
 
  elif issubclass(type_cls, Decimal):