Added support for ingesting Arrow data types date32 and date64 (#534).

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -41,6 +41,9 @@ Common Changes
 #) Added support for all of the signed and unsigned fixed width integer types
  when ingesting data frames supporting the Arrow PyCapsule interface.
  Previously only ``int64`` was supported.
+#) Added support for types ``date32`` and ``date64`` when ingesting data
+ frames supporting the Arrow PyCapsule interface as requested
+ (`issue 534 <https://github.com/oracle/python-oracledb/issues/534>`__).
 #) Added ``fetch_lobs`` and ``fetch_decimals`` parameters where applicable to
  the methods used for fetching rows or data frames from the database. Note
  that for the creation of pipeline operations, if these parameters are not

src/oracledb/arrow_impl.pxd

@@ -62,6 +62,8 @@ cdef extern from "nanoarrow.h":
  cpdef enum ArrowType:
  NANOARROW_TYPE_BOOL
  NANOARROW_TYPE_BINARY
+ NANOARROW_TYPE_DATE32
+ NANOARROW_TYPE_DATE64
  NANOARROW_TYPE_DECIMAL128
  NANOARROW_TYPE_DOUBLE
  NANOARROW_TYPE_FIXED_SIZE_BINARY

src/oracledb/base_impl.pyx

@@ -55,6 +55,8 @@ from .arrow_impl cimport (
  NANOARROW_TYPE_NA,
  NANOARROW_TYPE_BOOL,
  NANOARROW_TYPE_BINARY,
+ NANOARROW_TYPE_DATE32,
+ NANOARROW_TYPE_DATE64,
  NANOARROW_TYPE_DECIMAL128,
  NANOARROW_TYPE_DOUBLE,
  NANOARROW_TYPE_FIXED_SIZE_BINARY,

src/oracledb/impl/arrow/array.pyx

@@ -365,7 +365,7 @@ cdef class ArrowArrayImpl:
  value[0] = (<int8_t*> ptr)[index]
  elif arrow_type == NANOARROW_TYPE_INT16:
  value[0] = (<int16_t*> ptr)[index]
- elif arrow_type == NANOARROW_TYPE_INT32:
+ elif arrow_type in (NANOARROW_TYPE_INT32, NANOARROW_TYPE_DATE32):
  value[0] = (<int32_t*> ptr)[index]
  else:
  value[0] = (<int64_t*> ptr)[index]

src/oracledb/impl/arrow/schema.pyx

@@ -129,6 +129,8 @@ cdef class ArrowSchemaImpl:
  self.fixed_size = schema_view.fixed_size
  if schema_view.type == NANOARROW_TYPE_TIMESTAMP:
  self._set_time_unit(schema_view.time_unit)
+ elif schema_view.type == NANOARROW_TYPE_DATE64:
+ self._set_time_unit(NANOARROW_TIME_UNIT_MILLI)
  elif schema_view.type in (
  NANOARROW_TYPE_FIXED_SIZE_LIST,
  NANOARROW_TYPE_LIST
@@ -143,6 +145,8 @@ cdef class ArrowSchemaImpl:
  NANOARROW_TYPE_BINARY,
  NANOARROW_TYPE_BOOL,
  NANOARROW_TYPE_DECIMAL128,
+ NANOARROW_TYPE_DATE32,
+ NANOARROW_TYPE_DATE64,
  NANOARROW_TYPE_DOUBLE,
  NANOARROW_TYPE_FIXED_SIZE_BINARY,
  NANOARROW_TYPE_FLOAT,

src/oracledb/impl/base/converters.pyx

@@ -90,7 +90,7 @@ cdef object convert_arrow_to_oracle_data(OracleMetadata metadata,
  rb = &data.buffer.as_raw_bytes
  array_impl.get_bytes(array_index, &data.is_null, <char**> &rb.ptr,
  &rb.num_bytes)
- elif arrow_type == NANOARROW_TYPE_TIMESTAMP:
+ elif arrow_type in (NANOARROW_TYPE_TIMESTAMP, NANOARROW_TYPE_DATE64):
  array_impl.get_int(arrow_type, array_index, &data.is_null, &int_value)
  if not data.is_null:
  seconds = int_value // array_impl.schema_impl.time_factor
@@ -103,6 +103,10 @@ cdef object convert_arrow_to_oracle_data(OracleMetadata metadata,
  useconds //= 1_000
  return EPOCH_DATE + \
  cydatetime.timedelta_new(days, seconds, useconds)
+ elif arrow_type == NANOARROW_TYPE_DATE32:
+ array_impl.get_int(arrow_type, array_index, &data.is_null, &int_value)
+ if not data.is_null:
+ return EPOCH_DATE + cydatetime.timedelta_new(int_value, 0, 0)
  elif arrow_type == NANOARROW_TYPE_DECIMAL128:
  temp_bytes = array_impl.get_decimal(array_index, &data.is_null)
  if not data.is_null:

src/oracledb/impl/base/metadata.pyx

@@ -187,6 +187,8 @@ cdef class OracleMetadata:
  metadata.dbtype = DB_TYPE_BOOLEAN
  elif arrow_type == NANOARROW_TYPE_TIMESTAMP:
  metadata.dbtype = DB_TYPE_TIMESTAMP
+ elif arrow_type in (NANOARROW_TYPE_DATE32, NANOARROW_TYPE_DATE64):
+ metadata.dbtype = DB_TYPE_DATE
  elif arrow_type == NANOARROW_TYPE_LARGE_STRING:
  metadata.dbtype = DB_TYPE_LONG
  elif arrow_type == NANOARROW_TYPE_LARGE_BINARY:

tests/test_8900_dataframe_ingestion.py

@@ -898,6 +898,53 @@ def test_8919(self):
  fetched_values = [int(s) for s, in self.cursor]
  self.assertEqual(fetched_values, values)
 
+ def test_8920(self):
+ "8920 - test ingestion with alternative date types"
+ scenarios = [
+ (
+ [
+ datetime.datetime(1915, 9, 11),
+ None,
+ datetime.datetime(2045, 2, 28),
+ ],
+ pyarrow.date32(),
+ ),
+ (
+ [
+ datetime.datetime(1905, 3, 30),
+ None,
+ datetime.datetime(2060, 10, 5),
+ ],
+ pyarrow.date64(),
+ ),
+ ]
+ names = ["Id", "DateOfBirth"]
+ for values, dtype in scenarios:
+ with self.subTest(dtype=str(dtype)):
+ arrays = [
+ pyarrow.array([1, 2, 3], pyarrow.int8()),
+ pyarrow.array(values, dtype),
+ ]
+ df = pyarrow.table(arrays, names)
+ self.cursor.execute("delete from TestDataFrame")
+ self.cursor.executemany(
+ """
+ insert into TestDataFrame (Id, DateOfBirth)
+ values (:1, :2)
+ """,
+ df,
+ )
+ self.conn.commit()
+ self.cursor.execute(
+ """
+ select DateOfBirth
+ from TestDataFrame
+ order by Id
+ """
+ )
+ fetched_values = [d for d, in self.cursor]
+ self.assertEqual(fetched_values, values)
+
 
 if __name__ == "__main__":
  test_env.run_test_cases()

tests/test_9000_dataframe_ingestion_async.py

@@ -899,6 +899,53 @@ async def test_9019(self):
  fetched_values = [int(s) async for s, in self.cursor]
  self.assertEqual(fetched_values, values)
 
+ async def test_9020(self):
+ "9020 - test ingestion with alternative date types"
+ scenarios = [
+ (
+ [
+ datetime.datetime(1915, 9, 11),
+ None,
+ datetime.datetime(2045, 2, 28),
+ ],
+ pyarrow.date32(),
+ ),
+ (
+ [
+ datetime.datetime(1905, 3, 30),
+ None,
+ datetime.datetime(2060, 10, 5),
+ ],
+ pyarrow.date64(),
+ ),
+ ]
+ names = ["Id", "DateOfBirth"]
+ for values, dtype in scenarios:
+ with self.subTest(dtype=str(dtype)):
+ arrays = [
+ pyarrow.array([1, 2, 3], pyarrow.int8()),
+ pyarrow.array(values, dtype),
+ ]
+ df = pyarrow.table(arrays, names)
+ await self.cursor.execute("delete from TestDataFrame")
+ await self.cursor.executemany(
+ """
+ insert into TestDataFrame (Id, DateOfBirth)
+ values (:1, :2)
+ """,
+ df,
+ )
+ await self.conn.commit()
+ await self.cursor.execute(
+ """
+ select DateOfBirth
+ from TestDataFrame
+ order by Id
+ """
+ )
+ fetched_values = [d async for d, in self.cursor]
+ self.assertEqual(fetched_values, values)
+
 
 if __name__ == "__main__":
  test_env.run_test_cases()