ENH: Adding hint to to_sql

doc/source/whatsnew/v3.0.0.rst

@@ -156,6 +156,7 @@ All warnings for upcoming changes in pandas will have the base class :class:`pan
 
 Other enhancements
 ^^^^^^^^^^^^^^^^^^
+- :func:`DataFrame.to_sql` now accepts a ``hints`` parameter to pass database-specific query hints for optimizing insert performance. The hints are specified as a dictionary mapping dialect names to hint strings (e.g., ``{'oracle': '/*+ APPEND PARALLEL(4) */', 'mysql': 'DELAYED'}``). Users are responsible for providing correctly formatted hint strings for their target database (:issue:`61370`)
 - :func:`pandas.merge` propagates the ``attrs`` attribute to the result if all
  inputs have identical ``attrs``, as has so far already been the case for
  :func:`pandas.concat`.

pandas/core/generic.py

@@ -2798,6 +2798,7 @@ def to_sql(
  chunksize: int | None = None,
  dtype: DtypeArg | None = None,
  method: Literal["multi"] | Callable | None = None,
+ hints: dict[str, str] | None = None,
  ) -> int | None:
  """
  Write records stored in a DataFrame to a SQL database.
@@ -2861,6 +2862,21 @@ def to_sql(
 
  Details and a sample callable implementation can be found in the
  section :ref:`insert method <io.sql.method>`.
+ hints : dict[str, str], optional
+ Dictionary of SQL hints to optimize insertion performance, keyed by
+ database dialect name (e.g., 'oracle', 'mysql', 'postgresql', 'mssql').
+ Each value should be a complete hint string formatted exactly as required
+ by the target database. The user is responsible for providing correctly
+ formatted hint strings.
+
+ Examples: ``{'oracle': '/*+ APPEND PARALLEL(4) */', 'mysql': 'DELAYED'}``
+
+ .. note::
+ - Hints are database-specific and ignored for unsupported dialects.
+ - SQLite raises a ``UserWarning`` (hints not supported).
+ - ADBC connections raise ``NotImplementedError``.
+
+ .. versionadded:: 3.0.0
 
  Returns
  -------
@@ -3044,6 +3060,7 @@ def to_sql(
  chunksize=chunksize,
  dtype=dtype,
  method=method,
+ hints=hints,
  )
 
  @final

pandas/io/sql.py

@@ -18,7 +18,6 @@
  datetime,
  time,
 )
-from functools import partial
 import re
 from typing import (
  TYPE_CHECKING,
@@ -235,6 +234,18 @@ def _wrap_result_adbc(
  return df
 
 
+def _process_sql_hints(hints: dict[str, str] | None, dialect_name: str) -> str | None:
+ if hints is None:
+ return None
+
+ dialect_name = dialect_name.lower()
+ for key, value in hints.items():
+ if key.lower() == dialect_name:
+ return value
+
+ return None
+
+
 # -----------------------------------------------------------------------------
 # -- Read and write to DataFrames
 
@@ -753,6 +764,7 @@ def to_sql(
  dtype: DtypeArg | None = None,
  method: Literal["multi"] | Callable | None = None,
  engine: str = "auto",
+ hints: dict[str, str] | None = None,
  **engine_kwargs,
 ) -> int | None:
  """
@@ -813,6 +825,23 @@ def to_sql(
 
  .. versionadded:: 1.3.0
 
+ hints : dict[str, str], optional
+ SQL hints to optimize insertion performance, keyed by database dialect name.
+ Each hint should be a complete string formatted exactly as required by the
+ target database. The user is responsible for constructing dialect-specific
+ syntax.
+
+ Examples: ``{'oracle': '/*+ APPEND PARALLEL(4) */'}``
+ ``{'mysql': 'DELAYED'}``
+ ``{'mssql': 'WITH (TABLOCK)'}``
+
+ .. note::
+ - Hints are database-specific and will be ignored for unsupported dialects
+ - SQLite will raise a UserWarning (hints not supported)
+ - ADBC connections will raise NotImplementedError
+
+ .. versionadded:: 3.0.0
+
  **engine_kwargs
  Any additional kwargs are passed to the engine.
 
@@ -855,6 +884,7 @@ def to_sql(
  dtype=dtype,
  method=method,
  engine=engine,
+ hints=hints,
  **engine_kwargs,
  )
 
@@ -1004,7 +1034,13 @@ def create(self) -> None:
  else:
  self._execute_create()
 
- def _execute_insert(self, conn, keys: list[str], data_iter) -> int:
+ def _execute_insert(
+ self,
+ conn,
+ keys: list[str],
+ data_iter,
+ hint_str: str | None = None,
+ ) -> int:
  """
  Execute SQL statement inserting data
 
@@ -1016,11 +1052,23 @@ def _execute_insert(self, conn, keys: list[str], data_iter) -> int:
  data_iter : generator of list
  Each item contains a list of values to be inserted
  """
- data = [dict(zip(keys, row, strict=True)) for row in data_iter]
- result = self.pd_sql.execute(self.table.insert(), data)
+ data = [dict(zip(keys, row, strict=False)) for row in data_iter]
+
+ if hint_str:
+ stmt = self.table.insert().prefix_with(hint_str)
+ else:
+ stmt = self.table.insert()
+
+ result = self.pd_sql.execute(stmt, data)
  return result.rowcount
 
- def _execute_insert_multi(self, conn, keys: list[str], data_iter) -> int:
+ def _execute_insert_multi(
+ self,
+ conn,
+ keys: list[str],
+ data_iter,
+ hint_str: str | None = None,
+ ) -> int:
  """
  Alternative to _execute_insert for DBs support multi-value INSERT.
 
@@ -1029,11 +1077,15 @@ def _execute_insert_multi(self, conn, keys: list[str], data_iter) -> int:
  but performance degrades quickly with increase of columns.
 
  """
-
  from sqlalchemy import insert
 
- data = [dict(zip(keys, row, strict=True)) for row in data_iter]
- stmt = insert(self.table).values(data)
+ data = [dict(zip(keys, row, strict=False)) for row in data_iter]
+
+ if hint_str:
+ stmt = insert(self.table).values(data).prefix_with(hint_str)
+ else:
+ stmt = insert(self.table).values(data)
+
  result = self.pd_sql.execute(stmt)
  return result.rowcount
 
@@ -1090,14 +1142,20 @@ def insert(
  self,
  chunksize: int | None = None,
  method: Literal["multi"] | Callable | None = None,
+ hints: dict[str, str] | None = None,
+ dialect_name: str | None = None,
  ) -> int | None:
  # set insert method
  if method is None:
  exec_insert = self._execute_insert
  elif method == "multi":
  exec_insert = self._execute_insert_multi
  elif callable(method):
- exec_insert = partial(method, self)
+
+ def callable_wrapper(conn, keys, data_iter, hint_str=None):
+ return method(self, conn, keys, data_iter)
+
+ exec_insert = callable_wrapper
  else:
  raise ValueError(f"Invalid parameter `method`: {method}")
 
@@ -1114,6 +1172,9 @@ def insert(
  raise ValueError("chunksize argument should be non-zero")
 
  chunks = (nrows // chunksize) + 1
+
+ hint_str = _process_sql_hints(hints, dialect_name) if dialect_name else None
+
  total_inserted = None
  with self.pd_sql.run_transaction() as conn:
  for i in range(chunks):
@@ -1125,7 +1186,7 @@ def insert(
  chunk_iter = zip(
  *(arr[start_i:end_i] for arr in data_list), strict=True
  )
- num_inserted = exec_insert(conn, keys, chunk_iter)
+ num_inserted = exec_insert(conn, keys, chunk_iter, hint_str)
  # GH 46891
  if num_inserted is not None:
  if total_inserted is None:
@@ -1509,6 +1570,7 @@ def to_sql(
  chunksize: int | None = None,
  dtype: DtypeArg | None = None,
  method: Literal["multi"] | Callable | None = None,
+ hints: dict[str, str] | None = None,
  engine: str = "auto",
  **engine_kwargs,
  ) -> int | None:
@@ -1545,6 +1607,8 @@ def insert_records(
  schema=None,
  chunksize: int | None = None,
  method=None,
+ hints: dict[str, str] | None = None,
+ dialect_name: str | None = None,
  **engine_kwargs,
  ) -> int | None:
  """
@@ -1569,6 +1633,8 @@ def insert_records(
  schema=None,
  chunksize: int | None = None,
  method=None,
+ hints: dict[str, str] | None = None,
+ dialect_name: str | None = None,
  **engine_kwargs,
  ) -> int | None:
  from sqlalchemy import exc
@@ -1980,6 +2046,7 @@ def to_sql(
  chunksize: int | None = None,
  dtype: DtypeArg | None = None,
  method: Literal["multi"] | Callable | None = None,
+ hints: dict[str, str] | None = None,
  engine: str = "auto",
  **engine_kwargs,
  ) -> int | None:
@@ -2053,6 +2120,8 @@ def to_sql(
  schema=schema,
  chunksize=chunksize,
  method=method,
+ hints=hints,
+ dialect_name=self.con.dialect.name,
  **engine_kwargs,
  )
 
@@ -2344,6 +2413,7 @@ def to_sql(
  chunksize: int | None = None,
  dtype: DtypeArg | None = None,
  method: Literal["multi"] | Callable | None = None,
+ hints: dict[str, str] | None = None,
  engine: str = "auto",
  **engine_kwargs,
  ) -> int | None:
@@ -2394,6 +2464,8 @@ def to_sql(
  raise NotImplementedError(
  "engine != 'auto' not implemented for ADBC drivers"
  )
+ if hints:
+ raise NotImplementedError("'hints' is not implemented for ADBC drivers")
 
  if schema:
  table_name = f"{schema}.{name}"
@@ -2575,7 +2647,9 @@ def insert_statement(self, *, num_rows: int) -> str:
  )
  return insert_statement
 
- def _execute_insert(self, conn, keys, data_iter) -> int:
+ def _execute_insert(
+ self, conn, keys: list[str], data_iter, hint_str: str | None = None
+ ) -> int:
  from sqlite3 import Error
 
  data_list = list(data_iter)
@@ -2585,7 +2659,9 @@ def _execute_insert(self, conn, keys, data_iter) -> int:
  raise DatabaseError("Execution failed") from exc
  return conn.rowcount
 
- def _execute_insert_multi(self, conn, keys, data_iter) -> int:
+ def _execute_insert_multi(
+ self, conn, keys: list[str], data_iter, hint_str: str | None = None
+ ) -> int:
  data_list = list(data_iter)
  flattened_data = [x for row in data_list for x in row]
  conn.execute(self.insert_statement(num_rows=len(data_list)), flattened_data)
@@ -2821,6 +2897,7 @@ def to_sql(
  chunksize: int | None = None,
  dtype: DtypeArg | None = None,
  method: Literal["multi"] | Callable | None = None,
+ hints: dict[str, str] | None = None,
  engine: str = "auto",
  **engine_kwargs,
  ) -> int | None:
@@ -2863,6 +2940,13 @@ def to_sql(
  Details and a sample callable implementation can be found in the
  section :ref:`insert method <io.sql.method>`.
  """
+ if hints:
+ warnings.warn(
+ "SQL hints are not supported for SQLite and will be ignored.",
+ UserWarning,
+ stacklevel=find_stack_level(),
+ )
+
  if dtype:
  if not is_dict_like(dtype):
  # error: Value expression in dictionary comprehension has incompatible