Re-land PyCapsule (#52021)

* Revert "Revert "Use PyCapsule for internal datetime functions" (#51951)" This reverts commit 8487b2e. * Revert "Revert "Re-enable parallel builds in CI" (#51952)" This reverts commit c8ea34c. * Revert "Revert "CI: parallel build follow up" (#51984)" This reverts commit 93b5135.

Author: mroeschke

.circleci/setup_env.sh

@@ -55,8 +55,7 @@ if pip list | grep -q ^pandas; then
 fi
 
 echo "Build extensions"
-# GH 47305: Parallel build can causes flaky ImportError from pandas/_libs/tslibs
-python setup.py build_ext -q -j1
+python setup.py build_ext -q -j4
 
 echo "Install pandas"
 python -m pip install --no-build-isolation --no-use-pep517 -e .

.github/actions/build_pandas/action.yml

@@ -16,7 +16,5 @@ runs:
  python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
  shell: bash -el {0}
  env:
- # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873
- # GH 47305: Parallel build causes flaky ImportError: /home/runner/work/pandas/pandas/pandas/_libs/tslibs/timestamps.cpython-38-x86_64-linux-gnu.so: undefined symbol: pandas_datetime_to_datetimestruct
- N_JOBS: 1
- #N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }}
+ # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+ N_JOBS: ${{ runner.os == 'macOS' && 3 || 2 }}

.github/actions/setup-conda/action.yml

@@ -30,7 +30,7 @@ runs:
  environment-name: ${{ inputs.environment-name }}
  extra-specs: ${{ inputs.extra-specs }}
  channels: conda-forge
- channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }}
+ channel-priority: 'strict'
  condarc-file: ci/condarc.yml
  cache-env: true
  cache-downloads: true

.github/workflows/32-bit-linux.yml

@@ -40,7 +40,7 @@ jobs:
  python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
  python -m pip install versioneer[toml] && \
  python -m pip install cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.34.2 && \
- python setup.py build_ext -q -j1 && \
+ python setup.py build_ext -q -j$(nproc) && \
  python -m pip install --no-build-isolation --no-use-pep517 -e . && \
  python -m pip list && \
  export PANDAS_CI=1 && \

.github/workflows/python-dev.yml

@@ -82,10 +82,9 @@ jobs:
  python -m pip install python-dateutil pytz cython hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
  python -m pip list
 
- # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs
  - name: Build Pandas
  run: |
- python setup.py build_ext -q -j1
+ python setup.py build_ext -q -j4
  python -m pip install -e . --no-build-isolation --no-use-pep517 --no-index
 
  - name: Build Version

MANIFEST.in

@@ -58,3 +58,5 @@ prune pandas/tests/io/parser/data
 # Selectively re-add *.cxx files that were excluded above
 graft pandas/_libs/src
 graft pandas/_libs/tslibs/src
+include pandas/_libs/pd_parser.h
+include pandas/_libs/pd_parser.c

pandas/_libs/__init__.py

@@ -10,6 +10,11 @@
 ]
 
 
+# Below imports needs to happen first to ensure pandas top level
+# module gets monkeypatched with the pandas_datetime_CAPI
+# see pandas_datetime_exec in pd_datetime.c
+import pandas._libs.pandas_parser # noqa # isort: skip # type: ignore[reportUnusedImport]
+import pandas._libs.pandas_datetime # noqa # isort: skip # type: ignore[reportUnusedImport]
 from pandas._libs.interval import Interval
 from pandas._libs.tslibs import (
  NaT,

pandas/_libs/index.pyx

@@ -20,7 +20,12 @@ from pandas._libs.tslibs.nattype cimport c_NaT as NaT
 from pandas._libs.tslibs.np_datetime cimport (
  NPY_DATETIMEUNIT,
  get_unit_from_dtype,
+ import_pandas_datetime,
 )
+
+import_pandas_datetime()
+
+
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.timedeltas cimport _Timedelta
 from pandas._libs.tslibs.timestamps cimport _Timestamp

pandas/_libs/lib.pyx

@@ -88,9 +88,11 @@ cdef extern from "numpy/arrayobject.h":
 cdef extern from "numpy/ndarrayobject.h":
  bint PyArray_CheckScalar(obj) nogil
 
-
-cdef extern from "src/parse_helper.h":
+cdef extern from "pd_parser.h":
  int floatify(object, float64_t *result, int *maybe_int) except -1
+ void PandasParser_IMPORT()
+
+PandasParser_IMPORT
 
 from pandas._libs cimport util
 from pandas._libs.util cimport (

pandas/_libs/missing.pyx

@@ -34,8 +34,11 @@ from pandas._libs.tslibs.np_datetime cimport (
  get_datetime64_unit,
  get_datetime64_value,
  get_timedelta64_value,
+ import_pandas_datetime,
 )
 
+import_pandas_datetime()
+
 from pandas._libs.ops_dispatch import maybe_dispatch_ufunc_to_dunder_op
 
 cdef: