bpo-43466: Add --with-openssl-rpath configure option (GH-24820)

Author: tiran

Doc/using/unix.rst

@@ -134,3 +134,53 @@ some Unices may not have the :program:`env` command, so you may need to hardcode
 ``/usr/bin/python3`` as the interpreter path.
 
 To use shell commands in your Python scripts, look at the :mod:`subprocess` module.
+
+
+Custom OpenSSL
+==============
+
+1. To use your vendor's OpenSSL configuration and system trust store, locate
+ the directory with ``openssl.cnf`` file or symlink in ``/etc``. On most
+ distribution the file is either in ``/etc/ssl`` or ``/etc/pki/tls``. The
+ directory should also contain a ``cert.pem`` file and/or a ``certs``
+ directory.
+
+ .. code-block:: shell-session
+
+ $ find /etc/ -name openssl.cnf -printf "%h\n"
+ /etc/ssl
+
+2. Download, build, and install OpenSSL. Make sure you use ``install_sw`` and
+ not ``install``. The ``install_sw`` target does not override
+ ``openssl.cnf``.
+
+ .. code-block:: shell-session
+
+ $ curl -O https://www.openssl.org/source/openssl-VERSION.tar.gz
+ $ tar xzf openssl-VERSION
+ $ pushd openssl-VERSION
+ $ ./config \
+ --prefix=/usr/local/custom-openssl \
+ --openssldir=/etc/ssl
+ $ make -j1 depend
+ $ make -j8
+ $ make install_sw
+ $ popd
+
+3. Build Python with custom OpenSSL
+
+ .. code-block:: shell-session
+
+ $ pushd python-3.x.x
+ $ ./configure -C \
+ --with-openssl=/usr/local/custom-openssl \
+ --with-openssl-rpath=auto \
+ --prefix=/usr/local/python-3.x.x
+ $ make -j8
+ $ make altinstall
+
+.. note::
+
+ Patch releases of OpenSSL have a backwards compatible ABI. You don't need
+ to recompile Python to update OpenSSL. It's sufficient to replace the
+ custom OpenSSL installation with a newer version.

Doc/whatsnew/3.10.rst

@@ -1181,6 +1181,12 @@ Build Changes
  and ``--with-tcltk-libs`` configuration options.
  (Contributed by Manolis Stamatogiannakis in :issue:`42603`.)
 
+* Add ``--with-openssl-rpath`` option to ``configure`` script. The option
+ simplifies building Python with a custom OpenSSL installation, e.g.
+ ``./configure --with-openssl=/path/to/openssl --with-openssl-rpath=auto``.
+ (Contributed by Christian Heimes in :issue:`43466`.)
+
+
 
 C API Changes
 =============

Makefile.pre.in

@@ -202,6 +202,7 @@ ENSUREPIP= @ENSUREPIP@
 OPENSSL_INCLUDES=@OPENSSL_INCLUDES@
 OPENSSL_LIBS=@OPENSSL_LIBS@
 OPENSSL_LDFLAGS=@OPENSSL_LDFLAGS@
+OPENSSL_RPATH=@OPENSSL_RPATH@
 
 # Default zoneinfo.TZPATH. Added here to expose it in sysconfig.get_config_var
 TZPATH=@TZPATH@

Misc/NEWS.d/next/Build/2021-03-11-00-14-47.bpo-43466.N861Z5.rst

@@ -0,0 +1 @@
+The ``configure`` script now supports ``--with-openssl-rpath`` option.

Tools/ssl/multissltests.py

@@ -48,16 +48,16 @@
 ]
 
 OPENSSL_RECENT_VERSIONS = [
- "1.1.1g",
- # "3.0.0-alpha2"
+ "1.1.1j",
+ # "3.0.0-alpha12"
 ]
 
 LIBRESSL_OLD_VERSIONS = [
  "2.9.2",
 ]
 
 LIBRESSL_RECENT_VERSIONS = [
- "3.1.0",
+ "3.2.4",
 ]
 
 # store files in ../multissl
@@ -169,7 +169,9 @@ class AbstractBuilder(object):
  url_templates = None
  src_template = None
  build_template = None
+ depend_target = None
  install_target = 'install'
+ jobs = os.cpu_count()
 
  module_files = ("Modules/_ssl.c",
  "Modules/_hashopenssl.c")
@@ -321,8 +323,11 @@ def _build_src(self):
  if self.system:
  env['SYSTEM'] = self.system
  self._subprocess_call(cmd, cwd=cwd, env=env)
- # Old OpenSSL versions do not support parallel builds.
- self._subprocess_call(["make", "-j1"], cwd=cwd, env=env)
+ if self.depend_target:
+ self._subprocess_call(
+ ["make", "-j1", self.depend_target], cwd=cwd, env=env
+ )
+ self._subprocess_call(["make", f"-j{self.jobs}"], cwd=cwd, env=env)
 
  def _make_install(self):
  self._subprocess_call(
@@ -409,6 +414,7 @@ class BuildOpenSSL(AbstractBuilder):
  build_template = "openssl-{}"
  # only install software, skip docs
  install_target = 'install_sw'
+ depend_target = 'depend'
 
  def _post_install(self):
  if self.version.startswith("3.0"):
@@ -434,11 +440,11 @@ def _post_install_300(self):
  self.openssl_cli, "fipsinstall",
  "-out", fipsinstall_cnf,
  "-module", fips_mod,
- "-provider_name", "fips",
- "-mac_name", "HMAC",
- "-macopt", "digest:SHA256",
- "-macopt", "hexkey:00",
- "-section_name", "fips_sect"
+ # "-provider_name", "fips",
+ # "-mac_name", "HMAC",
+ # "-macopt", "digest:SHA256",
+ # "-macopt", "hexkey:00",
+ # "-section_name", "fips_sect"
  ]
  )
  with open(openssl_fips_cnf, "w") as f:

aclocal.m4

@@ -67,7 +67,7 @@ AS_VAR_POPDEF([CACHEVAR])dnl
 ])dnl AX_CHECK_COMPILE_FLAGS
 
 # pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*-
-# serial 12 (pkg-config-0.29.2)
+# serial 11 (pkg-config-0.29.1)
 
 dnl Copyright © 2004 Scott James Remnant <scott@netsplit.com>.
 dnl Copyright © 2012-2015 Dan Nicholson <dbn.lists@gmail.com>
@@ -109,7 +109,7 @@ dnl
 dnl See the "Since" comment for each macro you use to see what version
 dnl of the macros you require.
 m4_defun([PKG_PREREQ],
-[m4_define([PKG_MACROS_VERSION], [0.29.2])
+[m4_define([PKG_MACROS_VERSION], [0.29.1])
 m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1,
  [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])])
 ])dnl PKG_PREREQ
@@ -210,7 +210,7 @@ AC_ARG_VAR([$1][_CFLAGS], [C compiler flags for $1, overriding pkg-config])dnl
 AC_ARG_VAR([$1][_LIBS], [linker flags for $1, overriding pkg-config])dnl
 
 pkg_failed=no
-AC_MSG_CHECKING([for $2])
+AC_MSG_CHECKING([for $1])
 
 _PKG_CONFIG([$1][_CFLAGS], [cflags], [$2])
 _PKG_CONFIG([$1][_LIBS], [libs], [$2])
@@ -220,11 +220,11 @@ and $1[]_LIBS to avoid the need to call pkg-config.
 See the pkg-config man page for more details.])
 
 if test $pkg_failed = yes; then
-  AC_MSG_RESULT([no])
+ AC_MSG_RESULT([no])
  _PKG_SHORT_ERRORS_SUPPORTED
  if test $_pkg_short_errors_supported = yes; then
  $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "$2" 2>&1`
- else
+ else 
  $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "$2" 2>&1`
  fi
 # Put the nasty error message in config.log where it belongs
@@ -241,7 +241,7 @@ installed software in a non-standard prefix.
 _PKG_TEXT])[]dnl
  ])
 elif test $pkg_failed = untried; then
-  AC_MSG_RESULT([no])
+ AC_MSG_RESULT([no])
 m4_default([$4], [AC_MSG_FAILURE(
 [The pkg-config script could not be found or is too old. Make sure it
 is in your PATH or set the PKG_CONFIG environment variable to the full
@@ -342,5 +342,73 @@ AS_VAR_COPY([$1], [pkg_cv_][$1])
 AS_VAR_IF([$1], [""], [$5], [$4])dnl
 ])dnl PKG_CHECK_VAR
 
+dnl PKG_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl [ACTION-IF-FOUND],[ACTION-IF-NOT-FOUND],
+dnl [DESCRIPTION], [DEFAULT])
+dnl ------------------------------------------
+dnl
+dnl Prepare a "--with-" configure option using the lowercase
+dnl [VARIABLE-PREFIX] name, merging the behaviour of AC_ARG_WITH and
+dnl PKG_CHECK_MODULES in a single macro.
+AC_DEFUN([PKG_WITH_MODULES],
+[
+m4_pushdef([with_arg], m4_tolower([$1]))
+
+m4_pushdef([description],
+ [m4_default([$5], [build with ]with_arg[ support])])
+
+m4_pushdef([def_arg], [m4_default([$6], [auto])])
+m4_pushdef([def_action_if_found], [AS_TR_SH([with_]with_arg)=yes])
+m4_pushdef([def_action_if_not_found], [AS_TR_SH([with_]with_arg)=no])
+
+m4_case(def_arg,
+ [yes],[m4_pushdef([with_without], [--without-]with_arg)],
+ [m4_pushdef([with_without],[--with-]with_arg)])
+
+AC_ARG_WITH(with_arg,
+ AS_HELP_STRING(with_without, description[ @<:@default=]def_arg[@:>@]),,
+ [AS_TR_SH([with_]with_arg)=def_arg])
+
+AS_CASE([$AS_TR_SH([with_]with_arg)],
+ [yes],[PKG_CHECK_MODULES([$1],[$2],$3,$4)],
+ [auto],[PKG_CHECK_MODULES([$1],[$2],
+ [m4_n([def_action_if_found]) $3],
+ [m4_n([def_action_if_not_found]) $4])])
+
+m4_popdef([with_arg])
+m4_popdef([description])
+m4_popdef([def_arg])
+
+])dnl PKG_WITH_MODULES
+
+dnl PKG_HAVE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl [DESCRIPTION], [DEFAULT])
+dnl -----------------------------------------------
+dnl
+dnl Convenience macro to trigger AM_CONDITIONAL after PKG_WITH_MODULES
+dnl check._[VARIABLE-PREFIX] is exported as make variable.
+AC_DEFUN([PKG_HAVE_WITH_MODULES],
+[
+PKG_WITH_MODULES([$1],[$2],,,[$3],[$4])
+
+AM_CONDITIONAL([HAVE_][$1],
+ [test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"])
+])dnl PKG_HAVE_WITH_MODULES
+
+dnl PKG_HAVE_DEFINE_WITH_MODULES(VARIABLE-PREFIX, MODULES,
+dnl [DESCRIPTION], [DEFAULT])
+dnl ------------------------------------------------------
+dnl
+dnl Convenience macro to run AM_CONDITIONAL and AC_DEFINE after
+dnl PKG_WITH_MODULES check. HAVE_[VARIABLE-PREFIX] is exported as make
+dnl and preprocessor variable.
+AC_DEFUN([PKG_HAVE_DEFINE_WITH_MODULES],
+[
+PKG_HAVE_WITH_MODULES([$1],[$2],[$3],[$4])
+
+AS_IF([test "$AS_TR_SH([with_]m4_tolower([$1]))" = "yes"],
+ [AC_DEFINE([HAVE_][$1], 1, [Enable ]m4_tolower([$1])[ support])])
+])dnl PKG_HAVE_DEFINE_WITH_MODULES
+
 m4_include([m4/ax_c_float_words_bigendian.m4])
 m4_include([m4/ax_check_openssl.m4])