AWS_CRT_BUILD_USE_SYSTEM_LIBS (awslabs#593)

*Issue #:* awslabs#588 User wants explicit way to choose whether aws-crt-python builds its own dependencies (their source code is under `crt/`), or find them prebuilt on the system. There's currently a secret way to do this (delete `crt/` folder, or don't sync submodules), but it's not documented. *Description of changes:* Set `AWS_CRT_BUILD_USE_SYSTEM_LIBS=1` to explicitly disable building dependencies

Author: graebm

README.md

@@ -45,10 +45,10 @@ To simplify installation, aws-crt-python has its own copy of libcrypto.
 This lets you install a wheel from PyPI without having OpenSSL installed.
 Unix wheels on PyPI come with libcrypto statically compiled in.
 Code to build libcrypto comes from [AWS-LC](https://github.com/aws/aws-lc).
-AWS-LC's code is included in the PyPI source package, 
+AWS-LC's code is included in the PyPI source package,
 and the git repository includes it as a submodule.
 
-If you need aws-crt-python to use the libcrypto included on your system, 
+If you need aws-crt-python to use the libcrypto included on your system,
 set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1` while building from source:
 
 ```sh
@@ -59,6 +59,19 @@ AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO=1 python3 -m pip install --no-binary :all: --
 You can ignore all this on Windows and Apple platforms, where aws-crt-python
 uses the OS's default libraries for TLS and cryptography math.
 
+### AWS_CRT_BUILD_USE_SYSTEM_LIBS ###
+
+aws-crt-python depends on several C libraries that make up the AWS Common Runtime (libaws-c-common, libaws-c-s3, etc).
+By default, these libraries are built along with aws-crt-python and statically compiled in
+(their source code is under [crt/](crt/)).
+
+To skip building these dependencies, because they're already available on your system,
+set environment variable `AWS_CRT_BUILD_USE_SYSTEM_LIBS=1` while building from source:
+
+```sh
+AWS_CRT_BUILD_USE_SYSTEM_LIBS=1 python3 -m pip install .
+```
+
 ## Mac-Only TLS Behavior
 
 Please note that on Mac, once a private key is used with a certificate, that certificate-key pair is imported into the Mac Keychain. All subsequent uses of that certificate will use the stored private key and ignore anything passed in programmatically. Beginning in v0.6.2, when a stored private key from the Keychain is used, the following will be logged at the "info" log level:

awscrt/auth.py

@@ -507,7 +507,7 @@ class AwsSignedBodyHeaderType(IntEnum):
  """Do not add a header."""
 
  X_AMZ_CONTENT_SHA_256 = 1
- """Add the "x-amz-content-sha-256" header with the canonical request's signed body value"""
+ """Add the "x-amz-content-sha256" header with the canonical request's signed body value"""
 
 
 class AwsSigningConfig(NativeResource):

setup.py

@@ -134,8 +134,14 @@ def get_cmake_path():
  raise Exception("CMake must be installed to build from source.")
 
 
+def using_system_libs():
+ """If true, don't build any dependencies. Use the libs that are already on the system."""
+ return os.getenv('AWS_CRT_BUILD_USE_SYSTEM_LIBS') == '1'
+
+
 def using_system_libcrypto():
- return os.getenv('AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO') == '1'
+ """If true, don't build AWS-LC. Use the libcrypto that's already on the system."""
+ return using_system_libs() or os.getenv('AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO') == '1'
 
 
 class AwsLib:
@@ -223,7 +229,10 @@ def _build_dependencies_impl(self, build_dir, install_path, osx_arch=None):
  ]
  run_cmd(build_cmd)
 
- def _build_dependencies(self, build_dir, install_path):
+ def _build_dependencies(self):
+ build_dir = os.path.join(self.build_temp, 'deps')
+ install_path = os.path.join(self.build_temp, 'deps', 'install')
+
  if is_macos_universal2() and not is_development_mode():
  # create macOS universal binary by compiling for x86_64 and arm64,
  # each in its own subfolder, and then creating a universal binary
@@ -266,30 +275,28 @@ def _build_dependencies(self, build_dir, install_path):
  # normal build for a single architecture
  self._build_dependencies_impl(build_dir, install_path)
 
- def run(self):
- # build dependencies
- dep_build_dir = os.path.join(self.build_temp, 'deps')
- dep_install_path = os.path.join(self.build_temp, 'deps', 'install')
-
- if os.path.exists(os.path.join(PROJECT_DIR, 'crt', 'aws-c-common', 'CMakeLists.txt')):
- self._build_dependencies(dep_build_dir, dep_install_path)
- else:
- print("Skip building dependencies, source not found.")
-
  # update paths so awscrt_ext can access dependencies.
  # add to the front of any list so that our dependencies are preferred
  # over anything that might already be on the system (i.e. libcrypto.a)
 
- self.include_dirs.insert(0, os.path.join(dep_install_path, 'include'))
+ self.include_dirs.insert(0, os.path.join(install_path, 'include'))
 
  # some platforms (ex: fedora) use /lib64 instead of just /lib
  lib_dir = 'lib'
- if is_64bit() and os.path.exists(os.path.join(dep_install_path, 'lib64')):
+ if is_64bit() and os.path.exists(os.path.join(install_path, 'lib64')):
  lib_dir = 'lib64'
- if is_32bit() and os.path.exists(os.path.join(dep_install_path, 'lib32')):
+ if is_32bit() and os.path.exists(os.path.join(install_path, 'lib32')):
  lib_dir = 'lib32'
 
- self.library_dirs.insert(0, os.path.join(dep_install_path, lib_dir))
+ self.library_dirs.insert(0, os.path.join(install_path, lib_dir))
+
+ def run(self):
+ if using_system_libs():
+ print("Skip building dependencies, using system libs.")
+ elif not os.path.exists(os.path.join(PROJECT_DIR, 'crt', 'aws-c-common', 'CMakeLists.txt')):
+ print("Skip building dependencies, source not found.")
+ else:
+ self._build_dependencies()
 
  # continue with normal build_ext.run()
  super().run()