Feature/support non standard vcpkg configs

scripts/gha/build_desktop.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-Assuming pre-requisites are in place (from running 
+Assuming pre-requisites are in place (from running
 `scripts/gha/install_prereqs_desktop.py`), this builds the firebase cpp sdk.
 
 It does the following,
@@ -29,17 +29,23 @@
 # Build all targets with default options and also build unit tests
 python scripts/gha/build_desktop.py --build_tests --arch x64
 
-# Build only firebase_app and firebase_auth 
+# Build only firebase_app and firebase_auth
 python scripts/gha/build_desktop.py --target firebase_app firebase_auth
 
+# /MT Build with static runtime libraries in MSVC (Windows) and x86.
+python scripts/gha/build_desktop.py --crt_linkage static --arch x86
+
+# /MD Build with dynamic runtime libraries in MSVC (Windows) and x86.
+python scripts/gha/build_desktop.py --crt_linkage dynamic --arch x86
+
 """
 
 import argparse
 import os
 import utils
 
 
-def install_cpp_dependencies_with_vcpkg(arch):
+def install_cpp_dependencies_with_vcpkg(arch, crt_linkage):
  """Install packages with vcpkg.
 
  This does the following,
@@ -48,6 +54,7 @@ def install_cpp_dependencies_with_vcpkg(arch):
  - install packages via vcpkg.
  Args:
  arch (str): Architecture (eg: 'x86', 'x64').
+ crt_linkage (str): Runtime linkage for MSVC (eg: 'dynamic', 'static')
  """
 
  # Install vcpkg executable if its not installed already
@@ -60,20 +67,22 @@ def install_cpp_dependencies_with_vcpkg(arch):
 
  # for each desktop platform, there exists a vcpkg response file in the repo
  # (external/vcpkg_<triplet>_response_file.txt) defined for each target triplet
- vcpkg_triplet = utils.get_vcpkg_triplet(arch)
- vcpkg_response_file_path = os.path.join(os.getcwd(), 'external', 
- 'vcpkg_' + vcpkg_triplet + '_response_file.txt')
+ vcpkg_triplet = utils.get_vcpkg_triplet(arch, crt_linkage)
+ vcpkg_response_file_path = utils.get_vcpkg_response_file_path(vcpkg_triplet)
 
- # Eg: ./external/vcpkg/vcpkg install @external/vcpkg_x64-osx_response_file.txt 
+ # Eg: ./external/vcpkg/vcpkg install @external/vcpkg_x64-osx_response_file.txt
  # --disable-metrics
  utils.run_command([vcpkg_executable_file_path, 'install',
  '@' + vcpkg_response_file_path, '--disable-metrics'])
 
- vcpkg_root_dir_path = utils.get_vcpkg_root_path()
-
  # Clear temporary directories and files created by vcpkg buildtrees
  # could be several GBs and cause github runners to run out of space
  utils.clean_vcpkg_temp_data()
+ print("Successfully built C++ dependencies via vcpkg!\n"
+ "Please set the following cmake options to use these libraries.\n"
+ "VCPKG_TARGET_TRIPLET: {0}\n"
+ "CMAKE_TOOLCHAIN_FILE: {1}\n".format(vcpkg_triplet,
+ utils.get_vcpkg_cmake_toolchain_file_path()))
 
 
 def cmake_configure(build_dir, arch, build_tests=True, config=None):
@@ -88,45 +97,48 @@ def cmake_configure(build_dir, arch, build_tests=True, config=None):
  build_tests (bool): Build cpp unit tests.
  config (str): Release/Debug config.
  If its not specified, cmake's default is used (most likely Debug).
- """ 
+ """
  cmd = ['cmake', '-S', '.', '-B', build_dir]
-
- # If generator is not specifed, default for platform is used by cmake, else 
+ if utils.is_windows_os() and arch == 'x86':
+ cmd.extend(['-A', 'Win32'])
+
+ # If generator is not specifed, default for platform is used by cmake, else
  # use the specified value
  if config:
  cmd.append('-DCMAKE_BUILD_TYPE={0}'.format(config))
  if build_tests:
  cmd.append('-DFIREBASE_CPP_BUILD_TESTS=ON')
  cmd.append('-DFIREBASE_FORCE_FAKE_SECURE_STORAGE=ON')
 
- vcpkg_toolchain_file_path = os.path.join(os.getcwd(), 'external',
- 'vcpkg', 'scripts', 
- 'buildsystems', 'vcpkg.cmake')
+ vcpkg_toolchain_file_path = utils.get_vcpkg_cmake_toolchain_file_path()
  cmd.append('-DCMAKE_TOOLCHAIN_FILE={0}'.format(vcpkg_toolchain_file_path))
 
  vcpkg_triplet = utils.get_vcpkg_triplet(arch)
  cmd.append('-DVCPKG_TARGET_TRIPLET={0}'.format(vcpkg_triplet))
- 
+
  utils.run_command(cmd)
 
 
 def main():
  args = parse_cmdline_args()
- 
+
  # Ensure that the submodules are initialized and updated
  # Example: vcpkg is a submodule (external/vcpkg)
  utils.run_command(['git', 'submodule', 'init'])
  utils.run_command(['git', 'submodule', 'update'])
 
- # Install platform dependent cpp dependencies with vcpkg 
- install_cpp_dependencies_with_vcpkg(args.arch)
+ # Install platform dependent cpp dependencies with vcpkg
+ install_cpp_dependencies_with_vcpkg(args.arch, args.crt_linkage)
+
+ if args.vcpkg_step_only:
+ return
 
  # CMake configure
  cmake_configure(args.build_dir, args.arch, args.build_tests, args.config)
 
- # CMake build 
- # cmake --build build -j 8 
- cmd = ['cmake', '--build', args.build_dir, '-j', str(os.cpu_count())] 
+ # CMake build
+ # cmake --build build -j 8
+ cmd = ['cmake', '--build', args.build_dir, '-j', str(os.cpu_count())]
  if args.target:
  # Example: cmake --build build -j 8 --target firebase_app firebase_auth
  cmd.append('--target')
@@ -137,13 +149,19 @@ def main():
 def parse_cmdline_args():
  parser = argparse.ArgumentParser(description='Install Prerequisites for building cpp sdk')
  parser.add_argument('-a', '--arch', default='x64', help='Platform architecture (x64, x86)')
+ parser.add_argument('--crt_linkage', default='static', help='Runtime linkage (works only for MSVC)')
  parser.add_argument('--build_dir', default='build', help='Output build directory')
  parser.add_argument('--build_tests', action='store_true', help='Build unit tests too')
  parser.add_argument('--config', help='Release/Debug config')
  parser.add_argument('--target', nargs='+', help='A list of CMake build targets (eg: firebase_app firebase_auth)')
+ parser.add_argument('--vcpkg_step_only', action='store_true', help='Run vcpkg only and avoid subsequent cmake commands')
  args = parser.parse_args()
+ if utils.is_linux_os() or utils.is_mac_os():
+ # This flag makes sense only for Windows and MSVC.
+ # For Linux and Mac, we can use the default value (dynamic) as it allows us to just use
+ # the prebuilt vcpkg configuration files instead of creating new ones.
+ args.crt_linkage = 'dynamic'
  return args
 
 if __name__ == '__main__':
  main()
-

scripts/gha/utils.py

@@ -52,7 +52,8 @@ def run_command(cmd, capture_output=False, cwd=None, check=False, as_root=False)
  print('Running cmd: {0}\n'.format(cmd_string))
  # If capture_output is requested, we also set text=True to store the returned value of the
  # command as a string instead of bytes object
- return subprocess.run(cmd, capture_output=capture_output, cwd=cwd, check=check, text=capture_output)
+ return subprocess.run(cmd, capture_output=capture_output, cwd=cwd, check=check,
+ text=capture_output)
 
 
 def is_command_installed(tool):
@@ -89,15 +90,79 @@ def is_linux_os():
  return platform.system() == 'Linux'
 
 
-def get_vcpkg_triplet(arch):
- """ Get vcpkg target triplet (platform definition).
+def check_vcpkg_triplet(triplet_name, arch, crt_linkage):
+ """Check if a vcpkg triplet exists that match our specification.
 
  Args:
  arch (str): Architecture (eg: 'x86', 'x64').
+ crt_linkage (str): Runtime linkage for MSVC (eg: 'dynamic', 'static')
+
+ Returns:
+ (bool): If the triplet is valid.
+ """
+ triplet_file_path = get_vcpkg_triplet_file_path(triplet_name)
+ if not os.path.exists(triplet_file_path):
+ return False
+
+ _arch = None
+ _crt_linkage = None
+
+ with open(triplet_file_path, 'r') as triplet_file:
+ for line in triplet_file:
+ # Eg: set(VCPKG_TARGET_ARCHITECTURE x86) ->
+ # set(VCPKG_TARGET_ARCHITECTURE x86
+ line = line.rstrip(')')
+ if not line.startswith('set('):
+ continue
+ # Eg: 'set(', 'VCPKG_TARGET_ARCHITECTURE x86'
+ variable = line.split('set(')[-1]
+ variable_name, variable_value = variable.split(' ')
+ if variable_name == 'VCPKG_TARGET_ARCHITECTURE':
+ _arch = variable_value
+ elif variable_name == 'VCPKG_CRT_LINKAGE':
+ _crt_linkage = variable_value
+
+ return (_arch == arch) and (_crt_linkage == crt_linkage)
+
+
+def create_vcpkg_triplet(arch, crt_linkage):
+ """Create a new triplet configuration.
+
+ Args:
+ arch (str): Architecture (eg: 'x86', 'x64').
+ crt_linkage (str): Runtime linkage for MSVC (eg: 'dynamic', 'static')
+
+ Returns:
+ (str): Triplet name
+ Eg: 'x86-windows-dynamic'
+ """
+ contents = [ 'VCPKG_TARGET_ARCHITECTURE {0}'.format(arch),
+ 'VCPKG_CRT_LINKAGE {0}'.format(crt_linkage),
+ 'VCPKG_LIBRARY_LINKAGE static']
+
+ contents = ['set({0})\n'.format(content) for content in contents]
+ if is_linux_os() or is_mac_os():
+ contents.append('\n')
+ contents.append('set(VCPKG_CMAKE_SYSTEM_NAME {0})\n'.format(platform.system()))
+
+ triplet_name = '{0}-{1}-{2}'.format(arch.lower(), platform.system().lower(), crt_linkage)
+ with open(get_vcpkg_triplet_file_path(triplet_name), 'w') as triplet_file:
+ triplet_file.writelines(contents)
+ print('Created new triplet: {0}'.format(triplet_name))
+
+ return triplet_name
+
+
+def get_vcpkg_triplet(arch='x64', crt_linkage='dynamic'):
+ """Get vcpkg target triplet (platform definition).
+
+ Args:
+ arch (str): Architecture (eg: 'x86', 'x64').
+ crt_linkage (str): Runtime linkage for MSVC (eg: 'dynamic', 'static')
 
  Raises:
  ValueError: If current OS is not win,mac or linux.
- 
+
  Returns:
  (str): Triplet name.
  Eg: "x64-windows-static".
@@ -114,10 +179,53 @@ def get_vcpkg_triplet(arch):
  triplet_name.append('linux')
 
  triplet_name = '-'.join(triplet_name)
+ ok = check_vcpkg_triplet(triplet_name, arch, crt_linkage)
+ if not ok:
+ triplet_name = create_vcpkg_triplet(arch, crt_linkage)
+
  print("Using vcpkg triplet: {0}".format(triplet_name))
  return triplet_name
 
 
+def get_vcpkg_triplet_file_path(triplet_name):
+ """Get absolute path to vcpkg triplet configuration file."""
+ return os.path.join(get_vcpkg_root_path(), 'triplets',
+ triplet_name + '.cmake')
+
+
+def get_vcpkg_response_file_path(triplet_name):
+ """Get absolute path to vcpkg response file containing packages to install"""
+ response_file_dir_path = os.path.join(os.getcwd(), 'external')
+ response_file_path = os.path.join(response_file_dir_path,
+ 'vcpkg_' + triplet_name + '_response_file.txt')
+ # The firebase-cpp-sdk repo ships with pre-created response files
+ # (list of packages to install + a triplet name at the end)
+ # for common triplets supported by a fresh vcpkg install.
+ # These response files are located at <repo_root>/external/vcpkg_*
+ # If we do not find a matching response file to specified triplet,
+ # we have to create a new file. In order to do this, we copy any of
+ # the existing files (vcpkg_x64-linux_response_file.txt in this case)
+ # and modify the triplet name contained in it. Copying makes sure that the
+ # list of packages to install is the same as other response files.
+ if not os.path.exists(response_file_path):
+ existing_response_file_path = os.path.join(response_file_dir_path,
+ 'vcpkg_x64-linux_response_file.txt')
+ with open(existing_response_file_path, 'r') as existing_file:
+ # Structure of a vcpkg response file
+ # <package1>
+ # <package2>
+ # ...
+ # --triplet
+ # <triplet_name>
+ lines = existing_file.readlines()
+ # Modify the line containing triplet name
+ lines[-1] = triplet_name + '\n'
+ with open(response_file_path, 'w') as response_file:
+ response_file.writelines(lines)
+ print("Created new response file: {0}".format(response_file_path))
+ return response_file_path
+
+
 def get_vcpkg_root_path():
  """Get absolute path to vcpkg root directory in repo."""
  return os.path.join(os.getcwd(), 'external', 'vcpkg')
@@ -144,6 +252,12 @@ def get_vcpkg_installation_script_path():
  return script_absolute_path
 
 
+def get_vcpkg_cmake_toolchain_file_path():
+ """Get absolute path to toolchain file used for cmake builds"""
+ vcpkg_root_dir = get_vcpkg_root_path()
+ return os.path.join(vcpkg_root_dir, 'scripts', 'buildsystems', 'vcpkg.cmake')
+
+
 def clean_vcpkg_temp_data():
  """Delete files/directories that vcpkg uses during its build"""
  # Clear temporary directories and files created by vcpkg buildtrees
@@ -153,4 +267,3 @@ def clean_vcpkg_temp_data():
  delete_directory(buildtrees_dir_path)
  downloads_dir_path = os.path.join(vcpkg_root_dir_path, 'downloads')
  delete_directory(downloads_dir_path)
-