[3.6] bpo-31340: Change to building with MSVC v141 (included with Visual Studio 2017) (GH-3311) (#3386)

Author: zooba

.github/appveyor.yml

@@ -14,6 +14,9 @@ test_script:
 - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
 environment:
  HOST_PYTHON: C:\Python36\python.exe
+image:
+- Visual Studio 2015
+- Visual Studio 2017
 
 # Only trigger AppVeyor if actual code or its configuration changes
 only_commits:

Doc/make.bat

@@ -6,18 +6,29 @@ pushd %~dp0
 set this=%~n0
 
 call ..\PCBuild\find_python.bat %PYTHON%
-if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" (
- set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe
- rem Cannot use %SPHINXBUILD% in the same block where we set it
- if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" (
+if not defined SPHINXBUILD if defined PYTHON (
+ %PYTHON% -c "import sphinx" > nul 2> nul
+ if errorlevel 1 (
  echo Installing sphinx with %PYTHON%
- "%PYTHON%" -m pip install sphinx
+ %PYTHON% -m pip install sphinx
  if errorlevel 1 exit /B
  )
+ set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
 )
 
-if "%PYTHON%" EQU "" set PYTHON=py
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
+if not defined BLURB if defined PYTHON (
+ %PYTHON% -c "import blurb" > nul 2> nul
+ if errorlevel 1 (
+ echo Installing blurb with %PYTHON%
+ %PYTHON% -m pip install blurb
+ if errorlevel 1 exit /B
+ )
+ set BLURB=%PYTHON% -m blurb
+)
+
+if not defined PYTHON set PYTHON=py
+if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
+if not defined BLURB set BLURB=blurb
 
 if "%1" NEQ "htmlhelp" goto :skiphhcsearch
 if exist "%HTMLHELP%" goto :skiphhcsearch
@@ -96,6 +107,19 @@ echo.be passed by setting the SPHINXOPTS environment variable.
 goto end
 
 :build
+if exist ..\Misc\NEWS (
+ echo.Copying Misc\NEWS to build\NEWS
+ copy ..\Misc\NEWS build\NEWS > nul
+) else if exist ..\Misc\NEWS.D (
+ if defined BLURB (
+ echo.Merging Misc/NEWS with %BLURB%
+ %BLURB% merge -f build\NEWS
+ ) else (
+ echo.No Misc/NEWS file and Blurb is not available.
+ exit /B 1
+ )
+)
+
 if NOT "%PAPER%" == "" (
  set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
 )

Lib/distutils/command/bdist_wininst.py

@@ -318,26 +318,30 @@ def get_exe_bytes(self):
  # string compares seem wrong, but are what sysconfig.py itself uses
  if self.target_version and self.target_version < cur_version:
  if self.target_version < "2.4":
- bv = 6.0
+ bv = '6.0'
  elif self.target_version == "2.4":
- bv = 7.1
+ bv = '7.1'
  elif self.target_version == "2.5":
- bv = 8.0
+ bv = '8.0'
  elif self.target_version <= "3.2":
- bv = 9.0
+ bv = '9.0'
  elif self.target_version <= "3.4":
- bv = 10.0
+ bv = '10.0'
  else:
- bv = 14.0
+ bv = '14.0'
  else:
  # for current version - use authoritative check.
  try:
  from msvcrt import CRT_ASSEMBLY_VERSION
  except ImportError:
  # cross-building, so assume the latest version
- bv = 14.0
+ bv = '14.0'
  else:
- bv = float('.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2]))
+ bv = '.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2])
+ if bv == '14.11':
+ # v141 and v140 are binary compatible,
+ # so keep using the 14.0 stub.
+ bv = '14.0'
 
 
  # wininst-x.y.exe is in the same directory as this file
@@ -353,7 +357,7 @@ def get_exe_bytes(self):
  else:
  sfix = ''
 
- filename = os.path.join(directory, "wininst-%.1f%s.exe" % (bv, sfix))
+ filename = os.path.join(directory, "wininst-%s%s.exe" % (bv, sfix))
  f = open(filename, "rb")
  try:
  return f.read()

Misc/NEWS.d/next/Windows/2017-09-04-13-19-05.bpo-31340.MbkzLi.rst

@@ -0,0 +1 @@
+Change to building with MSVC v141 (included with Visual Studio 2017)

PCbuild/build.bat

@@ -104,7 +104,7 @@ if "%kill%"=="true" call :Kill
 
 if "%do_pgo%"=="true" (
  set conf=PGInstrument
- call :Build
+ call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
  del /s "%dir%\*.pgc"
  del /s "%dir%\..\Lib\*.pyc"
  echo on

PCbuild/find_msbuild.bat

@@ -29,20 +29,21 @@
 @where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
 @if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
 
-@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
-@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
-@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
- @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
-)
-@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
-
 @rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
 @reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
 @if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
  @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
 )
 @if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
 
+@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+ @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
 
 @exit /b 1
 

PCbuild/pyproject.props

@@ -147,13 +147,28 @@ foreach (System.Diagnostics.Process p in System.Diagnostics.Process.GetProcesses
  Targets="CleanAll" />
  </Target>
 
+ <Target Name="CopyPGCFiles" BeforeTargets="PrepareForBuild" Condition="$(Configuration) == 'PGUpdate'">
+ <ItemGroup>
+ <_PGCFiles Include="$(OutDir)instrumented\$(TargetName)!*.pgc" />
+ <_PGDFile Include="$(OutDir)instrumented\$(TargetName).pgd" />
+ <_CopyFiles Include="@(_PGCFiles);@(_PGDFile)" Condition="Exists(%(FullPath))" />
+ </ItemGroup>
+ <Delete Files="@(_CopyFiles->'$(OutDir)%(Filename)%(Extension)')" />
+ <Error Text="PGO run did not succeed (no $(TargetName)!*.pgc files) and there is no data to merge"
+ Condition="$(RequirePGCFiles) == 'true' and @(_PGCFiles) == ''" />
+ <Copy SourceFiles="@(_CopyFiles)"
+ DestinationFolder="$(OutDir)"
+ UseHardLinksIfPossible="true"
+ OverwriteReadOnlyFiles="true" />
+ </Target>
+
  <PropertyGroup>
- <SdkBinPath Condition="'$(SdkBinPath)' == '' or !Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86</SdkBinPath>
+ <SdkBinPath Condition="'$(SdkBinPath)' == '' or !Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\$(DefaultWindowsSDKVersion)\x86</SdkBinPath>
+ <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot10)\bin\x86</SdkBinPath>
  <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86</SdkBinPath>
  <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86</SdkBinPath>
  <SdkBinPath Condition="!Exists($(SdkBinPath))">$(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\</SdkBinPath>
- <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand>
- <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe"</_MakeCatCommand>
+ <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)"</_SignCommand> <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe"</_MakeCatCommand>
  </PropertyGroup>
 
  <Target Name="_SignBuild" AfterTargets="AfterBuild" Condition="'$(SigningCertificate)' != '' and $(SupportSigning)">

PCbuild/python.props

@@ -10,6 +10,7 @@
 
  We set BasePlatformToolset for ICC's benefit, it's otherwise ignored.
  -->
+ <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and ('$(MSBuildToolsVersion)' == '15.0' or '$(VisualStudioVersion)' == '15.0')">v141</BasePlatformToolset>
  <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath14)' != ''">v140</BasePlatformToolset>
  <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath12)' != ''">v120</BasePlatformToolset>
  <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VCTargetsPath11)' != ''">v110</BasePlatformToolset>
@@ -39,6 +40,7 @@
  <BuildPath Condition="'$(ArchName)' == 'amd64'">$(BuildPath64)</BuildPath>
  <BuildPath Condition="'$(BuildPath)' == ''">$(PySourcePath)PCBuild\$(ArchName)\</BuildPath>
  <BuildPath Condition="!HasTrailingSlash($(BuildPath))">$(BuildPath)\</BuildPath>
+ <BuildPath Condition="$(Configuration) == 'PGInstrument'">$(BuildPath)instrumented\</BuildPath>
 
  <!-- Directories of external projects. tcltk is handled in tcltk.props -->
  <ExternalsDir>$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`))</ExternalsDir>
@@ -62,7 +64,24 @@
  <!-- Full path of the resulting python.exe binary -->
  <PythonExe Condition="'$(PythonExe)' == ''">$(BuildPath)python$(PyDebugExt).exe</PythonExe>
  </PropertyGroup>
- 
+
+ <PropertyGroup Condition="$(DefaultWindowsSDKVersion) == ''">
+ <!--
+ Attempt to select the latest installed WinSDK. If we don't find any, then we will
+ let the MSBuild targets determine which one it wants to use (typically the earliest
+ possible version). Since we limit WINVER to Windows 7 anyway, it doesn't really
+ matter which WinSDK version we use.
+ -->
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.15063'">10.0.15063.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.15063'">10.0.15063.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.14393'">10.0.14393.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.14393'">10.0.14393.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10586'">10.0.10586.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10586'">10.0.10586.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10240'">10.0.10240.0</DefaultWindowsSDKVersion>
+ <DefaultWindowsSDKVersion Condition="$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) == '10.0.10240'">10.0.10240.0</DefaultWindowsSDKVersion>
+ </PropertyGroup>
+
  <PropertyGroup Condition="'$(OverrideVersion)' == ''">
  <!--
  Read version information from Include\patchlevel.h. The following properties are set:

PCbuild/python.vcxproj

@@ -96,6 +96,7 @@ set PYTHONPATH=$(PySourcePath)Lib
  <PropertyGroup>
  <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'Win32'">@set PATH=%PATH%%3B$(VCInstallDir)bin</_PGOPath>
  <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(Platform) == 'x64'">@set PATH=%PATH%%3B$(VCInstallDir)bin\amd64</_PGOPath>
+ <_PGOPath Condition="$(Configuration) == 'PGInstrument' and $(VC_PGO_RunTime_Dir) != ''">@set PATH=%PATH%%3B$(VC_PGO_RunTime_Dir)</_PGOPath>
  <_Content>@rem This script invokes the most recently built Python with all arguments
 @rem passed through to the interpreter. This file is generated by the
 @rem build process and any changes *will* be thrown away by the next

PCbuild/pythoncore.vcxproj

@@ -49,6 +49,7 @@
  </ImportGroup>
  <PropertyGroup>
  <KillPython>true</KillPython>
+ <RequirePGCFiles>true</RequirePGCFiles>
  </PropertyGroup>
  <ImportGroup Label="PropertySheets">
  <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
@@ -428,7 +429,7 @@
  </ClCompile>
  </ItemGroup>
  </Target>
- <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140'">
+ <Target Name="_WarnAboutToolset" BeforeTargets="PrepareForBuild" Condition="$(PlatformToolset) != 'v140' and $(PlatformToolset) != 'v141'">
  <Warning Text="Toolset $(PlatformToolset) is not used for official builds. Your build may have errors or incompatibilities." />
  </Target>
 </Project>