bpo-35381 Make all posixmodule types heap-allocated

.azure-pipelines/ci.yml

@@ -59,7 +59,7 @@ jobs:
  variables:
  testRunTitle: '$(build.sourceBranchName)-linux'
  testRunPlatform: linux
- openssl_version: 1.1.0j
+ openssl_version: 1.1.1c
 
  steps:
  - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
  variables:
  testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
  testRunPlatform: linux-coverage
- openssl_version: 1.1.0j
+ openssl_version: 1.1.1c
 
  steps:
  - template: ./posix-steps.yml

.azure-pipelines/docs-steps.yml

@@ -12,7 +12,7 @@ steps:
  inputs:
  versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==1.8.2 blurb python-docs-theme
+- script: python -m pip install sphinx==2.0.1 blurb python-docs-theme
  displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:

.azure-pipelines/posix-steps.yml

@@ -10,6 +10,10 @@ steps:
  clean: true
  fetchDepth: 5
 
+# Work around a known issue affecting Ubuntu VMs on Pipelines
+- script: sudo setfacl -Rb /home/vsts
+ displayName: 'Workaround ACL issue'
+
 - script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
  displayName: 'Install dependencies'
 

.azure-pipelines/pr.yml

@@ -59,7 +59,7 @@ jobs:
  variables:
  testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
  testRunPlatform: linux
- openssl_version: 1.1.0j
+ openssl_version: 1.1.1c
 
  steps:
  - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
  variables:
  testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
  testRunPlatform: linux-coverage
- openssl_version: 1.1.0j
+ openssl_version: 1.1.1c
 
  steps:
  - template: ./posix-steps.yml

.azure-pipelines/windows-release.yml

@@ -0,0 +1,129 @@
+name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+ __RealSigningCertificate: 'Python Software Foundation'
+# QUEUE TIME VARIABLES
+# GitRemote: python
+# SourceTag:
+# DoPGO: true
+# SigningCertificate: 'Python Software Foundation'
+# SigningDescription: 'Built: $(Build.BuildNumber)'
+# DoLayout: true
+# DoMSIX: true
+# DoNuget: true
+# DoEmbed: true
+# DoMSI: true
+# DoPublish: false
+# PyDotOrgUsername: ''
+# PyDotOrgServer: ''
+# BuildToPublish: ''
+
+trigger: none
+pr: none
+
+stages:
+- stage: Build
+ displayName: Build binaries
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-build.yml
+
+- stage: Sign
+ displayName: Sign binaries
+ dependsOn: Build
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-sign.yml
+
+- stage: Layout
+ displayName: Generate layouts
+ dependsOn: Sign
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-layout-full.yml
+ - template: windows-release/stage-layout-embed.yml
+ - template: windows-release/stage-layout-nuget.yml
+
+- stage: Pack
+ dependsOn: Layout
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-pack-nuget.yml
+
+- stage: Test
+ dependsOn: Pack
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-test-embed.yml
+ - template: windows-release/stage-test-nuget.yml
+
+- stage: Layout_MSIX
+ displayName: Generate MSIX layouts
+ dependsOn: Sign
+ condition: and(succeeded(), and(eq(variables['DoMSIX'], 'true'), not(variables['BuildToPublish'])))
+ jobs:
+ - template: windows-release/stage-layout-msix.yml
+
+- stage: Pack_MSIX
+ displayName: Package MSIX
+ dependsOn: Layout_MSIX
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-pack-msix.yml
+
+- stage: Build_MSI
+ displayName: Build MSI installer
+ dependsOn: Sign
+ condition: and(succeeded(), and(eq(variables['DoMSI'], 'true'), not(variables['BuildToPublish'])))
+ jobs:
+ - template: windows-release/stage-msi.yml
+
+- stage: Test_MSI
+ displayName: Test MSI installer
+ dependsOn: Build_MSI
+ condition: and(succeeded(), not(variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-test-msi.yml
+
+- stage: PublishPyDotOrg
+ displayName: Publish to python.org
+ dependsOn: ['Test_MSI', 'Test']
+ condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+ jobs:
+ - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishNuget
+ displayName: Publish to nuget.org
+ dependsOn: Test
+ condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+ jobs:
+ - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishStore
+ displayName: Publish to Store
+ dependsOn: Pack_MSIX
+ condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), not(variables['BuildToPublish'])))
+ jobs:
+ - template: windows-release/stage-publish-store.yml
+
+
+- stage: PublishExistingPyDotOrg
+ displayName: Publish existing build to python.org
+ dependsOn: []
+ condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-publish-pythonorg.yml
+
+- stage: PublishExistingNuget
+ displayName: Publish existing build to nuget.org
+ dependsOn: []
+ condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-publish-nugetorg.yml
+
+- stage: PublishExistingStore
+ displayName: Publish existing build to Store
+ dependsOn: []
+ condition: and(succeeded(), and(eq(variables['DoPublish'], 'true'), variables['BuildToPublish']))
+ jobs:
+ - template: windows-release/stage-publish-store.yml

.azure-pipelines/windows-release/build-steps.yml

@@ -0,0 +1,83 @@
+parameters:
+ ShouldPGO: false
+
+steps:
+- template: ./checkout.yml
+
+- powershell: |
+ $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }};
+ Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)"
+ Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)"
+ Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)"
+ Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)"
+ Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)"
+ Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)"
+ displayName: 'Extract version numbers'
+
+- ${{ if eq(parameters.ShouldPGO, 'false') }}:
+ - powershell: |
+ $env:SigningCertificate = $null
+ .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration)
+ displayName: 'Run build'
+ env:
+ IncludeUwp: true
+ Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- ${{ if eq(parameters.ShouldPGO, 'true') }}:
+ - powershell: |
+ $env:SigningCertificate = $null
+ .\PCbuild\build.bat -v -p $(Platform) --pgo
+ displayName: 'Run build with PGO'
+ env:
+ IncludeUwp: true
+ Py_OutDir: '$(Build.BinariesDirectory)\bin'
+
+- powershell: |
+ $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+ $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1)
+ if (-not $tool) {
+ throw "SDK is not available"
+ }
+ Write-Host "##vso[task.prependpath]$($tool.Directory)"
+ displayName: 'Add WinSDK tools to path'
+
+- powershell: |
+ $env:SigningCertificate = $null
+ .\python.bat PC\layout -vv -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default
+ makecat "${env:CAT}.cdf"
+ del "${env:CAT}.cdf"
+ if (-not (Test-Path "${env:CAT}.cat")) {
+ throw "Failed to build catalog file"
+ }
+ displayName: 'Generate catalog'
+ env:
+ CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python
+
+- task: PublishPipelineArtifact@0
+ displayName: 'Publish binaries'
+ condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])))
+ inputs:
+ targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)'
+ artifactName: bin_$(Name)
+
+- task: PublishPipelineArtifact@0
+ displayName: 'Publish binaries for signing'
+ condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))
+ inputs:
+ targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)'
+ artifactName: unsigned_bin_$(Name)
+
+- task: CopyFiles@2
+ displayName: 'Layout Artifact: symbols'
+ inputs:
+ sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch)
+ targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name)
+ flatten: true
+ contents: |
+ **\*.pdb
+
+- task: PublishBuildArtifacts@1
+ displayName: 'Publish Artifact: symbols'
+ inputs:
+ PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols'
+ ArtifactName: symbols

.azure-pipelines/windows-release/checkout.yml

@@ -0,0 +1,21 @@
+parameters:
+ depth: 3
+
+steps:
+- checkout: none
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git .
+ displayName: 'git clone ($(GitRemote)/$(SourceTag))'
+ condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag']))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) .
+ displayName: 'git clone (<default>/$(SourceTag))'
+ condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag']))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git .
+ displayName: 'git clone ($(GitRemote)/<default>)'
+ condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag'])))
+
+- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) .
+ displayName: 'git clone'
+ condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag'])))

.azure-pipelines/windows-release/find-sdk.yml

@@ -0,0 +1,17 @@
+# Locate the Windows SDK and add its binaries directory to PATH
+#
+# `toolname` can be overridden to use a different marker file.
+
+parameters:
+ toolname: signtool.exe
+
+steps:
+ - powershell: |
+ $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
+ $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1)
+ if (-not $tool) {
+ throw "SDK is not available"
+ }
+ Write-Host "##vso[task.prependpath]$($tool.Directory)"
+ Write-Host "Adding $($tool.Directory) to PATH"
+ displayName: 'Add WinSDK tools to path'

.azure-pipelines/windows-release/gpg-sign.yml

@@ -0,0 +1,28 @@
+parameters:
+ GPGKeyFile: $(GPGKey)
+ GPGPassphrase: $(GPGPassphrase)
+ Files: '*'
+ WorkingDirectory: $(Build.BinariesDirectory)
+
+steps:
+- task: DownloadSecureFile@1
+ name: gpgkey
+ inputs:
+ secureFile: ${{ parameters.GPGKeyFile }}
+ displayName: 'Download GPG key'
+
+- powershell: |
+ git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg"
+ gpg/gpg2.exe --import "$(gpgkey.secureFilePath)"
+ (gci -File ${{ parameters.Files }}).FullName | %{
+ gpg/gpg2.exe -ba --batch --passphrase ${{ parameters.GPGPassphrase }} $_
+ "Made signature for $_"
+ }
+ displayName: 'Generate GPG signatures'
+ workingDirectory: ${{ parameters.WorkingDirectory }}
+
+- powershell: |
+ $p = gps "gpg-agent" -EA 0
+ if ($p) { $p.Kill() }
+ displayName: 'Kill GPG agent'
+ condition: true

.azure-pipelines/windows-release/layout-command.yml

@@ -0,0 +1,15 @@
+steps:
+- powershell: >
+ Write-Host (
+ '##vso[task.setvariable variable=LayoutCmd]&
+ "{0}\bin\python.exe"
+ "{1}\PC\layout"
+ -vv
+ --source "{1}"
+ --build "{0}\bin"
+ --temp "{0}\layout-temp"
+ --include-cat "{0}\bin\python.cat"
+ --doc-build "{0}\doc"'
+ -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)")
+ )
+ displayName: 'Set LayoutCmd'

.azure-pipelines/windows-release/mingw-lib.yml

@@ -0,0 +1,13 @@
+parameters:
+ DllToolOpt: -m i386:x86-64
+ #DllToolOpt: -m i386 --as-flags=--32
+
+steps:
+- powershell: |
+ git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils"
+ gci "bin\$(Arch)\python*.dll" | %{
+ & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def
+ & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }}
+ }
+ displayName: 'Generate MinGW import library'
+ workingDirectory: $(Build.BinariesDirectory)