Update debugger to ptvsd latest (microsoft#9653)

* Update debugger to ptvsd v5.0.0a12 * Add news item * Use `--pre` to avoid updating debugger to often * Fix issues with debugger installation * fix build * Fix merge issues

Author: karthiknadig

build/ci/templates/steps/build.yml

@@ -14,56 +14,55 @@
 steps:
  - template: initialization.yml
  parameters:
- PythonVersion: $(PythonVersion)
- workingDirectory: $(Build.SourcesDirectory)
- compile: 'false'
+  PythonVersion: $(PythonVersion)
+  workingDirectory: $(Build.SourcesDirectory)
+  compile: 'false'
 
  - bash: |
  python -m pip install -U pip
  python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
  python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2'
- python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/new_ptvsd/no_wheels --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==5.0.0a11'
+ python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/new_ptvsd/no_wheels --no-cache-dir --implementation py --no-deps --upgrade --pre ptvsd
  failOnStderr: true
- displayName: "pip install requirements"
+ displayName: 'pip install requirements'
 
- - task: PythonScript@0
- displayName: "Install PTVSD wheels"
- inputs:
- scriptSource: "filePath"
- scriptPath: "./pythonFiles/install_ptvsd.py"
- failOnStderr: true
+ - bash: |
+ python -m pip --disable-pip-version-check install -r build/debugger-install-requirements.txt
+ python ./pythonFiles/install_ptvsd.py
+ failOnStderr: true
+ displayName: 'Install PTVSD wheels'
 
  - bash: npm run clean
- displayName: "Clean"
+ displayName: 'Clean'
 
  # This is a slow process, hence do this only on master and release branches.
  # I.e. skip on in PRs.
  - bash: npm run checkDependencies
- displayName: "Check Dependencies"
+ displayName: 'Check Dependencies'
  condition: and(succeeded(), or(eq(variables['Build.SourceBranchName'], 'master'), startsWith(variables['Build.SourceBranchName'], 'release')))
 
  - bash: |
  npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID
- displayName: "Update dev Version of Extension"
+ displayName: 'Update dev Version of Extension'
  condition: and(succeeded(), eq(variables['Build.SourceBranchName'], 'master'))
 
  - bash: |
  npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID --updateChangelog
- displayName: "Update release Version of Extension"
+ displayName: 'Update release Version of Extension'
  condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'release'))
 
  - bash: |
  npm run package
- displayName: "Build VSIX"
+ displayName: 'Build VSIX'
 
  - task: CopyFiles@2
  inputs:
- contents: "*.vsix"
+ contents: '*.vsix'
  targetFolder: $(Build.ArtifactStagingDirectory)
- displayName: "Copy VSIX"
+ displayName: 'Copy VSIX'
 
  - task: PublishBuildArtifacts@1
  inputs:
  pathtoPublish: $(Build.ArtifactStagingDirectory)
  artifactName: VSIX
- displayName: "Publish VSIX to Artifacts"
+ displayName: 'Publish VSIX to Artifacts'

build/ci/templates/test_phases.yml

@@ -97,7 +97,7 @@ steps:
  python -m pip install --upgrade -r build/test-requirements.txt
  python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
  python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2'
- python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/new_ptvsd/no_wheels --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==5.0.0a11'
+ python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/new_ptvsd/no_wheels --no-cache-dir --implementation py --no-deps --upgrade --pre ptvsd
  displayName: 'pip install system test requirements'
  condition: and(succeeded(), eq(variables['NeedsPythonTestReqs'], 'true'))
 

build/debugger-install-requirements.txt

@@ -0,0 +1,2 @@
+# Requirements needed to run install_ptvsd.py
+packaging

gulpfile.js

@@ -29,6 +29,7 @@ const nativeDependencyChecker = require('node-has-native-dependencies');
 const flat = require('flat');
 const argv = require('yargs').argv;
 const os = require('os');
+const rmrf = require('rimraf');
 
 const isCI = process.env.TRAVIS === 'true' || process.env.TF_BUILD !== undefined;
 
@@ -283,18 +284,35 @@ gulp.task('installPythonRequirements', async () => {
 
 // See https://github.com/microsoft/vscode-python/issues/7136
 gulp.task('installNewPtvsd', async () => {
+ // Install dependencies needed for 'install_ptvsd.py'
+ const depsArgs = ['-m', 'pip', '--disable-pip-version-check', 'install', '-t', './pythonFiles/lib/temp', '-r', './build/debugger-install-requirements.txt'];
+ const successWithWheelsDeps = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', depsArgs)
+ .then(() => true)
+ .catch(ex => {
+ console.error("Failed to install new PTVSD wheels using 'python3'", ex);
+ return false;
+ });
+ if (!successWithWheelsDeps) {
+ console.info("Failed to install dependencies need by 'install_ptvsd.py' using 'python3', attempting to install using 'python'");
+ await spawnAsync('python', depsArgs).catch(ex => console.error("Failed to install dependencies need by 'install_ptvsd.py' using 'python'", ex));
+ }
+
  // Install new PTVSD with wheels for python 3.7
- const successWithWheels = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', ['./pythonFiles/install_ptvsd.py'])
+ const wheelsArgs = ['./pythonFiles/install_ptvsd.py'];
+ const wheelsEnv = { PYTHONPATH: './pythonFiles/lib/temp' };
+ const successWithWheels = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', wheelsArgs, wheelsEnv)
  .then(() => true)
  .catch(ex => {
  console.error("Failed to install new PTVSD wheels using 'python3'", ex);
  return false;
  });
  if (!successWithWheels) {
  console.info("Failed to install new PTVSD wheels using 'python3', attempting to install using 'python'");
- await spawnAsync('python', args).catch(ex => console.error("Failed to install PTVSD 5.0 wheels using 'python'", ex));
+ await spawnAsync('python', wheelsArgs, wheelsEnv).catch(ex => console.error("Failed to install PTVSD 5.0 wheels using 'python'", ex));
  }
 
+ rmrf.sync('./pythonFiles/lib/temp');
+
  // Install source only version of new PTVSD for use with all other python versions.
  const args = [
  '-m',
@@ -308,7 +326,8 @@ gulp.task('installNewPtvsd', async () => {
  'py',
  '--no-deps',
  '--upgrade',
- 'ptvsd==5.0.0a11'
+ '--pre',
+ 'ptvsd'
  ];
  const successWithoutWheels = await spawnAsync(process.env.CI_PYTHON_PATH || 'python3', args)
  .then(() => true)

news/2 Fixes/9652.md

@@ -0,0 +1 @@
+Update debugger version to `ptvsd` v5.0.0a12.

pythonFiles/install_ptvsd.py

@@ -3,35 +3,38 @@
 import os
 import urllib.request as url_lib
 import zipfile
+from packaging.version import parse as version_parser
 
 
 EXTENSION_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 DEBUGGER_DEST = os.path.join(EXTENSION_ROOT, "pythonFiles", "lib", "python", "new_ptvsd", "wheels")
 DEBUGGER_PACKAGE = "ptvsd"
-DEBUGGER_VERSION = "5.0.0a11"
 DEBUGGER_PYTHON_VERSIONS = ("cp37",)
 
 
 def _contains(s, parts=()):
  return any(p for p in parts if p in s)
 
 
-def _get_debugger_wheel_urls():
+def _get_package_data():
  json_uri = "https://pypi.org/pypi/{0}/json".format(DEBUGGER_PACKAGE)
  # Response format: https://warehouse.readthedocs.io/api-reference/json/#project
  # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst
  with url_lib.urlopen(json_uri) as response:
- json_response = json.loads(response.read())
- return list(
- r["url"]
- for r in json_response["releases"][DEBUGGER_VERSION]
- if _contains(r["url"], DEBUGGER_PYTHON_VERSIONS)
- )
+ return json.loads(response.read())
 
 
-def _download_and_extract(root, url):
+def _get_debugger_wheel_urls(data, version):
+ return list(
+ r["url"]
+ for r in data["releases"][version]
+ if _contains(r["url"], DEBUGGER_PYTHON_VERSIONS)
+ )
+
+
+def _download_and_extract(root, url, version):
  root = os.getcwd() if root is None or root == "." else root
- prefix = os.path.join("ptvsd-{0}.data".format(DEBUGGER_VERSION), "purelib")
+ prefix = os.path.join("ptvsd-{0}.data".format(version), "purelib")
  with url_lib.urlopen(url) as response:
  # Extract only the contents of the purelib subfolder (parent folder of ptvsd),
  # since ptvsd files rely on the presence of a 'ptvsd' folder.
@@ -49,8 +52,11 @@ def _download_and_extract(root, url):
 
 
 def main(root):
- for url in _get_debugger_wheel_urls():
- _download_and_extract(root, url)
+ data = _get_package_data()
+ latest_version = max(data["releases"].keys(), key=version_parser)
+
+ for url in _get_debugger_wheel_urls(data, latest_version):
+ _download_and_extract(root, url, latest_version)
 
 
 if __name__ == "__main__":