Implement global virtual environment locator (microsoft#14416)

* Add Global virtual environments locator * Fix tests and address comments * Add missing files * Address comments * Move envs * Test Fix * Address comments * Address more comments * Try a different approach * . * . * Remove version parsing. * more tweaks

Author: karthiknadig

src/client/pythonEnvironments/common/commonUtils.ts

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as fsapi from 'fs-extra';
+import * as path from 'path';
+import { chain, iterable } from '../../common/utils/async';
+import { getOSType, OSType } from '../../common/utils/platform';
+import { isPosixPythonBin } from './posixUtils';
+import { isWindowsPythonExe } from './windowsUtils';
+
+export async function* findInterpretersInDir(root:string, recurseLevels?:number): AsyncIterableIterator<string> {
+ const dirContents = (await fsapi.readdir(root)).map((c) => path.join(root, c));
+ const os = getOSType();
+ const checkBin = os === OSType.Windows ? isWindowsPythonExe : isPosixPythonBin;
+ const generators = dirContents.map((item) => {
+ async function* generator() {
+ const stat = await fsapi.lstat(item);
+
+ if (stat.isDirectory()) {
+ if (recurseLevels && recurseLevels > 0) {
+ const subItems = findInterpretersInDir(item, recurseLevels - 1);
+
+ for await (const subItem of subItems) {
+ yield subItem;
+ }
+ }
+ } else if (checkBin(item)) {
+ yield item;
+ }
+ }
+
+ return generator();
+ });
+
+ yield* iterable(chain(generators));
+}

src/client/pythonEnvironments/common/environmentIdentifier.ts

@@ -4,9 +4,7 @@
 import { isCondaEnvironment } from '../discovery/locators/services/condaLocator';
 import { isPipenvEnvironment } from '../discovery/locators/services/pipEnvHelper';
 import { isPyenvEnvironment } from '../discovery/locators/services/pyenvLocator';
-import { isVenvEnvironment } from '../discovery/locators/services/venvLocator';
-import { isVirtualenvEnvironment } from '../discovery/locators/services/virtualenvLocator';
-import { isVirtualenvwrapperEnvironment } from '../discovery/locators/services/virtualenvwrapperLocator';
+import { isVenvEnvironment, isVirtualenvEnvironment, isVirtualenvwrapperEnvironment } from '../discovery/locators/services/virtualEnvironmentIdentifier';
 import { isWindowsStoreEnvironment } from '../discovery/locators/services/windowsStoreLocator';
 import { EnvironmentType } from '../info';
 

src/client/pythonEnvironments/common/virtualenvwrapperUtils.ts

@@ -0,0 +1,163 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as fsapi from 'fs-extra';
+import { toUpper, uniq } from 'lodash';
+import * as path from 'path';
+import { traceVerbose } from '../../../../common/logger';
+import { chain, iterable } from '../../../../common/utils/async';
+import {
+ getEnvironmentVariable, getOSType, getUserHomeDir, OSType,
+} from '../../../../common/utils/platform';
+import {
+ PythonEnvInfo, PythonEnvKind, UNKNOWN_PYTHON_VERSION,
+} from '../../../base/info';
+import { buildEnvInfo } from '../../../base/info/env';
+import { ILocator, IPythonEnvsIterator } from '../../../base/locator';
+import { PythonEnvsWatcher } from '../../../base/watcher';
+import { findInterpretersInDir } from '../../../common/commonUtils';
+import { getFileInfo, pathExists } from '../../../common/externalDependencies';
+import { isVenvEnvironment, isVirtualenvEnvironment, isVirtualenvwrapperEnvironment } from './virtualEnvironmentIdentifier';
+
+const DEFAULT_SEARCH_DEPTH = 2;
+
+/**
+ * Gets all default virtual environment locations. This uses WORKON_HOME,
+ * and user home directory to find some known locations where global virtual
+ * environments are often created.
+ */
+async function getGlobalVirtualEnvDirs(): Promise<string[]> {
+ const venvDirs:string[] = [];
+
+ const workOnHome = getEnvironmentVariable('WORKON_HOME');
+ if (workOnHome && await pathExists(workOnHome)) {
+ venvDirs.push(workOnHome);
+ }
+
+ const homeDir = getUserHomeDir();
+ if (homeDir && await pathExists(homeDir)) {
+ const os = getOSType();
+ let subDirs = ['Envs', 'envs', '.direnv', '.venvs', '.virtualenvs'];
+ if (os === OSType.Windows) {
+ subDirs = uniq(subDirs.map(toUpper));
+ }
+
+ (await fsapi.readdir(homeDir))
+ .filter((d) => subDirs.includes(os === OSType.Windows ? d.toUpperCase() : d))
+ .forEach((d) => venvDirs.push(path.join(homeDir, d)));
+ }
+
+ return venvDirs;
+}
+
+/**
+ * Gets the virtual environment kind for a given interpreter path.
+ * This only checks for environments created using venv, virtualenv,
+ * and virtualenvwrapper based environments.
+ * @param interpreterPath: Absolute path to the interpreter paths.
+ */
+async function getVirtualEnvKind(interpreterPath:string): Promise<PythonEnvKind> {
+ if (await isVenvEnvironment(interpreterPath)) {
+ return PythonEnvKind.Venv;
+ }
+
+ if (await isVirtualenvwrapperEnvironment(interpreterPath)) {
+ return PythonEnvKind.VirtualEnvWrapper;
+ }
+
+ if (await isVirtualenvEnvironment(interpreterPath)) {
+ return PythonEnvKind.VirtualEnv;
+ }
+
+ return PythonEnvKind.Unknown;
+}
+
+/**
+ * Finds and resolves virtual environments created in known global locations.
+ */
+export class GlobalVirtualEnvironmentLocator extends PythonEnvsWatcher implements ILocator {
+ private virtualEnvKinds = [
+ PythonEnvKind.Venv,
+ PythonEnvKind.VirtualEnv,
+ PythonEnvKind.VirtualEnvWrapper,
+ ];
+
+ public constructor(private readonly searchDepth?:number) {
+ super();
+ }
+
+ public iterEnvs(): IPythonEnvsIterator {
+ // Number of levels of sub-directories to recurse when looking for
+ // interpreters
+ const searchDepth = this.searchDepth ?? DEFAULT_SEARCH_DEPTH;
+
+ async function* iterator(virtualEnvKinds:PythonEnvKind[]) {
+ const envRootDirs = await getGlobalVirtualEnvDirs();
+ const envGenerators = envRootDirs.map((envRootDir) => {
+ async function* generator() {
+ traceVerbose(`Searching for global virtual envs in: ${envRootDir}`);
+
+ const envGenerator = findInterpretersInDir(envRootDir, searchDepth);
+
+ for await (const env of envGenerator) {
+ // We only care about python.exe (on windows) and python (on linux/mac)
+ // Other version like python3.exe or python3.8 are often symlinks to
+ // python.exe or python in the same directory in the case of virtual
+ // environments.
+ const name = path.basename(env).toLowerCase();
+ if (name === 'python.exe' || name === 'python') {
+ // We should extract the kind here to avoid doing is*Environment()
+ // check multiple times. Those checks are file system heavy and
+ // we can use the kind to determine this anyway.
+ const kind = await getVirtualEnvKind(env);
+
+ const timeData = await getFileInfo(env);
+ if (virtualEnvKinds.includes(kind)) {
+ traceVerbose(`Global Virtual Environment: [added] ${env}`);
+ const envInfo = buildEnvInfo({
+ kind,
+ executable: env,
+ version: UNKNOWN_PYTHON_VERSION,
+ });
+ envInfo.executable.ctime = timeData.ctime;
+ envInfo.executable.mtime = timeData.mtime;
+ yield envInfo;
+ } else {
+ traceVerbose(`Global Virtual Environment: [skipped] ${env}`);
+ }
+ } else {
+ traceVerbose(`Global Virtual Environment: [skipped] ${env}`);
+ }
+ }
+ }
+ return generator();
+ });
+
+ yield* iterable(chain(envGenerators));
+ }
+
+ return iterator(this.virtualEnvKinds);
+ }
+
+ public async resolveEnv(env: string | PythonEnvInfo): Promise<PythonEnvInfo | undefined> {
+ const executablePath = typeof env === 'string' ? env : env.executable.filename;
+ if (await pathExists(executablePath)) {
+ // We should extract the kind here to avoid doing is*Environment()
+ // check multiple times. Those checks are file system heavy and
+ // we can use the kind to determine this anyway.
+ const kind = await getVirtualEnvKind(executablePath);
+ if (this.virtualEnvKinds.includes(kind)) {
+ const timeData = await getFileInfo(executablePath);
+ const envInfo = buildEnvInfo({
+ kind,
+ version: UNKNOWN_PYTHON_VERSION,
+ executable: executablePath,
+ });
+ envInfo.executable.ctime = timeData.ctime;
+ envInfo.executable.mtime = timeData.mtime;
+ return envInfo;
+ }
+ }
+ return undefined;
+ }
+}

src/client/pythonEnvironments/discovery/locators/services/globalVirtualEnvronmentLocator.ts

@@ -0,0 +1,110 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import * as fsapi from 'fs-extra';
+import * as path from 'path';
+import {
+ getEnvironmentVariable, getOSType, getUserHomeDir, OSType,
+} from '../../../../common/utils/platform';
+import { pathExists } from '../../../common/externalDependencies';
+
+/**
+ * Checks if the given interpreter belongs to a venv based environment.
+ * @param {string} interpreterPath: Absolute path to the python interpreter.
+ * @returns {boolean} : Returns true if the interpreter belongs to a venv environment.
+ */
+export async function isVenvEnvironment(interpreterPath:string): Promise<boolean> {
+ const pyvenvConfigFile = 'pyvenv.cfg';
+
+ // Check if the pyvenv.cfg file is in the parent directory relative to the interpreter.
+ // env
+ // |__ pyvenv.cfg <--- check if this file exists
+ // |__ bin or Scripts
+ // |__ python <--- interpreterPath
+ const venvPath1 = path.join(path.dirname(path.dirname(interpreterPath)), pyvenvConfigFile);
+
+ // Check if the pyvenv.cfg file is in the directory as the interpreter.
+ // env
+ // |__ pyvenv.cfg <--- check if this file exists
+ // |__ python <--- interpreterPath
+ const venvPath2 = path.join(path.dirname(interpreterPath), pyvenvConfigFile);
+
+ // The paths are ordered in the most common to least common
+ const venvPaths = [venvPath1, venvPath2];
+
+ // We don't need to test all at once, testing each one here
+ for (const venvPath of venvPaths) {
+ if (await pathExists(venvPath)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/**
+ * Checks if the given interpreter belongs to a virtualenv based environment.
+ * @param {string} interpreterPath: Absolute path to the python interpreter.
+ * @returns {boolean} : Returns true if the interpreter belongs to a virtualenv environment.
+ */
+export async function isVirtualenvEnvironment(interpreterPath:string): Promise<boolean> {
+ // Check if there are any activate.* files in the same directory as the interpreter.
+ //
+ // env
+ // |__ activate, activate.* <--- check if any of these files exist
+ // |__ python <--- interpreterPath
+ const directory = path.dirname(interpreterPath);
+ const files = await fsapi.readdir(directory);
+ const regex = /^activate(\.([A-z]|\d)+)?$/i;
+
+ return files.find((file) => regex.test(file)) !== undefined;
+}
+
+async function getDefaultVirtualenvwrapperDir(): Promise<string> {
+ const homeDir = getUserHomeDir() || '';
+
+ // In Windows, the default path for WORKON_HOME is %USERPROFILE%\Envs.
+ // If 'Envs' is not available we should default to '.virtualenvs'. Since that
+ // is also valid for windows.
+ if (getOSType() === OSType.Windows) {
+ // ~/Envs with uppercase 'E' is the default home dir for
+ // virtualEnvWrapper.
+ const envs = path.join(homeDir, 'Envs');
+ if (await pathExists(envs)) {
+ return envs;
+ }
+ }
+ return path.join(homeDir, '.virtualenvs');
+}
+
+function getWorkOnHome(): Promise<string> {
+ // The WORKON_HOME variable contains the path to the root directory of all virtualenvwrapper environments.
+ // If the interpreter path belongs to one of them then it is a virtualenvwrapper type of environment.
+ const workOnHome = getEnvironmentVariable('WORKON_HOME');
+ if (workOnHome) {
+ return Promise.resolve(workOnHome);
+ }
+ return getDefaultVirtualenvwrapperDir();
+}
+
+/**
+ * Checks if the given interpreter belongs to a virtualenvWrapper based environment.
+ * @param {string} interpreterPath: Absolute path to the python interpreter.
+ * @returns {boolean}: Returns true if the interpreter belongs to a virtualenvWrapper environment.
+ */
+export async function isVirtualenvwrapperEnvironment(interpreterPath:string): Promise<boolean> {
+ const workOnHomeDir = await getWorkOnHome();
+ let pathToCheck = interpreterPath;
+ let workOnRoot = workOnHomeDir;
+
+ if (getOSType() === OSType.Windows) {
+ workOnRoot = workOnHomeDir.toUpperCase();
+ pathToCheck = interpreterPath.toUpperCase();
+ }
+
+ // For environment to be a virtualenvwrapper based it has to follow these two rules:
+ // 1. It should be in a sub-directory under the WORKON_HOME
+ // 2. It should be a valid virtualenv environment
+ return await pathExists(workOnHomeDir)
+ && pathToCheck.startsWith(`${workOnRoot}${path.sep}`)
+ && isVirtualenvEnvironment(interpreterPath);
+}