Added support for A/B testing and staged rollouts of new functionality (microsoft#5530)

* Added functionality * Added functionality * News entry * Code reviews * Addressed more comments * Added LS-enabled exp * Added tests for experiments * Corrected check and tests * Updated tests * Added inExperiment tests * Fixed bugs and updated tests * Send telemetry and log experiment group * Code reviews * More reviews * Accept suggestions * Send telemetry for the control group appropriately * Updated tests: * Ensure experiment groups are logged in the beginning * Modified decorator * Code reviews * Added docstrings * Minor fixing * code reviews... more coming * Reverted suggestion and added tests: * Add more doc strings * Refactoring and read config file * Added docstrings... * Simplified stuff * Local experiments file docstring * Updated tests * Added more tests * Added check for if experiments are valid * Added tests for registering IOCs * Move HTTP Client * Added tests * More tests * Refactored tests * Added crypto tests * Added httpClient.getJSON() tests * Apply quotes * Fix CI * Commit experiments file * Indent tests * Fix launch.json

Author: Kartik Raj

.vscode/launch.json

@@ -189,4 +189,4 @@
  ]
  }
  ]
-}
+}

experiments.json

@@ -0,0 +1,3 @@
+[
+ {}
+]

news/1 Enhancements/5042.md

@@ -0,0 +1 @@
+Add support for A/B testing and staged rollouts of new functionality

package.nls.json

@@ -102,6 +102,7 @@
  "DataScience.importingFormat": "Importing {0}",
  "DataScience.startingJupyter": "Starting Jupyter server",
  "DataScience.connectingToJupyter": "Connecting to Jupyter server",
+ "Experiments.inGroup": "User belongs to experiment group '{0}'",
  "Interpreters.RefreshingInterpreters": "Refreshing Python Interpreters",
  "Interpreters.LoadingInterpreters": "Loading Python Interpreters",
  "Common.doNotShowAgain": "Do not show again",

src/client/activation/activationManager.ts

@@ -26,7 +26,7 @@ export class ExtensionActivationManager implements IExtensionActivationManager {
  @inject(IInterpreterAutoSelectionService) private readonly autoSelection: IInterpreterAutoSelectionService,
  @inject(IApplicationDiagnostics) private readonly appDiagnostics: IApplicationDiagnostics,
  @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService
- ) {}
+ ) { }
 
  public dispose() {
  while (this.disposables.length > 0) {

src/client/activation/activationService.ts

@@ -9,8 +9,10 @@ import { LSNotSupportedDiagnosticServiceId } from '../application/diagnostics/ch
 import { IDiagnosticsService } from '../application/diagnostics/types';
 import { IApplicationShell, ICommandManager, IWorkspaceService } from '../common/application/types';
 import { STANDARD_OUTPUT_CHANNEL } from '../common/constants';
+import { LSControl, LSEnabled } from '../common/experimentGroups';
 import '../common/extensions';
-import { IConfigurationService, IDisposableRegistry, IOutputChannel, IPersistentStateFactory, IPythonSettings, Resource } from '../common/types';
+import { traceError } from '../common/logger';
+import { IConfigurationService, IDisposableRegistry, IExperimentsManager, IOutputChannel, IPersistentStateFactory, IPythonSettings, Resource } from '../common/types';
 import { swallowExceptions } from '../common/utils/decorators';
 import { IServiceContainer } from '../ioc/types';
 import { sendTelemetryEvent } from '../telemetry';
@@ -33,7 +35,8 @@ export class LanguageServerExtensionActivationService implements IExtensionActiv
  private resource!: Resource;
 
  constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer,
- @inject(IPersistentStateFactory) private stateFactory: IPersistentStateFactory) {
+ @inject(IPersistentStateFactory) private stateFactory: IPersistentStateFactory,
+ @inject(IExperimentsManager) private readonly abExperiments: IExperimentsManager) {
  this.workspaceService = this.serviceContainer.get<IWorkspaceService>(IWorkspaceService);
  this.output = this.serviceContainer.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
  this.appShell = this.serviceContainer.get<IApplicationShell>(IApplicationShell);
@@ -113,6 +116,39 @@ export class LanguageServerExtensionActivationService implements IExtensionActiv
  sendTelemetryEvent(EventName.PYTHON_LANGUAGE_SERVER_SWITCHED, undefined, { change: message });
  }
  }
+
+ /**
+ * Checks if user has not manually set `jediEnabled` setting
+ * @param resource
+ * @returns `true` if user has NOT manually added the setting and is using default configuration, `false` if user has `jediEnabled` setting added
+ */
+ public isJediUsingDefaultConfiguration(resource?: Uri): boolean {
+ const settings = this.workspaceService.getConfiguration('python', resource).inspect<boolean>('jediEnabled');
+ if (!settings) {
+ traceError('WorkspaceConfiguration.inspect returns `undefined` for setting `python.jediEnabled`');
+ return false;
+ }
+ return (settings.globalValue === undefined && settings.workspaceValue === undefined && settings.workspaceFolderValue === undefined);
+ }
+
+ /**
+ * Checks if user is using Jedi as intellisense
+ * @returns `true` if user is using jedi, `false` if user is using language server
+ */
+ public useJedi(): boolean {
+ if (this.isJediUsingDefaultConfiguration()) {
+ if (this.abExperiments.inExperiment(LSEnabled)) {
+ return false;
+ }
+ // Send telemetry if user is in control group
+ this.abExperiments.sendTelemetryIfInExperiment(LSControl);
+ }
+ const configurationService = this.serviceContainer.get<IConfigurationService>(IConfigurationService);
+ const enabled = configurationService.getSettings(this.resource).jediEnabled;
+ this.trackLangaugeServerSwitch(enabled).ignoreErrors();
+ return enabled;
+ }
+
  protected onWorkspaceFoldersChanged() {
  //If an activated workspace folder was removed, dispose its activator
  const workspaceKeys = this.workspaceService.workspaceFolders!.map(workspaceFolder => this.getWorkspacePathKey(workspaceFolder.uri));
@@ -153,12 +189,6 @@ export class LanguageServerExtensionActivationService implements IExtensionActiv
  this.serviceContainer.get<ICommandManager>(ICommandManager).executeCommand('workbench.action.reloadWindow');
  }
  }
- private useJedi(): boolean {
- const configurationService = this.serviceContainer.get<IConfigurationService>(IConfigurationService);
- const enabled = configurationService.getSettings(this.resource).jediEnabled;
- this.trackLangaugeServerSwitch(enabled).ignoreErrors();
- return enabled;
- }
  private getWorkspacePathKey(resource: Resource): string {
  return this.workspaceService.getWorkspaceFolderIdentifier(resource, workspacePathNameForGlobalWorkspaces);
  }