Undo loop optimization (#2077)

* Undo loop optimization * Undo loop optimization * Restore changes from unmergeable PRs * Unused file * Port changes from skipper PRs * Minimize changes * Further minimize * Further simplify * Fix test

Author: Mikhail Arkhipov

src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs

@@ -40,6 +40,7 @@ private AnalysisModuleKey(string name, string filePath, bool isTypeshed, bool is
  Name = name;
  FilePath = filePath;
  IsTypeshed = isTypeshed;
+ IsNonUserAsDocument = isNonUserAsDocument;
  }
 
  public AnalysisModuleKey GetNonUserAsDocumentKey() => new AnalysisModuleKey(Name, FilePath, IsTypeshed, true);
@@ -70,6 +71,8 @@ public void Deconstruct(out string moduleName, out string filePath, out bool isT
 
  public override string ToString() => $"{Name}({FilePath})";
 
+ public bool IsNonUserAsDocument { get; }
+
  private static bool IsNonUserAsDocumentModule(IPythonModule module)
  => (module.IsNonUserFile() || module.IsCompiled()) && module is IDocument document && document.IsOpen;
  }

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -40,9 +40,9 @@ internal abstract class AnalysisWalker : PythonWalker {
  public PythonAst Ast => Eval.Ast;
  protected ModuleSymbolTable SymbolTable => Eval.SymbolTable;
 
- protected AnalysisWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) {
+ protected AnalysisWalker(ExpressionEval eval) {
  Eval = eval;
- ImportHandler = new ImportHandler(this, importedVariableHandler);
+ ImportHandler = new ImportHandler(this);
  AssignmentHandler = new AssignmentHandler(this);
  LoopHandler = new LoopHandler(this);
  ConditionalHandler = new ConditionalHandler(this);

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -21,6 +21,11 @@ namespace Microsoft.Python.Analysis.Analyzer {
  /// Represents document that can be analyzed asynchronously.
  /// </summary>
  internal interface IAnalyzable {
+ /// <summary>
+ /// Returns object that can calculate dependencies of this entry.
+ /// </summary>
+ IDependencyProvider DependencyProvider { get; }
+
  /// <summary>
  /// Notifies document that analysis is about to begin.
  /// </summary>

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs

@@ -60,8 +60,8 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
  public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty;
 
  public Location GetLocationOfName(Node node) {
- if (node == null || 
- Module.ModuleType == ModuleType.Specialized || Module.ModuleType == ModuleType.Compiled || 
+ if (node == null ||
+ Module.ModuleType == ModuleType.Specialized || Module.ModuleType == ModuleType.Compiled ||
  Module.ModuleType == ModuleType.CompiledBuiltin || Module.ModuleType == ModuleType.Builtins) {
  return DefaultLocation;
  }

src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs

@@ -14,7 +14,6 @@
 // permissions and limitations under the License.
 
 using System;
-using Microsoft.Python.Analysis.Analyzer.Handlers;
 using Microsoft.Python.Analysis.Analyzer.Symbols;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Parsing.Ast;
@@ -29,7 +28,7 @@ internal sealed class FunctionCallEvaluator: AnalysisWalker {
  private readonly FunctionDefinition _function;
  private IMember _result;
 
- public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval, SimpleImportedVariableHandler.Instance) {
+ public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) {
  _declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule));
  _eval = eval ?? throw new ArgumentNullException(nameof(eval));
  _function = fd ?? throw new ArgumentNullException(nameof(fd));

src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

@@ -68,13 +68,6 @@ private void AssignVariables(FromImportStatement node, IImportSearchResult impor
  if (!string.IsNullOrEmpty(variableName)) {
  DeclareVariable(variableModule, memberName, imports, variableName, node.StartIndex, nameExpression);
  }
-
- if (imports is IImportChildrenSource cs
- && cs.TryGetChildImport(memberName, out var csr)
- && HandleImportSearchResult(csr, variableModule, null, names[i], out var childModule)) {
-
- _importedVariableHandler.EnsureModule(childModule);
- }
  }
  }
 
@@ -83,11 +76,11 @@ private void HandleModuleImportStar(PythonVariableModule variableModule, IImport
  // from self import * won't define any new members
  return;
  }
-
  // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
  var memberNames = imports is ImplicitPackageImport
  ? variableModule.GetMemberNames()
- : _importedVariableHandler.GetMemberNames(variableModule).ToArray();
+ : variableModule.Analysis.StarImportMemberNames
+ ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")).ToArray();
 
  foreach (var memberName in memberNames) {
  DeclareVariable(variableModule, memberName, imports, memberName, importPosition, nameExpression);
@@ -114,27 +107,14 @@ private void DeclareVariable(PythonVariableModule variableModule, string memberN
  value = GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
 
  // First try exported or child submodules.
- var member = variableModule.GetMember(memberName);
+ value = value ?? variableModule.GetMember(memberName);
 
  // Value may be variable or submodule. If it is variable, we need it in order to add reference.
- var variable = _importedVariableHandler.GetVariable(variableModule, memberName);
-
- if (member is PythonVariableModule vm && vm.Equals(variable?.Value)) {
- // If member is submodule, use actual variable so it can be linked through for goto definition.
- value = variable;
- } else if (value == null) {
- if (member is PythonVariableModule) {
- // If member is submodule, use it.
- value = member;
- } else if (variable?.Value != null) {
- // Otherwise use variable, if available so references can be linked.
- value = variable;
- } else if (member != null) {
- value = member;
- } else {
- value = Eval.UnknownType;
- }
- }
+ var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
+ value = variable?.Value?.Equals(value) == true ? variable : value;
+
+ // If nothing is exported, variables are still accessible.
+ value = value ?? variableModule.Analysis?.GlobalScope?.Variables[memberName]?.Value ?? Eval.UnknownType;
  }
 
  // Do not allow imported variables to override local declarations

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -29,12 +29,9 @@
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
  internal sealed partial class ImportHandler : StatementHandler {
- private readonly IImportedVariableHandler _importedVariableHandler;
  private readonly Dictionary<string, PythonVariableModule> _variableModules = new Dictionary<string, PythonVariableModule>();
 
- public ImportHandler(AnalysisWalker walker, in IImportedVariableHandler importedVariableHandler) : base(walker) {
- _importedVariableHandler = importedVariableHandler;
- }
+ public ImportHandler(AnalysisWalker walker) : base(walker) { }
 
  public bool HandleImport(ImportStatement node) {
  if (Module.ModuleType == ModuleType.Specialized) {
@@ -68,9 +65,7 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
  lastModule = default;
  break;
  }
-
  resolvedModules[i] = (nameExpression.Name, lastModule);
- _importedVariableHandler.EnsureModule(lastModule);
  }
 
  // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
@@ -94,6 +89,15 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa
  Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, moduleImportExpression.Names[0]);
  }
  }
+
+ // import a.b.c.d => declares a, b in the current module, c in b, d in c.
+ for (var i = 1; i < resolvedModules.Length - 1; i++) {
+ var (childName, childModule) = resolvedModules[i + 1];
+ if (!string.IsNullOrEmpty(childName) && childModule != null) {
+ var parent = resolvedModules[i].module;
+ parent?.AddChildModule(childName, childModule);
+ }
+ }
  }
 
  private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonVariableModule parent, in NameExpression asNameExpression, in Node location, out PythonVariableModule variableModule) {
@@ -108,7 +112,8 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV
  return TryGetPackageFromImport(packageImport, parent, out variableModule);
  case RelativeImportBeyondTopLevel importBeyondTopLevel:
  var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName);
- Eval.ReportDiagnostics(Eval.Module.Uri, new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
+ Eval.ReportDiagnostics(Eval.Module.Uri,
+ new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis));
  variableModule = default;
  return false;
  case ImportNotFound importNotFound:
@@ -172,7 +177,7 @@ private bool TryGetModulePossibleImport(PossibleModuleImport possibleModuleImpor
  return false;
  }
  }
- 
+
  return true;
  }
 
@@ -195,7 +200,7 @@ private PythonVariableModule GetOrCreateVariableModule(in string fullName, in Py
  variableModule = new PythonVariableModule(fullName, Eval.Interpreter);
  _variableModules[fullName] = variableModule;
  }
- 
+
  parentModule?.AddChildModule(memberName, variableModule);
  return variableModule;
  }

src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs

@@ -18,7 +18,6 @@
 using System.Linq;
 using System.Threading;
 using Microsoft.Python.Analysis.Analyzer.Evaluation;
-using Microsoft.Python.Analysis.Analyzer.Handlers;
 using Microsoft.Python.Analysis.Documents;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Types.Collections;
@@ -38,7 +37,8 @@ internal class ModuleWalker : AnalysisWalker {
  private int _allReferencesCount;
  private bool _allIsUsable = true;
 
- public ModuleWalker(ExpressionEval eval, IImportedVariableHandler importedVariableHandler) : base(eval, importedVariableHandler) {
+ public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast, CancellationToken cancellationToken)
+ : base(new ExpressionEval(services, module, ast)) {
  _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null;
  _cancellationToken = CancellationToken.None;
  }
@@ -206,7 +206,7 @@ public void Complete() {
  new StubMerger(Eval).MergeStub(_stubAnalysis, _cancellationToken);
 
  if (_allIsUsable && _allReferencesCount >= 1 && GlobalScope.Variables.TryGetVariable(AllVariableName, out var variable)
- && variable.Value is IPythonCollection collection && collection.IsExact) {
+ && variable?.Value is IPythonCollection collection && collection.IsExact) {
  StarImportMemberNames = collection.Contents
  .OfType<IPythonConstant>()
  .Select(c => c.GetString())