Merge pull request #14851 from dotty-staging/fix-2887

Guard against stackoverflows when instantiating HK-Lambdas

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeApplications.scala

@@ -343,11 +343,15 @@ class TypeApplications(val self: Type) extends AnyVal {
  }
  }
  if ((dealiased eq stripped) || followAlias)
- try {
+ try
  val instantiated = dealiased.instantiate(args)
  if (followAlias) instantiated.normalized else instantiated
- }
- catch { case ex: IndexOutOfBoundsException => AppliedType(self, args) }
+ catch
+ case ex: IndexOutOfBoundsException =>
+ AppliedType(self, args)
+ case ex: Throwable =>
+ handleRecursive("try to instantiate", i"$dealiased[$args%, %]", ex)
+
  else AppliedType(self, args)
  }
  else dealiased.resType match {

tests/neg/i2887b.check

@@ -0,0 +1,30 @@
+-- Error: tests/neg/i2887b.scala:11:7 ----------------------------------------------------------------------------------
+11 | baz(new A { type S[X[_] <: [_] =>> Any, Y[_]] = [Z] =>> X[Z][Y[Z]]; type I[X] = X })(1) // error
+ | ^
+ | Recursion limit exceeded.
+ | Maybe there is an illegal cyclic reference?
+ | If that's not the case, you could also try to increase the stacksize using the -Xss JVM option.
+ | A recurring operation is (inner to outer):
+ |
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | ...
+ |
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]
+ | try to instantiate Z[Z]

tests/neg/i2887b.scala

@@ -1,13 +1,13 @@
-trait A { type S[X[_] <: [_] => Any, Y[_]] <: [_] => Any; type I[_] } // error // error
-trait B { type S[X[_],Y[_]]; type I[_] <: [_] => Any } // error
+trait A { type S[X[_] <: [_] =>> Any, Y[_]] <: [_] =>> Any; type I[_] }
+trait B { type S[X[_],Y[_]]; type I[_] <: [_] =>> Any }
 trait C { type M <: B }
 trait D { type M >: A }
 
 object Test {
  def test(x: C with D): Unit = {
- def foo(a: A, b: B)(z: a.S[b.I,a.I][b.S[a.I,a.I]]) = z // error
+ def foo(a: A, b: B)(z: a.S[b.I,a.I][b.S[a.I,a.I]]) = z
  def bar(a: A, y: x.M) = foo(a,y)
  def baz(a: A) = bar(a, a)
- baz(new A { type S[X[_] <: [_] => Any, Y[_]] = [Z] => X[Z][Y[Z]]; type I[X] = X })(1) // error // error
+ baz(new A { type S[X[_] <: [_] =>> Any, Y[_]] = [Z] =>> X[Z][Y[Z]]; type I[X] = X })(1) // error
  }
-}
+}