Generate separated parametrized tests for test sets with different result type #510 (#652)

Author: sofurihafe

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -28,6 +28,7 @@ import org.utbot.framework.plugin.api.util.isPrimitive
 import org.utbot.framework.plugin.api.util.jClass
 import org.utbot.framework.plugin.api.util.longClassId
 import org.utbot.framework.plugin.api.util.method
+import org.utbot.framework.plugin.api.util.objectClassId
 import org.utbot.framework.plugin.api.util.primitiveTypeJvmNameOrNull
 import org.utbot.framework.plugin.api.util.safeJField
 import org.utbot.framework.plugin.api.util.shortClassId
@@ -98,22 +99,6 @@ data class UtMethodTestSet(
  val clustersInfo: List<Pair<UtClusterInfo?, IntRange>> = listOf(null to executions.indices)
 )
 
-data class CgMethodTestSet private constructor(
- val executableId: ExecutableId,
- val executions: List<UtExecution> = emptyList(),
- val jimpleBody: JimpleBody? = null,
- val errors: Map<String, Int> = emptyMap(),
- val clustersInfo: List<Pair<UtClusterInfo?, IntRange>> = listOf(null to executions.indices)
-) {
- constructor(from: UtMethodTestSet) : this(
- from.method.callable.executableId,
- from.executions,
- from.jimpleBody,
- from.errors,
- from.clustersInfo
- )
-}
-
 data class Step(
  val stmt: Stmt,
  val depth: Int,

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/CodeGenerator.kt

@@ -6,12 +6,12 @@ import org.utbot.framework.codegen.ParametrizedTestSource
 import org.utbot.framework.codegen.RuntimeExceptionTestsBehaviour
 import org.utbot.framework.codegen.StaticsMocking
 import org.utbot.framework.codegen.TestFramework
+import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
 import org.utbot.framework.codegen.model.constructor.context.CgContext
 import org.utbot.framework.codegen.model.constructor.tree.CgTestClassConstructor
 import org.utbot.framework.codegen.model.constructor.tree.TestsGenerationReport
 import org.utbot.framework.codegen.model.tree.CgTestClassFile
 import org.utbot.framework.codegen.model.visitor.CgAbstractRenderer
-import org.utbot.framework.plugin.api.CgMethodTestSet
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.CodegenLanguage
 import org.utbot.framework.plugin.api.ExecutableId

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/CgMethodTestSet.kt

@@ -0,0 +1,68 @@
+package org.utbot.framework.codegen.model.constructor
+
+import org.utbot.framework.plugin.api.ClassId
+import org.utbot.framework.plugin.api.ExecutableId
+import org.utbot.framework.plugin.api.UtClusterInfo
+import org.utbot.framework.plugin.api.UtExecution
+import org.utbot.framework.plugin.api.UtExecutionSuccess
+import org.utbot.framework.plugin.api.UtMethodTestSet
+import org.utbot.framework.plugin.api.util.executableId
+import org.utbot.framework.plugin.api.util.objectClassId
+import soot.jimple.JimpleBody
+
+data class CgMethodTestSet private constructor(
+ val executableId: ExecutableId,
+ val jimpleBody: JimpleBody? = null,
+ val errors: Map<String, Int> = emptyMap(),
+ val clustersInfo: List<Pair<UtClusterInfo?, IntRange>>,
+) {
+ var executions: List<UtExecution> = emptyList()
+ private set
+
+ constructor(from: UtMethodTestSet) : this(
+ from.method.callable.executableId,
+ from.jimpleBody,
+ from.errors,
+ from.clustersInfo
+ ) {
+ executions = from.executions
+ }
+
+ /**
+ * Splits [CgMethodTestSet] into separate test sets having
+ * unique result model [ClassId] in each subset.
+ */
+ fun splitExecutionsByResult() : List<CgMethodTestSet> {
+ val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
+ val executionsByResult: Map<ClassId, List<UtExecution>> =
+ if (successfulExecutions.isNotEmpty()) {
+ successfulExecutions.groupBy { (it.result as UtExecutionSuccess).model.classId }
+ } else {
+ mapOf(objectClassId to executions)
+ }
+
+ return executionsByResult.map{ (_, executions) -> substituteExecutions(executions) }
+ }
+
+ /**
+ * Finds a [ClassId] of all result models in executions.
+ *
+ * Tries to find an unique result type in testSets or
+ * gets executable return type.
+ */
+ fun resultType(): ClassId {
+ val successfulExecutions = executions.filter { it.result is UtExecutionSuccess }
+ return if (successfulExecutions.isNotEmpty()) {
+ successfulExecutions
+ .map { (it.result as UtExecutionSuccess).model.classId }
+ .distinct()
+ .singleOrNull()
+ ?: executableId.returnType
+ } else {
+ executableId.returnType
+ }
+ }
+
+ private fun substituteExecutions(newExecutions: List<UtExecution>): CgMethodTestSet =
+ copy().apply { executions = newExecutions }
+}

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/context/CgContext.kt

@@ -51,9 +51,9 @@ import kotlinx.collections.immutable.PersistentSet
 import kotlinx.collections.immutable.persistentListOf
 import kotlinx.collections.immutable.persistentMapOf
 import kotlinx.collections.immutable.persistentSetOf
+import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
 import org.utbot.framework.codegen.model.constructor.builtin.streamsDeepEqualsMethodId
 import org.utbot.framework.codegen.model.tree.CgParameterKind
-import org.utbot.framework.plugin.api.CgMethodTestSet
 import org.utbot.framework.plugin.api.util.id
 import org.utbot.framework.plugin.api.util.isCheckedException
 import org.utbot.framework.plugin.api.util.isSubtypeOf

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgMethodConstructor.kt

@@ -8,6 +8,7 @@ import org.utbot.framework.codegen.Junit5
 import org.utbot.framework.codegen.ParametrizedTestSource
 import org.utbot.framework.codegen.RuntimeExceptionTestsBehaviour.PASS
 import org.utbot.framework.codegen.TestNg
+import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
 import org.utbot.framework.codegen.model.constructor.builtin.closeMethodIdOrNull
 import org.utbot.framework.codegen.model.constructor.builtin.forName
 import org.utbot.framework.codegen.model.constructor.builtin.getClass
@@ -84,7 +85,6 @@ import org.utbot.framework.fields.ExecutionStateAnalyzer
 import org.utbot.framework.fields.FieldPath
 import org.utbot.framework.plugin.api.BuiltinClassId
 import org.utbot.framework.plugin.api.BuiltinMethodId
-import org.utbot.framework.plugin.api.CgMethodTestSet
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.CodegenLanguage
 import org.utbot.framework.plugin.api.ConcreteExecutionFailureException
@@ -1265,9 +1265,7 @@ internal class CgMethodConstructor(val context: CgContext) : CgContextOwner by c
  currentMethodParameters[CgParameterKind.Argument(index)] = argument.parameter
  }
 
- val method = currentExecutable!!
- val expectedResultClassId = wrapTypeIfRequired(method.returnType)
-
+ val expectedResultClassId = wrapTypeIfRequired(testSet.resultType())
  if (expectedResultClassId != voidClassId) {
  val wrappedType = wrapIfPrimitive(expectedResultClassId)
  //We are required to wrap the type of expected result if it is primitive

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt

@@ -3,6 +3,7 @@ package org.utbot.framework.codegen.model.constructor.tree
 import org.utbot.common.appendHtmlLine
 import org.utbot.engine.displayName
 import org.utbot.framework.codegen.ParametrizedTestSource
+import org.utbot.framework.codegen.model.constructor.CgMethodTestSet
 import org.utbot.framework.codegen.model.constructor.context.CgContext
 import org.utbot.framework.codegen.model.constructor.context.CgContextOwner
 import org.utbot.framework.codegen.model.constructor.util.CgComponents
@@ -23,7 +24,6 @@ import org.utbot.framework.codegen.model.tree.buildTestClass
 import org.utbot.framework.codegen.model.tree.buildTestClassBody
 import org.utbot.framework.codegen.model.tree.buildTestClassFile
 import org.utbot.framework.codegen.model.visitor.importUtilMethodDependencies
-import org.utbot.framework.plugin.api.CgMethodTestSet
 import org.utbot.framework.plugin.api.ExecutableId
 import org.utbot.framework.plugin.api.MethodId
 import org.utbot.framework.plugin.api.UtMethodTestSet
@@ -83,7 +83,7 @@ internal class CgTestClassConstructor(val context: CgContext) :
  return null
  }
 
- val (methodUnderTest, executions, _, _, clustersInfo) = testSet
+ val (methodUnderTest, _, _, clustersInfo) = testSet
  val regions = mutableListOf<CgRegion<CgMethod>>()
  val requiredFields = mutableListOf<CgParameterDeclaration>()
 
@@ -94,7 +94,7 @@ internal class CgTestClassConstructor(val context: CgContext) :
  emptyLineIfNeeded()
  for (i in executionIndices) {
  runCatching {
- currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, executions[i])
+ currentTestCaseTestMethods += methodConstructor.createTestMethod(methodUnderTest, testSet.executions[i])
  }.onFailure { e -> processFailure(testSet, e) }
  }
  val clusterHeader = clusterSummary?.header
@@ -107,22 +107,9 @@ internal class CgTestClassConstructor(val context: CgContext) :
  }
  }
  ParametrizedTestSource.PARAMETRIZE -> {
- runCatching {
- val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(testSet.executableId)
-
- val parameterizedTestMethod =
- methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
-
- requiredFields += parameterizedTestMethod.requiredFields
-
- cgDataProviderMethods +=
- methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
-
- regions += CgSimpleRegion(
- "Parameterized test for method ${methodUnderTest.displayName}",
- listOf(parameterizedTestMethod),
- )
- }.onFailure { error -> processFailure(testSet, error) }
+ for (currentTestSet in testSet.splitExecutionsByResult()) {
+ createParametrizedTestAndDataProvider(currentTestSet, requiredFields, regions, methodUnderTest)
+ }
  }
  }
 
@@ -141,6 +128,30 @@ internal class CgTestClassConstructor(val context: CgContext) :
  .merge(failure.description, 1, Int::plus)
  }
 
+ private fun createParametrizedTestAndDataProvider(
+ testSet: CgMethodTestSet,
+ requiredFields: MutableList<CgParameterDeclaration>,
+ regions: MutableList<CgRegion<CgMethod>>,
+ methodUnderTest: ExecutableId,
+ ) {
+ runCatching {
+ val dataProviderMethodName = nameGenerator.dataProviderMethodNameFor(testSet.executableId)
+
+ val parameterizedTestMethod =
+ methodConstructor.createParameterizedTestMethod(testSet, dataProviderMethodName)
+
+ requiredFields += parameterizedTestMethod.requiredFields
+
+ cgDataProviderMethods +=
+ methodConstructor.createParameterizedTestDataProvider(testSet, dataProviderMethodName)
+
+ regions += CgSimpleRegion(
+ "Parameterized test for method ${methodUnderTest.displayName}",
+ listOf(parameterizedTestMethod),
+ )
+ }.onFailure { error -> processFailure(testSet, error) }
+ }
+
  // TODO: collect imports of util methods
  private fun createUtilMethods(): List<CgUtilMethod> {
  val utilMethods = mutableListOf<CgUtilMethod>()

utbot-framework/src/test/kotlin/org/utbot/framework/codegen/TestCodeGeneratorPipeline.kt

@@ -130,10 +130,17 @@ class TestCodeGeneratorPipeline(private val testFrameworkConfiguration: TestFram
  "Errors regions has been generated: $errorText"
  }
 
- require(generatedMethodsCount == expectedNumberOfGeneratedMethods) {
- "Something went wrong during the code generation for ${classUnderTest.simpleName}. " +
- "Expected to generate $expectedNumberOfGeneratedMethods test methods, " +
- "but got only $generatedMethodsCount"
+ // for now, we skip a comparing of generated and expected test methods
+ // in parametrized test generation mode
+ // because there are problems with determining expected number of methods,
+ // due to a feature that generates several separated parametrized tests
+ // when we have several executions with different result type
+ if (parametrizedTestSource != ParametrizedTestSource.PARAMETRIZE) {
+ require(generatedMethodsCount == expectedNumberOfGeneratedMethods) {
+ "Something went wrong during the code generation for ${classUnderTest.simpleName}. " +
+ "Expected to generate $expectedNumberOfGeneratedMethods test methods, " +
+ "but got only $generatedMethodsCount"
+ }
  }
  }.onFailure {
  val classes = listOf(classPipeline).retrieveClasses()