Unified error and exception clusters for all UtExecutions (#1250)

* Unified error and exception clusters for all UtExecutions * Fixed JavaDocs

Author: amandelpie

utbot-summary/src/main/kotlin/org/utbot/summary/Summarization.kt

@@ -23,15 +23,6 @@ import java.io.File
 import java.nio.file.Path
 import java.nio.file.Paths
 import mu.KotlinLogging
-import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
-import org.utbot.framework.plugin.api.UtExecutionSuccess
-import org.utbot.framework.plugin.api.UtExplicitlyThrownException
-import org.utbot.framework.plugin.api.UtImplicitlyThrownException
-import org.utbot.framework.plugin.api.UtOverflowFailure
-import org.utbot.framework.plugin.api.UtSandboxFailure
-import org.utbot.framework.plugin.api.UtStreamConsumingFailure
-import org.utbot.framework.plugin.api.UtTimeoutException
-import org.utbot.framework.plugin.api.util.humanReadableName
 import org.utbot.framework.plugin.api.util.jClass
 import org.utbot.fuzzer.FuzzedMethodDescription
 import org.utbot.fuzzer.FuzzedValue
@@ -222,9 +213,8 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
  testSet: UtMethodTestSet
  ): List<UtExecutionCluster> {
  val clustersToReturn: MutableList<UtExecutionCluster> = mutableListOf()
- val executionsProducedByFuzzer = testSet.executions.filterIsInstance<UtFuzzedExecution>()
- val successfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
- val unsuccessfulFuzzerExecutions = mutableListOf<UtFuzzedExecution>()
+ val testSetWithFuzzedExecutions = prepareTestSetWithFuzzedExecutions(testSet)
+ val executionsProducedByFuzzer = testSetWithFuzzedExecutions.executions as List<UtFuzzedExecution>
 
  if (executionsProducedByFuzzer.isNotEmpty()) {
  executionsProducedByFuzzer.forEach { utExecution ->
@@ -239,44 +229,21 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
  )
  }.firstOrNull()
  } catch (t: Throwable) {
- logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behavoiur
+ logger.error(t) { "Cannot create suggested test name for $utExecution" } // TODO: add better explanation or default behaviour
  null
  }
-
  utExecution.testMethodName = testMethodName?.testName
  utExecution.displayName = testMethodName?.displayName
  utExecution.summary = testMethodName?.javaDoc
-
- when (utExecution.result) {
- is UtConcreteExecutionFailure,
- is UtExplicitlyThrownException,
- is UtImplicitlyThrownException,
- is UtOverflowFailure,
- is UtSandboxFailure,
- is UtTimeoutException,
- is UtStreamConsumingFailure -> unsuccessfulFuzzerExecutions.add(utExecution)
- is UtExecutionSuccess -> successfulFuzzerExecutions.add(utExecution)
- }
  }
 
- if (successfulFuzzerExecutions.isNotEmpty()) {
- val clusterHeader = buildFuzzerClusterHeaderForSuccessfulExecutions(testSet)
-
- clustersToReturn.add(
- UtExecutionCluster(
- UtClusterInfo(clusterHeader, null),
- successfulFuzzerExecutions
- )
- )
- }
-
- if (unsuccessfulFuzzerExecutions.isNotEmpty()) {
- val clusterHeader = buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet)
+ val clusteredExecutions = groupFuzzedExecutions(testSetWithFuzzedExecutions)
 
+ clusteredExecutions.forEach {
  clustersToReturn.add(
  UtExecutionCluster(
- UtClusterInfo(clusterHeader, null),
- unsuccessfulFuzzerExecutions
+ UtClusterInfo(it.header),
+ it.executions
  )
  )
  }
@@ -285,20 +252,6 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
  return clustersToReturn.toList()
  }
 
- private fun buildFuzzerClusterHeaderForSuccessfulExecutions(testSet: UtMethodTestSet): String {
- val commentPrefix = "FUZZER:"
- val commentPostfix = "for method ${testSet.method.humanReadableName}"
-
- return "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix"
- }
-
- private fun buildFuzzerClusterHeaderForUnsuccessfulExecutions(testSet: UtMethodTestSet): String {
- val commentPrefix = "FUZZER:"
- val commentPostfix = "for method ${testSet.method.humanReadableName}"
-
- return "$commentPrefix ${ExecutionGroup.EXPLICITLY_THROWN_UNCHECKED_EXCEPTIONS} $commentPostfix"
- }
-
  /** Filter and copies executions with non-empty paths. */
  private fun prepareTestSetForByteCodeAnalysis(testSet: UtMethodTestSet): UtMethodTestSet {
  val executions =
@@ -314,6 +267,19 @@ class Summarization(val sourceFile: File?, val invokeDescriptions: List<InvokeDe
  )
  }
 
+ /** Filter and copies fuzzed executions. */
+ private fun prepareTestSetWithFuzzedExecutions(testSet: UtMethodTestSet): UtMethodTestSet {
+ val executions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
+
+ return UtMethodTestSet(
+ method = testSet.method,
+ executions = executions,
+ jimpleBody = testSet.jimpleBody,
+ errors = testSet.errors,
+ clustersInfo = testSet.clustersInfo
+ )
+ }
+
  /** Filter and copies executions with non-empty paths. */
  private fun prepareTestSetWithEmptyPaths(testSet: UtMethodTestSet): UtMethodTestSet {
  val executions =

utbot-summary/src/main/kotlin/org/utbot/summary/TagGenerator.kt

@@ -2,18 +2,20 @@ package org.utbot.summary
 
 import org.utbot.framework.plugin.api.Step
 import org.utbot.framework.plugin.api.UtConcreteExecutionFailure
-import org.utbot.framework.plugin.api.UtSymbolicExecution
+import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.framework.plugin.api.UtExecutionResult
 import org.utbot.framework.plugin.api.UtExecutionSuccess
 import org.utbot.framework.plugin.api.UtExplicitlyThrownException
 import org.utbot.framework.plugin.api.UtImplicitlyThrownException
-import org.utbot.framework.plugin.api.UtOverflowFailure
 import org.utbot.framework.plugin.api.UtMethodTestSet
+import org.utbot.framework.plugin.api.UtOverflowFailure
 import org.utbot.framework.plugin.api.UtSandboxFailure
 import org.utbot.framework.plugin.api.UtStreamConsumingFailure
+import org.utbot.framework.plugin.api.UtSymbolicExecution
 import org.utbot.framework.plugin.api.UtTimeoutException
 import org.utbot.framework.plugin.api.util.humanReadableName
 import org.utbot.framework.plugin.api.util.isCheckedException
+import org.utbot.fuzzer.UtFuzzedExecution
 import org.utbot.summary.UtSummarySettings.MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING
 import org.utbot.summary.clustering.MatrixUniqueness
 import org.utbot.summary.clustering.SplitSteps
@@ -29,7 +31,7 @@ class TagGenerator {
 
  if (clusteredExecutions.isNotEmpty()) {
  val listOfSplitSteps = clusteredExecutions.map {
- val mUniqueness = MatrixUniqueness(it.executions)
+ val mUniqueness = MatrixUniqueness(it.executions as List<UtSymbolicExecution>)
  mUniqueness.splitSteps()
  }
 
@@ -64,7 +66,7 @@ class TagGenerator {
  traceTagClusters.add(
  TraceTagCluster(
  cluster.header,
- generateExecutionTags(cluster.executions, splitSteps),
+ generateExecutionTags(cluster.executions as List<UtSymbolicExecution>, splitSteps),
  TraceTagWithoutExecution(
  commonStepsInCluster.toList(),
  cluster.executions.first().result,
@@ -88,51 +90,67 @@ private fun generateExecutionTags(executions: List<UtSymbolicExecution>, splitSt
 
 
 /**
- * Splits executions into clusters
- * By default there is 5 types of clusters:
- * Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
- * These are split by the type of execution result
+ * Splits executions with empty paths into clusters.
  *
- * @return clustered executions
+ * @return clustered executions.
  */
 fun groupExecutionsWithEmptyPaths(testSet: UtMethodTestSet): List<ExecutionCluster> {
  val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
  val clusters = mutableListOf<ExecutionCluster>()
- val commentPrefix =  "OTHER:"
+ val commentPrefix = "OTHER:"
  val commentPostfix = "for method ${testSet.method.humanReadableName}"
 
  val grouped = methodExecutions.groupBy { it.result.clusterKind() }
 
  val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
  if (successfulExecutions.isNotEmpty()) {
  clusters += SuccessfulExecutionCluster(
- "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
- successfulExecutions.toList())
+ "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
+ successfulExecutions.toList()
+ )
  }
 
- clusters += grouped
- .filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
- .map { (suffixId, group) ->
- FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
- }
+ clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
+ return clusters
+}
+
+/**
+ * Splits fuzzed executions into clusters.
+ *
+ * @return clustered executions.
+ */
+fun groupFuzzedExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
+ val methodExecutions = testSet.executions.filterIsInstance<UtFuzzedExecution>()
+ val clusters = mutableListOf<ExecutionCluster>()
+ val commentPrefix = "FUZZER:"
+ val commentPostfix = "for method ${testSet.method.humanReadableName}"
+
+ val grouped = methodExecutions.groupBy { it.result.clusterKind() }
+
+ val successfulExecutions = grouped[ExecutionGroup.SUCCESSFUL_EXECUTIONS] ?: emptyList()
+ if (successfulExecutions.isNotEmpty()) {
+ clusters += SuccessfulExecutionCluster(
+ "$commentPrefix ${ExecutionGroup.SUCCESSFUL_EXECUTIONS.displayName} $commentPostfix",
+ successfulExecutions.toList()
+ )
+ }
+
+ clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
  return clusters
 }
 
 /**
- * Splits executions produced by symbolic execution engine into clusters
- * By default there is 5 types of clusters:
- * Success, UnexpectedFail, ExpectedCheckedThrow, ExpectedUncheckedThrow, UnexpectedUncheckedThrow
- * These are split by the type of execution result
+ * Splits symbolic executions into clusters.
  *
- * If Success cluster has more than MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING execution
- * then clustering algorithm splits those into more clusters
+ * If Success cluster has more than [MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING] execution
+ * then clustering algorithm splits those into more clusters.
  *
- * @return clustered executions
+ * @return clustered executions.
  */
 private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster> {
  val methodExecutions = testSet.executions.filterIsInstance<UtSymbolicExecution>()
  val clusters = mutableListOf<ExecutionCluster>()
- val commentPrefix =  "SYMBOLIC EXECUTION:"
+ val commentPrefix = "SYMBOLIC EXECUTION:"
  val commentPostfix = "for method ${testSet.method.humanReadableName}"
 
  val grouped = methodExecutions.groupBy { it.result.clusterKind() }
@@ -161,11 +179,21 @@ private fun toClusterExecutions(testSet: UtMethodTestSet): List<ExecutionCluster
  }
  }
 
- clusters += grouped
+ clusters += addClustersOfFailedExecutions(grouped, commentPrefix, commentPostfix)
+ return clusters
+}
+
+private fun addClustersOfFailedExecutions(
+ grouped: Map<ExecutionGroup, List<UtExecution>>,
+ commentPrefix: String,
+ commentPostfix: String
+): List<FailedExecutionCluster> {
+ val clusters = grouped
  .filterNot { (kind, _) -> kind == ExecutionGroup.SUCCESSFUL_EXECUTIONS }
  .map { (suffixId, group) ->
- FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
- }
+ FailedExecutionCluster("$commentPrefix ${suffixId.displayName} $commentPostfix", group)
+ }
+
  return clusters
 }
 
@@ -197,18 +225,18 @@ private fun UtExecutionResult.clusterKind() = when (this) {
 /**
  * Structure used to represent execution cluster with header
  */
-sealed class ExecutionCluster(var header: String, val executions: List<UtSymbolicExecution>)
+sealed class ExecutionCluster(var header: String, val executions: List<UtExecution>)
 
 /**
  * Represents successful execution cluster
  */
-private class SuccessfulExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
+private class SuccessfulExecutionCluster(header: String, executions: List<UtExecution>) :
  ExecutionCluster(header, executions)
 
 /**
  * Represents failed execution cluster
  */
-private class FailedExecutionCluster(header: String, executions: List<UtSymbolicExecution>) :
+private class FailedExecutionCluster(header: String, executions: List<UtExecution>) :
  ExecutionCluster(header, executions)
 
 /**

utbot-summary/src/main/kotlin/org/utbot/summary/UtSummarySettings.kt

@@ -2,27 +2,27 @@ package org.utbot.summary
 
 object UtSummarySettings {
  /**
- * If True test comments will be generated
+ * If True test comments will be generated.
  */
  var GENERATE_COMMENTS = true
 
  /**
- * If True cluster comments will be generated
+ * If True cluster comments will be generated.
  */
  var GENERATE_CLUSTER_COMMENTS = true
 
  /**
- * If True names for tests will be generated
+ * If True names for tests will be generated.
  */
  var GENERATE_NAMES = true
 
  /**
- * If True display names for tests will be generated
+ * If True display names for tests will be generated.
  */
  var GENERATE_DISPLAY_NAMES = true
 
  /**
- * generate display name in from -> to style
+ * If True display name in from -> to style will be generated.
  */
  var GENERATE_DISPLAYNAME_FROM_TO_STYLE = true
 
@@ -34,19 +34,21 @@ object UtSummarySettings {
 
  /**
  * Sets minimum number of successful execution
- * for applying the clustering algorithm
+ * for applying the clustering algorithm.
  */
  const val MIN_NUMBER_OF_EXECUTIONS_FOR_CLUSTERING: Int = 4
 
  /**
- * DBSCAN hyperparameter
- * Sets minimum number of executions to form a cluster
+ * DBSCAN hyperparameter.
+ *
+ * Sets minimum number of executions to form a cluster.
  */
  var MIN_EXEC_DBSCAN: Int = 2
 
  /**
- * DBSCAN hyperparameter
- * Sets radius of search for algorithm
+ * DBSCAN hyperparameter.
+ *
+ * Sets radius of search for algorithm.
  */
  var RADIUS_DBSCAN: Float = 5.0f
 }