Massively performance of gathering coverage thanks to improved multi threading.

Previously both the GCC and LLVM generators simply partitioned the amount of files and functions into equally sized batches and then processed them. This lead to harsh thread under utilization however as not every file or function is equally sized. Instead, we now use a ConcurrentLinkedQueue which is a lockless FIFO datastructures which contains all functions that should be processed. Every thread simply pops off functions from the front of the queue for processing until the queue is empty. This leads to practically 100% thread utilization

Author: zero9178

src/main/kotlin/net/zero9178/cov/data/GCCJSONCoverageGenerator.kt

@@ -36,7 +36,7 @@ import java.nio.file.LinkOption
 import java.nio.file.Paths
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.CompletionException
-import kotlin.math.ceil
+import java.util.concurrent.ConcurrentLinkedQueue
 
 class GCCJSONCoverageGenerator(private val myGcov: String, private val myMajorVersion: Int) : CoverageGenerator {
 
@@ -391,21 +391,56 @@ class GCCJSONCoverageGenerator(private val myGcov: String, private val myMajorVe
 
  val files = try {
  if (!CoverageGeneratorSettings.getInstance().branchCoverageEnabled) {
- processFiles(filesToProcess, env, project)
+ filesToProcess.mapNotNull {
+ processFile(it, env, project)
+ }
  } else {
- DumbService.getInstance(project).runReadActionInSmartMode<List<CoverageFileData>> {
- filesToProcess.chunked(ceil(filesToProcess.size / Thread.activeCount().toDouble()).toInt()).map {
+
+ val lineAssoc = System.nanoTime()
+ val functionsToProcess = filesToProcess.flatMap { file ->
+ val linesOfFunction = file.lines.groupBy { it.functionName }
+ val list = file.functions.map {
+ Triple(file, it, linesOfFunction[it.name] ?: emptyList())
+ }
+ list
+ }.sortedByDescending {
+ it.third.size
+ }
+ log.info("Line assoc took ${System.nanoTime() - lineAssoc} ns")
+
+ val parallelGen = System.nanoTime()
+ val queue = ConcurrentLinkedQueue(functionsToProcess)
+ val result = DumbService.getInstance(project).runReadActionInSmartMode<List<CoverageFileData>> {
+ (0 until Thread.activeCount()).map {
  CompletableFuture.supplyAsync {
  runReadAction {
- var list = emptyList<CoverageFileData>()
+ val map = mutableListOf<Pair<File, CoverageFunctionData>>()
  ProgressManager.getInstance().executeProcessUnderProgress({
- list = processFiles(it, env, project)
+ generateSequence { queue.poll() }.forEach {
+ map.add(
+ it.first
+ to
+ processFunction(
+ it.first,
+ it.second,
+ it.third,
+ env,
+ project
+ )
+ )
+ }
  }, indicator)
- list
+ map
  }
  }
- }.flatMap { it.join() }
+ }.flatMap { it.get() }.groupBy { it.first }.map { entry ->
+ CoverageFileData(
+ env.toLocalPath(entry.key.file).replace('\\', '/'),
+ entry.value.map { it.second }.associateBy { it.functionName })
+ }
  }
+ log.info("Parallel branch gen took ${System.nanoTime() - parallelGen} ns")
+ result
  }.associateBy { it.filePath }
  } catch (e: CompletionException) {
  val cause = e.cause
@@ -424,29 +459,39 @@ class GCCJSONCoverageGenerator(private val myGcov: String, private val myMajorVe
  )
  }
 
- private fun processFiles(
- files: List<File>,
+ private fun processFunction(
+ file: File,
+ function: Function,
+ lines: List<Line>,
  env: CPPEnvironment,
  project: Project
- ) = files.filter { it.lines.isNotEmpty() || it.functions.isNotEmpty() }.map { file ->
+ ) = CoverageFunctionData(
+ function.startLine.toInt() toCP 0,
+ function.endLine.toInt() toCP 0,
+ function.demangledName,
+ FunctionLineData(lines.associate { it.lineNumber.toInt() to it.count.toLong() }),
+ if (CoverageGeneratorSettings.getInstance().branchCoverageEnabled)
+ findStatementsForBranches(
+ lines,
+ env.toLocalPath(file.file),
+ project
+ ) else emptyList()
+ )
+
+ private fun processFile(
+ file: File,
+ env: CPPEnvironment,
+ project: Project
+ ): CoverageFileData? {
+ if (file.lines.isEmpty() && file.functions.isEmpty()) {
+ return null
+ }
  val linesOfFunction = file.lines.groupBy { it.functionName }
  val functions = file.functions.map { function ->
- ProgressManager.checkCanceled()
  val lines = linesOfFunction[function.name] ?: emptyList()
- CoverageFunctionData(
- function.startLine.toInt() toCP 0,
- function.endLine.toInt() toCP 0,
- function.demangledName,
- FunctionLineData(lines.associate { it.lineNumber.toInt() to it.count.toLong() }),
- if (CoverageGeneratorSettings.getInstance().branchCoverageEnabled)
- findStatementsForBranches(
- lines,
- env.toLocalPath(file.file),
- project
- ) else emptyList()
- )
+ processFunction(file, function, lines, env, project)
  }.associateBy { it.functionName }
- CoverageFileData(env.toLocalPath(file.file).replace('\\', '/'), functions)
+ return CoverageFileData(env.toLocalPath(file.file).replace('\\', '/'), functions)
  }
 
  private data class Root(

src/main/kotlin/net/zero9178/cov/data/LLVMCoverageGenerator.kt

@@ -47,7 +47,7 @@ import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.CompletionException
-import kotlin.math.ceil
+import java.util.concurrent.ConcurrentLinkedQueue
 
 private sealed class Component
 
@@ -327,70 +327,65 @@ class LLVMCoverageGenerator(
 
  val filesMap = datas.flatMap { data ->
  //Associates the filename with a list of all functions in that file
- val funcMap =
+ val functions =
  data.functions.flatMap { func ->
- func.filenames.map { it to func }
- }.groupBy({ it.first }) {
- it.second
+ func.filenames.filter {
+ val filePath = environment.toLocalPath(it).replace('\\', '/')
+ CoverageGeneratorSettings.getInstance().calculateExternalSources || sources.any {
+ it == filePath
+ }
+ }.map { it to func }
  }
 
- data.files.fold(emptyList<CoverageFileData>()) fileFold@{ result, file ->
-
-  val filePath = environment.toLocalPath(file).replace('\\', '/')
- if (!CoverageGeneratorSettings.getInstance().calculateExternalSources && !sources.any {
-   it == filePath
- }) {
- return@fileFold result
+ if (majorVersion >= 12 || !CoverageGeneratorSettings.getInstance()
+ .branchCoverageEnabled
+ ) {
+ functions.mapNotNull {
+ processFunction(environment, project, demangledNames, it.first, it.second)?.let { result ->
+  it.first to result
+ }
  }
-
- val llvmFunctions = funcMap.getOrDefault(
- file,
- emptyList()
- )
-
- val functionsMap = if (majorVersion >= 12 || !CoverageGeneratorSettings.getInstance()
- .branchCoverageEnabled
- ) {
- processFunctions(environment, project, demangledNames, file, llvmFunctions)
- } else {
- DumbService.getInstance(project).runReadActionInSmartMode<List<CoverageFunctionData>> {
- ProgressManager.checkCanceled()
- try {
- llvmFunctions.chunked(
- ceil(
- data.files.size / Thread.activeCount()
- .toDouble()
- ).toInt()
- ).map { functions ->
- CompletableFuture.supplyAsync {
- runReadAction {
- var list = emptyList<CoverageFunctionData>()
- ProgressManager.getInstance().executeProcessUnderProgress({
- list =
- processFunctions(environment, project, demangledNames, file, functions)
- }, indicator)
- list
- }
+ } else {
+ val queue = ConcurrentLinkedQueue(functions.sortedByDescending {
+ it.second.regions.size
+ })
+ DumbService.getInstance(project).runReadActionInSmartMode<List<Pair<String, CoverageFunctionData>>> {
+ ProgressManager.checkCanceled()
+ try {
+ (0 until Thread.activeCount()).map {
+ CompletableFuture.supplyAsync {
+ runReadAction {
+ val list = mutableListOf<Pair<String, CoverageFunctionData>>()
+ ProgressManager.getInstance().executeProcessUnderProgress({
+ generateSequence { queue.poll() }.forEach {
+ processFunction(
+ environment,
+ project,
+ demangledNames,
+ it.first,
+ it.second
+ )?.let { result ->
+ list.add(it.first to result)
+ }
+ }
+ }, indicator)
+ list
  }
- }.flatMap { it.join() }
- } catch (e: CompletionException) {
- val cause = e.cause
- if (cause != null) {
- throw cause
- } else {
- throw e
  }
+ }.flatMap { it.join() }
+ } catch (e: CompletionException) {
+ val cause = e.cause
+ if (cause != null) {
+ throw cause
+ } else {
+ throw e
  }
  }
- }.associateBy { it.functionName }
-
- ProgressManager.checkCanceled()
-
- result + CoverageFileData(
- filePath,
- functionsMap
- )
-
+ }
+ }.groupBy { it.first }.map { entry ->
+ CoverageFileData(
+ environment.toLocalPath(entry.key).replace('\\', '/'),
+ entry.value.map { it.second }.associateBy { it.functionName })
  }
  }.associateBy { it.filePath }
  log.info("Processing coverage data took ${System.nanoTime() - processDataStart}ns")
@@ -401,20 +396,20 @@ class LLVMCoverageGenerator(
  )
  }
 
- private fun processFunctions(
+ private fun processFunction(
  environment: CPPEnvironment,
  project: Project,
  demangledNames: Map<String, String>,
  file: String,
- functions: List<Function>
- ): List<CoverageFunctionData> = functions.fold(emptyList()) { result, function ->
+ function: Function
+ ): CoverageFunctionData? {
 
  var regions = function.regions.filter {
  function.filenames[it.fileId] == file
  }
 
  if (regions.isEmpty()) {
- return@fold result
+ return null
  }
 
  val functionRegions = regions.map { region ->
@@ -431,7 +426,7 @@ class LLVMCoverageGenerator(
  )
  }
 
- result + CoverageFunctionData(
+ return CoverageFunctionData(
  regions.first().start,
  regions.first().end,
  demangledNames[function.name] ?: function.name,