Updates from sbft version

utbot-cli-python/src/README.md

@@ -6,25 +6,25 @@
 
  - Required Java version: 11.
 
- - Prefered Python version: 3.8+.
+ - Preferred Python version: 3.11+.
 
  Make sure that your Python has `pip` installed (this is usually the case). [Read more about pip installation](https://pip.pypa.io/en/stable/installation/).
 
  Before running utbot install pip requirements (or use `--install-requirements` flag in `generate_python` command):
 
- python -m pip install mypy==1.0 utbot_executor==0.4.31 utbot_mypy_runner==0.2.8
+ python -m pip install mypy==1.0 utbot_executor==0.9.16 utbot_mypy_runner==0.2.16
 
 ## Basic usage
 
 Generate tests:
 
- java -jar utbot-cli.jar generate_python dir/file_with_sources.py -p <PYTHON_PATH> -o generated_tests.py -s dir
+ java -jar utbot-cli-python.jar generate_python dir/file_with_sources.py -p <PYTHON_PATH> -o generated_tests.py -s dir
 
 This will generate tests for top-level functions from `file_with_sources.py`.
 
 Run generated tests:
 
- java -jar utbot-cli.jar run_python generated_tests.py -p <PYTHON_PATH>
+ java -jar utbot-cli-python.jar run_python generated_tests.py -p <PYTHON_PATH>
 
 ### `generate_python` options
 

utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/PythonCliProcessor.kt

@@ -1,20 +1,22 @@
 package org.utbot.cli.language.python
 
-import mu.KLogger
+import mu.KotlinLogging
 import org.utbot.python.PythonTestGenerationConfig
 import org.utbot.python.PythonTestGenerationProcessor
 import org.utbot.python.PythonTestSet
 
+private val logger = KotlinLogging.logger {}
+
 class PythonCliProcessor(
  override val configuration: PythonTestGenerationConfig,
- private val output: String,
- private val logger: KLogger,
+ private val testWriter: TestWriter,
  private val coverageOutput: String?,
  private val executionCounterOutput: String?,
 ) : PythonTestGenerationProcessor() {
 
  override fun saveTests(testsCode: String) {
- writeToFileAndSave(output, testsCode)
+ testWriter.addTestCode(testsCode)
+// writeToFileAndSave(output, testsCode)
  }
 
  override fun notGeneratedTestsAction(testedFunctions: List<String>) {

utbot-cli-python/src/main/kotlin/org/utbot/cli/language/python/TestWriter.kt

@@ -0,0 +1,27 @@
+package org.utbot.cli.language.python
+
+class TestWriter {
+ private val testCode: MutableList<String> = mutableListOf()
+
+ fun addTestCode(code: String) {
+ testCode.add(code)
+ }
+
+ fun generateTestCode(): String {
+ val (importLines, code) = testCode.fold(mutableListOf<String>() to StringBuilder()) { acc, s ->
+ val lines = s.split(System.lineSeparator())
+ val firstClassIndex = lines.indexOfFirst { it.startsWith("class") }
+ lines.take(firstClassIndex).forEach { line -> if (line !in acc.first) acc.first.add(line) }
+ lines.drop(firstClassIndex).forEach { line -> acc.second.append(line + System.lineSeparator()) }
+ acc.first to acc.second
+ }
+ val codeBuilder = StringBuilder()
+ importLines.filter { it.isNotEmpty() }.forEach {
+ codeBuilder.append(it)
+ codeBuilder.append(System.lineSeparator())
+ }
+ codeBuilder.append(System.lineSeparator())
+ codeBuilder.append(code)
+ return codeBuilder.toString()
+ }
+}

utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/PythonDialogProcessor.kt

@@ -72,18 +72,18 @@ object PythonDialogProcessor {
  private fun runIndicatorWithTimeHandler(indicator: ProgressIndicator, range: ProgressRange, text: String, globalCount: Int, globalShift: Int, timeout: Long): ScheduledFuture<*> {
  val startTime = System.currentTimeMillis()
  return AppExecutorUtil.getAppScheduledExecutorService().scheduleWithFixedDelay({
-  val innerTimeoutRatio =
-  ((System.currentTimeMillis() - startTime).toDouble() / timeout)
-  .coerceIn(0.0, 1.0)
-  updateIndicator(
-  indicator,
-  range,
-  text,
-  innerTimeoutRatio,
-  globalCount,
-  globalShift,
-  )
-  }, 0, 100, TimeUnit.MILLISECONDS)
+ val innerTimeoutRatio =
+ ((System.currentTimeMillis() - startTime).toDouble() / timeout)
+ .coerceIn(0.0, 1.0)
+ updateIndicator(
+ indicator,
+ range,
+ text,
+ innerTimeoutRatio,
+ globalCount,
+ globalShift,
+ )
+ }, 0, 100, TimeUnit.MILLISECONDS)
  }
 
  private fun updateIndicatorTemplate(
@@ -163,28 +163,28 @@ object PythonDialogProcessor {
 
  private fun findSelectedPythonMethods(model: PythonTestLocalModel): List<PythonMethodHeader> {
  return ReadAction.nonBlocking<List<PythonMethodHeader>> {
- model.selectedElements
- .filter { model.selectedElements.contains(it) }
- .flatMap {
- when (it) {
- is PyFunction -> listOf(it)
- is PyClass -> it.methods.toList()
- else -> emptyList()
- }
- }
- .filter { fineFunction(it) }
- .mapNotNull {
- val functionName = it.name ?: return@mapNotNull null
- val moduleFilename = it.containingFile.virtualFile?.canonicalPath ?: ""
- val containingClassId = it.containingClass?.qualifiedName?.let{ cls -> PythonClassId(cls) }
- PythonMethodHeader(
- functionName,
- moduleFilename,
- containingClassId,
- )
+ model.selectedElements
+ .filter { model.selectedElements.contains(it) }
+ .flatMap {
+ when (it) {
+ is PyFunction -> listOf(it)
+ is PyClass -> it.methods.toList()
+ else -> emptyList()
  }
- .toSet()
- .toList()
+ }
+ .filter { fineFunction(it) }
+ .mapNotNull {
+ val functionName = it.name ?: return@mapNotNull null
+ val moduleFilename = it.containingFile.virtualFile?.canonicalPath ?: ""
+ val containingClassId = it.containingClass?.qualifiedName?.let{ cls -> PythonClassId(cls) }
+ PythonMethodHeader(
+ functionName,
+ moduleFilename,
+ containingClassId,
+ )
+ }
+ .toSet()
+ .toList()
  }.executeSynchronously() ?: emptyList()
  }
 
@@ -287,7 +287,7 @@ object PythonDialogProcessor {
 
  localUpdateIndicator(ProgressRange.ANALYZE, "Analyze module ${model.currentPythonModule}", 0.5)
 
- val (mypyStorage, _) = processor.sourceCodeAnalyze()
+ val mypyConfig = processor.sourceCodeAnalyze()
 
  localUpdateIndicator(ProgressRange.ANALYZE, "Analyze module ${model.currentPythonModule}", 1.0)
 
@@ -300,7 +300,7 @@ object PythonDialogProcessor {
  model.timeout,
  )
  try {
- val testSets = processor.testGenerate(mypyStorage)
+ val testSets = processor.testGenerate(mypyConfig)
  timerHandler.cancel(true)
  if (testSets.isEmpty()) return@forEachIndexed
 
@@ -312,7 +312,7 @@ object PythonDialogProcessor {
 
  logger.info(
  "Finished test generation for the following functions: ${
- testSets.joinToString { it.method.name }
+ testSets.map { it.method.name }.toSet().joinToString()
  }"
  )
  } finally {

utbot-intellij-python/src/main/kotlin/org/utbot/intellij/plugin/python/Utils.kt

@@ -13,7 +13,6 @@ import org.utbot.intellij.plugin.python.table.UtPyClassItem
 import org.utbot.intellij.plugin.python.table.UtPyFunctionItem
 import org.utbot.intellij.plugin.python.table.UtPyTableItem
 import org.utbot.python.utils.RequirementsUtils
-import kotlin.random.Random
 
 inline fun <reified T : PsiElement> getContainingElement(
  element: PsiElement,
@@ -37,13 +36,6 @@ fun getContentRoot(project: Project, file: VirtualFile): VirtualFile {
  .getContentRootForFile(file) ?: error("Source file lies outside of a module")
 }
 
-fun generateRandomString(length: Int): String {
- val charPool = ('a'..'z') + ('A'..'Z') + ('0'..'9')
- return (0..length)
- .map { Random.nextInt(0, charPool.size).let { charPool[it] } }
- .joinToString("")
-}
-
 fun VirtualFile.isProjectSubmodule(ancestor: VirtualFile?): Boolean {
  return VfsUtil.isUnder(this, setOf(ancestor).toMutableSet())
 }
@@ -52,10 +44,12 @@ fun checkModuleIsInstalled(pythonPath: String, moduleName: String): Boolean {
  return RequirementsUtils.requirementsAreInstalled(pythonPath, listOf(moduleName))
 }
 
-fun fineFunction(function: PyFunction): Boolean =
- !listOf("__init__", "__new__").contains(function.name) &&
- function.decoratorList?.decorators?.isNotEmpty() != true // TODO: add processing of simple decorators
- //(function.parent !is PyDecorator || (function.parent as PyDecorator).isBuiltin)
+fun fineFunction(function: PyFunction): Boolean {
+ val hasNotConstructorName = !listOf("__init__", "__new__").contains(function.name)
+ val decoratorNames = function.decoratorList?.decorators?.mapNotNull { it?.qualifiedName }
+ val knownDecorators = decoratorNames?.all { it.toString() in listOf("staticmethod") } ?: true
+ return hasNotConstructorName && knownDecorators
+}
 
 fun fineClass(pyClass: PyClass): Boolean =
  getAncestors(pyClass).dropLast(1).all { it !is PyClass && it !is PyFunction } &&

utbot-python-executor/.gitignore

@@ -23,4 +23,5 @@ env/
 venv/
 .mypy_cache/
 .dmypy.json
-dmypy.json
+dmypy.json
+utbot_executor.iml

utbot-python-executor/src/main/python/utbot_executor/README.md

@@ -29,6 +29,7 @@ $ python -m utbot_executor <hostname> <port> <logfile> [<loglevel DEBUG | INFO |
  "argumentsIds": ["1", "2"],
  "kwargumentsIds": ["4", "5"],
  "serializedMemory": "string",
+ "memoryMode": "REDUCE",
  "filepath": ["/home/user/my_project/my_module/submod1.py"],
  "coverageId": "1"
 }
@@ -41,6 +42,7 @@ $ python -m utbot_executor <hostname> <port> <logfile> [<loglevel DEBUG | INFO |
 * `argumentsIds` - list of argument's ids
 * `kwargumentsIds` - list of keyword argument's ids
 * `serializedMemory` - serialized memory throw `deep_serialization` algorithm
+* `memoryMode` - serialization mode (`PICKLE`, `REDUCE`)
 * `filepath` - path to the tested function's containing file
 * `coverageId` - special id witch will be used for sending information about covered lines
 

utbot-python-executor/src/main/python/utbot_executor/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "utbot-executor"
-version = "1.8.6"
+version = "1.9.19"
 description = ""
 authors = ["Vyacheslav Tamarin <vyacheslav.tamarin@yandex.ru>"]
 readme = "README.md"
@@ -19,3 +19,8 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry.scripts]
 utbot-executor = "utbot_executor:utbot_executor"
 
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "DEBUG"
+log_cli_format = "%(asctime)s [%(levelname)6s] (%(filename)s:%(lineno)s) %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"

utbot-python-executor/src/main/python/utbot_executor/tests/example/example.py

@@ -0,0 +1,45 @@
+import random
+
+from utbot_executor.config import CoverageConfig, HostConfig
+from utbot_executor.executor import PythonExecutor
+from utbot_executor.parser import ExecutionRequest, ExecutionSuccessResponse, MemoryMode
+from utbot_executor.utils import TraceMode
+
+
+def test_execution():
+ executor = PythonExecutor(
+ CoverageConfig(HostConfig("localhost", random.randint(10 ** 5, 10 ** 6)), TraceMode.Instructions, True), False)
+ id_ = '1500926645'
+ serialized_arg = (r'{"objects":{"1500926644":{"strategy":"repr","id":"1500926644","typeinfo":{"module":"builtins",'
+ r'"kind":"int"},"comparable":true,"value":"170141183460469231731687303715749887999"},'
+ r'"1500926652":{"strategy":"list","id":"1500926652","typeinfo":{"module":"builtins",'
+ r'"kind":"list"},"comparable":true,"items":["1500926644"]},"1500926650":{"strategy":"repr",'
+ r'"id":"1500926650","typeinfo":{"module":"builtins","kind":"str"},"comparable":true,'
+ r'"value":"\"x\""},"1500926646":{"strategy":"repr","id":"1500926646","typeinfo":{'
+ r'"module":"builtins","kind":"int"},"comparable":true,"value":"1"},"1500926651":{'
+ r'"strategy":"dict","id":"1500926651","typeinfo":{"module":"builtins","kind":"dict"},'
+ r'"comparable":true,"items":{"1500926650":"1500926646"}},"1500926653":{"strategy":"list",'
+ r'"id":"1500926653","typeinfo":{"module":"builtins","kind":"list"},"comparable":true,'
+ r'"items":[]},"1500926654":{"strategy":"dict","id":"1500926654","typeinfo":{'
+ r'"module":"builtins","kind":"dict"},"comparable":true,"items":{}},"1500926645":{'
+ r'"strategy":"reduce","id":"1500926645","typeinfo":{"module":"my_func","kind":"A"},'
+ r'"comparable":true,"constructor":{"module":"my_func","kind":"A"},"args":"1500926652",'
+ r'"state":"1500926651","listitems":"1500926653","dictitems":"1500926654"}}}')
+ request = ExecutionRequest(
+ 'f',
+ 'my_func',
+ ['my_func'],
+ ['./'],
+ [id_],
+ {},
+ serialized_arg,
+ MemoryMode.REDUCE,
+ 'my_func.py',
+ '0x1',
+ )
+ response = executor.run_reduce_function(request)
+
+ assert isinstance(response, ExecutionSuccessResponse)
+
+ assert response.status == "success"
+ assert response.is_exception is False

utbot-python-executor/src/main/python/utbot_executor/tests/example_input.json

@@ -0,0 +1 @@
+{"functionName":"im_list","functionModule":"src.foo.foo","imports":["typing","builtins","src.foo.foo"],"syspaths":["/home/vyacheslav/PycharmProjects/pythonProject/src","/home/vyacheslav/PycharmProjects/pythonProject"],"argumentsIds":["1500000001"],"kwargumentsIds":{},"serializedMemory":"{\"objects\":{\"1500000002\":{\"strategy\":\"repr\",\"id\":\"1500000002\",\"typeinfo\":{\"module\":\"builtins\",\"kind\":\"int\"},\"comparable\":true,\"value\":\"9\"},\"1500000003\":{\"strategy\":\"repr\",\"id\":\"1500000003\",\"typeinfo\":{\"module\":\"builtins\",\"kind\":\"int\"},\"comparable\":true,\"value\":\"1\"},\"1500000004\":{\"strategy\":\"repr\",\"id\":\"1500000004\",\"typeinfo\":{\"module\":\"builtins\",\"kind\":\"int\"},\"comparable\":true,\"value\":\"0\"},\"1500000001\":{\"strategy\":\"iterator\",\"id\":\"1500000001\",\"typeinfo\":{\"module\":\"typing\",\"kind\":\"Iterator\"},\"comparable\":true,\"items\":[\"1500000002\",\"1500000003\",\"1500000004\"],\"exception\":{\"module\":\"builtins\",\"kind\":\"StopIteration\"}}}}","memoryMode":"REDUCE","filepath":"/home/vyacheslav/PycharmProjects/pythonProject/src/foo/foo.py","coverageId":"59682f01"}