Merge pull request #32 from bakhtos/backend-pydriller

Use `PyDriller` as git backend

Author: SimonAdrian

README.md

@@ -32,13 +32,29 @@ The corresponding path only needs to be un-commented for analysis (all others ha
 
 ##### 2. Running the tool
 To start the tool via the terminal using the config file, simply enter `python3 code2DFD.py --config_path PATH_TO_CONFIG` in a command line opened in the root directory.
-For example, `python3 code2DFD.py --config_path config/config.ini` for the example config in this repository.
-The extraction will start and some status messages appear on the screen.
-If you want to analyse in application on GitHub, simply put in the GitHub handle, using the `--github_path` option.
-For example, for the repository `https://github.com/sqshq/piggymetrics`, run the command `python3 code2DFD.py --github_path sqshq/piggymetrics`
+For example, `python3 code2DFD.py --config_path config/config.ini` for the [example config](config/config.ini) in this repository.
 
+The config file needs to specify the following sections and parameters:
+- Repository
+ - `path`: `organization/repository` part of GitHub URL
+ - `url`: the full URL of the repository to clone from (may be local path)
+ - `local_path`: local directory to clone the repository to (without the repository name itself)
+- Technology profiles: same as in [example config](config/config.ini)
+- DFD: empty section
+- Analysis Settings (optional)
+ - `development_mode`: boolean, turns on development mode
+ - `commit`: hash of the commit to checkout and analyze; repository will be returned to the same commit it was in before analysis; if commit not provided, attempts to checkout `HEAD`
+ 
+It is possible to provide these parameters also by command line, see `python3 code2DFD.py --help` for exact usage
+
+If both config file and CLI arguments provided, CLI arguments take precedence
+
+###### 2.1 RESTful service
 To run the tool as a RESTful API service, run `python3 flask_code2DFD.py`.
-This will spawn up a Flask server and you can trigger DFD-extractions by sending a request to or opening your browser at `localhost:5000/dfd?path=*repository/path*`
+
+This will spawn up a Flask server and you can trigger DFD-extractions by sending a request to `localhost:5001/dfd` with parameters `url` and optionally `commit`.
+
+Currently only GitHub URLs are supported this way.
 
 
 ##### 3. Output

code2DFD.py

@@ -2,15 +2,14 @@
 #
 # Author: Simon Schneider, 2023
 # Contact: simon.schneider@tuhh.de
-
+import os
 from configparser import ConfigParser
 from datetime import datetime
 import argparse
 
-import core.dfd_extraction as dfd_extraction
+from core.dfd_extraction import perform_analysis
 from output_generators.logger import logger
 import tmp.tmp as tmp
-from core.file_interaction import get_output_path, get_local_path, clone_repo
 
 CONFIG_SECTIONS = ["Analysis Settings", "Repository", "Technology Profiles", "DFD"]
 COMMUNICATIONS_TECH_LIST = '[("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"),\
@@ -24,32 +23,34 @@
 DEFAULT_CONFIG.set("Technology Profiles", "communication_techs_list", COMMUNICATIONS_TECH_LIST)
 
 
-def api_invocation(path: str) -> dict:
+def api_invocation(url: str, commit: str) -> dict:
  """Entry function for when tool is called via API call.
  """
 
- print("New call for " + path)
- response = dict()
+ print("New call for " + url)
 
  start_time = datetime.now()
 
  logger.info("*** New execution ***")
- logger.debug("Copying config file to tmp file")
+ logger.debug("Initializing config to tmp file")
+ for section in CONFIG_SECTIONS: # Copying what is needed from default to temp
+ tmp.tmp_config.add_section(section)
+ for entry in DEFAULT_CONFIG[section]:
+ tmp.tmp_config.set(section, entry, DEFAULT_CONFIG[section][entry])
 
  # Overwrite repo_path from config file with the one from the API call
- repo_path = str(path)
- tmp.tmp_config.set("Repository", "path", repo_path)
-
- local_path = get_local_path(repo_path)
- tmp.tmp_config.set("Repository", "local_path", local_path)
-
- clone_repo(repo_path, local_path)
+ tmp.tmp_config.set("Repository", "url", url)
+ tmp.tmp_config.set("Repository", "local_path",
+ os.path.join(os.getcwd(), "analysed_repositories"))
+ if commit is not None:
+ tmp.tmp_config.set("Analysis Settings", "commit", commit)
 
  # Call extraction
- codeable_models, traceability = dfd_extraction.perform_analysis()
+ codeable_models, traceability = perform_analysis()
 
+ response = dict()
  response["codeable_models_file"] = codeable_models
- response["traceability"] = traceability
+ response["traceability_file"] = traceability
 
  # Execution time
  end_time = datetime.now()
@@ -61,47 +62,54 @@ def api_invocation(path: str) -> dict:
  return response
 
 
-def main():
+def cli_invocation():
  parser = argparse.ArgumentParser()
- source = parser.add_mutually_exclusive_group(required=True)
- source.add_argument("--config_path", type=str, help="Path to the config file to use")
- source.add_argument("--github_path", type=str, help="Path to the repository on GitHub as 'repository/path'")
- now = datetime.now()
- start_time = now.strftime("%H:%M:%S")
+ parser.add_argument("--config_path", type=str, help="Path to the config file to use (can be replaced with following CLI arguments")
+ repository = parser.add_argument_group("Repository", "Repository information")
+ repository.add_argument("--repo_url", type=str, help="URL to clone the repository from (might be local path)")
+ repository.add_argument("--repo_local_path", type=str, help="Location to clone repository to (default: 'analysed_repositories' in CWD)")
+ settings = parser.add_argument_group("Analysis Settings", "Parameters for additional analysis settings")
+ settings.add_argument("--commit", type=str, help="Analyze repository at this commit")
+ settings.add_argument("--development_mode", action='store_true', help="Switch on development mode")
 
  args = parser.parse_args()
 
  logger.info("*** New execution ***")
- logger.debug("Copying config file to tmp file")
 
- if args.config_path is not None:
+ if args.config_path:
  # Copy config to tmp file
+ logger.debug("Copying config file to tmp file")
  tmp.tmp_config.read(args.config_path)
- repo_path = tmp.tmp_config.get("Repository", "path")
-
- elif args.github_path is not None:
+ else:
  # global ini_config
+ logger.debug("Initializing tmp file with default config")
  for section in CONFIG_SECTIONS: # Copying what is needed from default to temp
  tmp.tmp_config.add_section(section)
  for entry in DEFAULT_CONFIG[section]:
  tmp.tmp_config.set(section, entry, DEFAULT_CONFIG[section][entry])
- repo_path = args.github_path.strip()
- tmp.tmp_config.set("Repository", "path", repo_path) # overwrite with user-provided path
 
- local_path = get_local_path(repo_path)
- clone_repo(repo_path, local_path)
- tmp.tmp_config.set("Repository", "local_path", local_path)
- tmp.tmp_config.set("Analysis Settings", "output_path", get_output_path(repo_path))
+ if args.repo_url:
+ tmp.tmp_config.set("Repository", "url", args.repo_url)
+ elif not tmp.tmp_config.has_option("Repository", "url"):
+ raise AttributeError("Parameter [Repository][url] must be provided either in config file or by --repo_url")
+
+ if args.repo_local_path:
+ tmp.tmp_config.set("Repository", "local_path", args.local_path)
+ elif not tmp.tmp_config.has_option("Repository", "local_path"):
+ tmp.tmp_config.set("Repository", "local_path", os.path.join(os.getcwd(), "analysed_repositories"))
 
- # calling the actual extraction
- dfd_extraction.perform_analysis()
+ if args.development_mode:
+  tmp.tmp_config.set("Analysis Settings", "development_mode", "True")
 
- now = datetime.now()
- end_time = now.strftime("%H:%M:%S")
+ if args.commit is not None:
+ commit = args.commit[:7]
+ tmp.tmp_config.set("Analysis Settings", "commit", commit)
+ elif tmp.tmp_config.has_option("Analysis Settings", "commit"):
+ commit = tmp.tmp_config.get("Analysis Settings", "commit")[:7]
+ tmp.tmp_config.set("Analysis Settings", "commit", commit)
 
- print("\nStarted", start_time)
- print("Finished", end_time)
+ perform_analysis()
 
 
 if __name__ == '__main__':
- main()
+ cli_invocation()

config/config.ini

@@ -1,27 +1,28 @@
 [Repository]
-path = apssouza22/java-microservice
-;path = callistaenterprise/blog-microservices
-;path = fernandoabcampos/spring-netflix-oss-microservices
-;path = georgwittberger/apache-spring-boot-microservice-example
-;path = mudigal-technologies/microservices-sample
-;path = spring-petclinic/spring-petclinic-microservices
-;path = sqshq/piggymetrics
-
-;path = anilallewar/microservices-basics-spring-boot
-;path = ewolff/microservice
-;path = ewolff/microservice-kafka
-;path = jferrater/tap-and-eat-microservices
-;path = koushikkothagal/spring-boot-microservices-workshop
-;path = mdeket/spring-cloud-movie-recommendation
-;path = blaugmail/clone-sample-spring-oauth2-microservices
-;path = shabbirdwd53/springboot-microservice
-;path = rohitghatol/spring-boot-microservices
-;path = yidongnan/spring-cloud-netflix-example
+url = https://github.com/apssouza22/java-microservice
+;url = https://github.com/callistaenterprise/blog-microservices
+;url = https://github.com/fernandoabcampos/spring-netflix-oss-microservices
+;url = https://github.com/georgwittberger/apache-spring-boot-microservice-example
+;url = https://github.com/mudigal-technologies/microservices-sample
+;url = https://github.com/spring-petclinic/spring-petclinic-microservices
+;url = https://github.com/sqshq/piggymetrics
+;url = https://github.com/anilallewar/microservices-basics-spring-boot
+;url = https://github.com/ewolff/microservice
+;url = https://github.com/ewolff/microservice-kafka
+;url = https://github.com/jferrater/tap-and-eat-microservices
+;url = https://github.com/koushikkothagal/spring-boot-microservices-workshop
+;url = https://github.com/mdeket/spring-cloud-movie-recommendation
+;url = https://github.com/blaugmail/clone-sample-spring-oauth2-microservices
+;url = https://github.com/shabbirdwd53/springboot-microservice
+;url = https://github.com/rohitghatol/spring-boot-microservices
+;url = https://github.com/yidongnan/spring-cloud-netflix-example
 
 [Technology Profiles]
 communication_techs_list = [("RabbitMQ", "rmq"), ("Kafka", "kfk"), ("RestTemplate", "rst"), ("FeignClient", "fgn"), ("Implicit Connections", "imp"), ("Database Connections", "dbc"), ("HTML", "html"), ("Docker-Compose", "dcm")]
 
 [Analysis Settings]
 development_mode = False
+; commit for apssouza22/java-microservice
+;commit = 056414c4c938e536f467a3f37532194b860d96a3
 
 [DFD]

core/dfd_extraction.py

@@ -1,6 +1,9 @@
 import ast
 from datetime import datetime
 from itertools import combinations
+import os
+
+from pydriller import Repository
 
 import output_generators.codeable_model as codeable_model
 import core.technology_switch as tech_sw
@@ -49,14 +52,37 @@
 
 
 def perform_analysis():
- """Main function for the extraction, calling all technology-specific extractors, managing output etc.
  """
+ Entrypoint for the DFD extraction that initializes the repository
+ """
+ local_path = tmp.tmp_config.get("Repository", "local_path")
+ url_path = tmp.tmp_config.get("Repository", "url")
+
+ os.makedirs(local_path, exist_ok=True)
+ repository = Repository(path_to_repo=url_path, clone_repo_to=local_path)
+ with repository._prep_repo(url_path) as git_repo:
+ tmp.tmp_config.set("Repository", "local_path", str(git_repo.path))
+ head = git_repo.get_head().hash[:7]
+ if tmp.tmp_config.has_option("Analysis Settings", "commit"):
+ commit = tmp.tmp_config.get("Analysis Settings", "commit")
+ else:
+ commit = head
+ repo_name = git_repo.project_name
+ tmp.tmp_config.set("Analysis Settings", "output_path", os.path.join(os.getcwd(), "code2DFD_output", repo_name.replace("/", "--"), commit))
+ git_repo.checkout(commit)
+ print(f"\nStart extraction of DFD for {repo_name} on commit {commit} at {datetime.now().strftime("%H:%M:%S")}")
+ codeable_models, traceability_content = DFD_extraction()
+ print(f"Finished: {datetime.now().strftime("%H:%M:%S")}")
+
+ git_repo.checkout(head)
+
+ return codeable_models, traceability_content
+
 
+def DFD_extraction():
+ """Main function for the extraction, calling all technology-specific extractors, managing output etc.
+ """
  dfd = CDFD("TestDFD")
- repo_path = tmp.tmp_config["Repository"]["path"]
- now = datetime.now()
- start_time = now.strftime("%H:%M:%S")
- print("\n\tStart extraction of DFD for " + repo_path + " at " + str(start_time))
 
  microservices, information_flows, external_components = dict(), dict(), dict()