Upgrade dependencies. Update github actions workflows. Code updates

Author: Gal Ben David

.github/workflows/build.yml

@@ -1,16 +1,19 @@
 name: Build
-on: [push, pull_request]
+on:
+ - push
+ - pull_request
 jobs:
  lint:
  if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags')
  runs-on: ubuntu-latest
  steps:
  - name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
  - name: Install latest rust
  uses: actions-rs/toolchain@v1
  with:
  toolchain: stable
+ profile: minimal
  override: true
  components: clippy
  - name: Lint with clippy
@@ -24,17 +27,24 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10']
- os: [ubuntu-latest , macos-latest, windows-latest]
+ python-version:
+ - '3.7'
+ - '3.8'
+ - '3.9'
+ - '3.10'
+ os:
+ - ubuntu-latest
+ - macos-latest
+ - windows-latest
  steps:
  - name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
  - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v3
  with:
  python-version: ${{ matrix.python-version }}
- - name: Run image
- uses: abatilo/actions-poetry@v2.0.0
+ - name: Install Poetry
+ uses: abatilo/actions-poetry@v2.1.3
  - name: Install Rust
  uses: actions-rs/toolchain@v1
  with:

.github/workflows/deploy.yml

@@ -1,20 +1,28 @@
 name: Deploy
 on:
  release:
- types: [released]
+ types:
+ - released
 jobs:
  deploy:
  runs-on: ${{ matrix.os }}
  strategy:
  fail-fast: false
  matrix:
- python-version: ['3.7', '3.8', '3.9', '3.10']
- os: [ubuntu-latest, macos-latest, windows-latest]
+ python-version:
+ - '3.7'
+ - '3.8'
+ - '3.9'
+ - '3.10'
+ os:
+ - ubuntu-latest
+ - macos-latest
+ - windows-latest
  steps:
  - name: Checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
  - name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v3
  with:
  python-version: ${{ matrix.python-version }}
  - name: Install Rust
@@ -24,22 +32,9 @@ jobs:
  toolchain: stable
  override: true
  - name: Publish Package
- if: matrix.os != 'windows-latest'
  uses: messense/maturin-action@v1
  with:
- maturin-version: latest
  command: publish
- manylinux: 2014
- args: --username=__token__ --no-sdist --interpreter=python${{ matrix.python-version }}
- env:
- MATURIN_PASSWORD: ${{ secrets.pypi_password }}
- - name: Publish Package
- if: matrix.os == 'windows-latest'
- uses: messense/maturin-action@v1
- with:
- maturin-version: latest
- command: publish
- manylinux: 2014
- args: --username=__token__ --no-sdist --interpreter=python
+ args: --username=__token__ --no-sdist --interpreter=python${{ !startsWith(matrix.os, 'windows') && matrix.python-version || '' }}
  env:
  MATURIN_PASSWORD: ${{ secrets.pypi_password }}

.gitignore

@@ -1,3 +1,8 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,rust,python
+# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,rust,python
+
+### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -94,6 +99,13 @@ ipython_config.py
 # install all needed dependencies.
 #Pipfile.lock
 
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
@@ -136,17 +148,48 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
-.gitignore
-.gitignore
 
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Rust ###
 # Generated by Cargo
 # will have compiled files and executables
 debug/
-target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
 Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# Support for Project snippet scope
+
+# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,rust,python

Cargo.toml

@@ -35,23 +35,23 @@ name = "pyrepscan"
 crate-type = ["cdylib"]
 
 [dependencies]
-regex = "1"
-chrono = "0.4"
-num_cpus = "1"
-parking_lot = "0.12"
-crossbeam = "0.8"
-crossbeam-utils = "0.8"
+regex = "1.6.0"
+chrono = "0.4.19"
+num_cpus = "1.13.1"
+parking_lot = "0.12.1"
+crossbeam = "0.8.1"
+crossbeam-utils = "0.8.10"
 
 [dependencies.libgit2-sys]
-version = "0.12"
+version = "0.13.4"
 features = ["https"]
 
 [dependencies.git2]
-version = "0.13"
+version = "0.14.4"
 features = ["vendored-openssl"]
 
 [dependencies.pyo3]
-version = "0.15.1"
+version = "0.16.5"
 features = ["extension-module"]
 
 [profile.release]

benchmarks/gitleaks.sh

@@ -1 +1,2 @@
-~/go/bin/gitleaks --repo-path=$REPO_PATH --config=gitleaks.toml
+docker pull zricethezav/gitleaks:latest
+docker run -v ${FOLDER_TO_SCAN}:/path -v ${PWD}/benchmarks/gitleaks.toml:/gitleaks.toml zricethezav/gitleaks:latest detect --source="/path" --config=/gitleaks.toml

benchmarks/pyrepscan_bench.py

@@ -12,3 +12,4 @@
  repository_path='/path/to/repository',
  branch_glob_pattern='*',
 )
+print(len(results))

pyproject.toml

@@ -1,13 +1,14 @@
 [build-system]
-requires = ["maturin>=0.11,<0.12"]
+requires = ["maturin>=0.12,<0.13"]
 build-backend = "maturin"
 
 [tool.maturin]
 sdist-include = [
- "src/*",
  "Cargo.toml",
+ "pyproject.toml",
  "pyrepscan/*.py",
- "pyrepscan/*.pyi"
+ "pyrepscan/*.pyi",
+ "src/*",
 ]
 
 [tool.poetry]

src/git_repository_scanner.rs

@@ -3,9 +3,10 @@ use crate::rules_manager;
 use chrono::prelude::*;
 use crossbeam_utils::atomic::AtomicCell;
 use crossbeam_utils::thread as crossbeam_thread;
-use crossbeam::queue::SegQueue;
+use crossbeam::queue::ArrayQueue;
 use git2::{Oid, Repository, Delta};
 use parking_lot::Mutex;
+use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
 use std::collections::HashMap;
 use std::path::Path;
@@ -124,18 +125,7 @@ fn scan_commit_oid(
  Ok(())
 }
 
-pub fn get_file_content(
- repository_path: &str,
- file_oid: &str,
-) -> Result<Vec<u8>, git2::Error> {
- let git_repo = Repository::open(repository_path)?;
- let oid = Oid::from_str(file_oid)?;
- let blob = git_repo.find_blob(oid)?;
-
- Ok(blob.content().to_vec())
-}
-
-fn get_oids(
+fn get_commit_oids(
  repository_path: &str,
  branch_glob_pattern: &str,
  from_timestamp: i64,
@@ -166,40 +156,46 @@ pub fn scan_repository(
  from_timestamp: i64,
  rules_manager: &rules_manager::RulesManager,
  output_matches: Arc<Mutex<Vec<HashMap<&str, String>>>>,
-) -> Result<(), PyErr> {
- let oids_queue = Arc::new(SegQueue::new());
- match get_oids(
+) -> PyResult<()> {
+ let commit_oids_queue;
+
+ match get_commit_oids(
  repository_path,
  branch_glob_pattern,
  from_timestamp
  ) {
- Ok(oids) => {
- for oid in oids {
- oids_queue.push(oid);
+ Ok(commit_oids) => {
+ if commit_oids.is_empty() {
+ return Ok(());
+ }
+
+ commit_oids_queue = ArrayQueue::new(commit_oids.len());
+ for commit_oid in commit_oids {
+ commit_oids_queue.push(commit_oid).unwrap();
  }
  },
  Err(error) => {
- return Err(pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))
+ return Err(PyRuntimeError::new_err(error.to_string()))
  },
  }
 
- py.check_signals()?;
-
  let mut py_signal_error: PyResult<()> = Ok(());
 
  let should_stop = AtomicCell::new(false);
+ let number_of_cores = std::thread::available_parallelism().unwrap().get();
+
  crossbeam_thread::scope(
  |scope| {
- for _ in 0..num_cpus::get() {
+ for _ in 0..number_of_cores {
  scope.spawn(
  |_| {
  if let Ok(git_repo) = Repository::open(repository_path) {
  while !should_stop.load() {
- if let Some(oid) = oids_queue.pop() {
+ if let Some(commit_oid) = commit_oids_queue.pop() {
  scan_commit_oid(
  &should_stop,
  &git_repo,
- &oid,
+ &commit_oid,
  rules_manager,
  output_matches.clone(),
  ).unwrap_or(());
@@ -212,7 +208,7 @@ pub fn scan_repository(
  );
  }
 
- while !oids_queue.is_empty() {
+ while !commit_oids_queue.is_empty() {
  py_signal_error = py.check_signals();
  if py_signal_error.is_err() {
  should_stop.store(true);
@@ -223,7 +219,7 @@ pub fn scan_repository(
  thread::sleep(time::Duration::from_millis(100));
  }
  }
- ).unwrap_or(());
+ ).unwrap_or_default();
 
  py_signal_error?;