introduce new function called scan_from_url

Author: Gal Ben David

Cargo.toml

@@ -1,14 +1,20 @@
 [package]
 name = "pyrepscan"
-version = "0.7.4"
+version = "0.8.0"
 authors = ["Gal Ben David <gal@intsights.com>"]
 edition = "2018"
 description = "A Git Repository Secrets Scanner written in Rust"
 readme = "README.md"
 repository = "https://github.com/intsights/pyrepscan"
 homepage = "https://github.com/intsights/pyrepscan"
 license = "MIT"
-keywords = ["adblock", "ads", "adblocker", "rust", "brave", "abp", "pyo3"]
+keywords = [
+ "git",
+ "secrets",
+ "scanner",
+ "rust",
+ "pyo3",
+]
 
 [package.metadata.maturin]
 requires-python = ">=3.6"
@@ -33,13 +39,13 @@ regex = "1"
 rayon = "1.5"
 chrono = "0.4"
 parking_lot = "0.11"
+num_cpus = "1"
 
 [dependencies.git2]
 version = "0.13"
-default-features = false
 
 [dependencies.pyo3]
-version = "0.13.1"
+version = "0.13.2"
 features = ["extension-module"]
 
 [profile.release]

README.md

@@ -142,6 +142,22 @@ A sample result would look like this:
 ```
 
 
+```python
+def scan_from_url(
+ self,
+ url: str,
+ repository_path: str,
+ branch_glob_pattern: typing.Optional[str],
+ from_timestamp: typing.Optional[int],
+) -> typing.List[typing.Dict[str, str]]
+```
+The same as `scan` function but also clones a repository from a given URL into the provided repository path.
+- `url` - URL of a git repository.
+- `repository_path` - The path to clone the repository to
+- `branch_glob_pattern` - A glob pattern to filter branches for the scan. If None is sent, defaults to `*`.
+- `from_timestamp` - A UTC timestamp (Int) that only commits that were created after this timestamp would be included in the scan. If None is sent, defaults to `0`.
+
+
 ```python
 def get_file_content(
  self,

pyproject.toml

@@ -14,7 +14,7 @@ strip = true
 
 [tool.poetry]
 name = "pyrepscan"
-version = "0.7.4"
+version = "0.8.0"
 authors = ["Gal Ben David <gal@intsights.com>"]
 description = "A Git Repository Secrets Scanner written in Rust"
 readme = "README.md"
@@ -49,3 +49,13 @@ gitpython = "*"
 wheel = "*"
 pytest-runner = "*"
 maturin = "*"
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = [
+ "--tb=native",
+ "--pythonwarnings=all",
+]
+testpaths = [
+ "tests",
+]

pyrepscan/pyrepscan.pyi

@@ -37,6 +37,14 @@ class GitRepositoryScanner:
  from_timestamp: typing.Optional[int],
  ) -> typing.List[typing.Dict[str, str]]: ...
 
+ def scan_from_url(
+ self,
+ url: str,
+ repository_path: str,
+ branch_glob_pattern: typing.Optional[str],
+ from_timestamp: typing.Optional[int],
+ ) -> typing.List[typing.Dict[str, str]]: ...
+
  def get_file_content(
  self,
  repository_path: str,

setup.cfg

@@ -2,12 +2,12 @@ use crate::rules_manager;
 
 use chrono::prelude::*;
 use git2::{Oid, Repository, Delta};
+use git2::Error;
 use parking_lot::Mutex;
 use rayon::prelude::*;
 use std::collections::HashMap;
-use std::sync::Arc;
-use git2::Error;
 use std::path::Path;
+use std::sync::Arc;
 
 fn scan_commit_oid(
  git_repo: &Repository,
@@ -152,13 +152,19 @@ pub fn scan_repository(
  }
  }
  }
+
+ let chunk_size = (oids.len() as f64 / (num_cpus::get() * 5) as f64).ceil() as usize;
  if !oids.is_empty() {
- let chunk_size = (oids.len() as f64 / 100.0).ceil();
- oids.par_chunks(chunk_size as usize).for_each_init(
- || Repository::open(repository_path).unwrap(),
- |git_repo, oids| {
+ oids.par_chunks(chunk_size).for_each(
+ |oids| {
+ let git_repo = Repository::open(repository_path).unwrap();
  for oid in oids {
- scan_commit_oid(git_repo, oid, rules_manager, output_matches.clone()).unwrap_or(());
+ scan_commit_oid(
+ &git_repo,
+ oid,
+ rules_manager,
+ output_matches.clone()
+ ).unwrap_or(());
  }
  },
  );

src/git_repository_scanner.rs

@@ -1,6 +1,7 @@
 mod git_repository_scanner;
 mod rules_manager;
 
+use git2::Repository;
 use parking_lot::Mutex;
 use pyo3::exceptions;
 use pyo3::prelude::*;
@@ -213,6 +214,52 @@ impl GitRepositoryScanner {
  Ok(matches.lock().to_object(py))
  }
  }
+
+ /// Scan a git repository for secrets. Rules shuld be loaded before calling this function.
+ ///
+ /// input:
+ /// url: str -> URL of a git repository
+ /// repository_path: str -> The path to clone the repository to
+ /// branch_glob_pattern: str -> A blob pattern to match against the git branches names.
+ /// Only matched branches will be scanned.
+ /// from_timestamp: int = 0 -> Unix epoch timestamp to start the scan from.
+ ///
+ /// returns:
+ /// list[dict] -> List of matches
+ ///
+ /// example:
+ /// grs.scan_from_url(
+ /// url="https://github.com/rust-lang/git2-rs",
+ /// repository_path="/path/to/repository",
+ /// branch_glob_pattern="*",
+ /// )
+ #[text_signature = "(url, repository_path, branch_glob_pattern, from_timestamp, /)"]
+ fn scan_from_url(
+ &self,
+ py: Python,
+ url: &str,
+ repository_path: &str,
+ branch_glob_pattern: Option<&str>,
+ from_timestamp: Option<i64>,
+ ) -> PyResult<Py<PyAny>> {
+ let matches = Arc::new(Mutex::new(Vec::<HashMap<&str, String>>::with_capacity(10000)));
+
+ if let Err(error) = Repository::clone(url, repository_path) {
+ return Err(exceptions::PyRuntimeError::new_err(error.to_string()));
+ };
+
+ if let Err(error) = git_repository_scanner::scan_repository(
+ repository_path,
+ branch_glob_pattern.unwrap_or("*"),
+ from_timestamp.unwrap_or(0),
+ &self.rules_manager,
+ matches.clone(),
+ ) {
+ Err(exceptions::PyRuntimeError::new_err(error.to_string()))
+ } else {
+ Ok(matches.lock().to_object(py))
+ }
+ }
 }
 
 /// PyRepScan is a Python library written in Rust. The library prodives an API to scan git repositories

src/lib.rs

@@ -13,6 +13,8 @@ def setUp(
  self,
  ):
  self.tmpdir = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmpdir.cleanup)
+
  bare_repo = git.Repo.init(
  path=self.tmpdir.name,
  )