feat(parser): relax host validation to support self-hosted GitLab & git.* domains (#314)

• Accept hosts starting with “git.” or “gitlab.” in _looks_like_git_host • Update doc-strings to document the heuristic • Adjust git-host-agnostic tests: expect ValueError for slug form with custom hosts; add real GitLab instance (git.rwth-aachen.de) to matrix

Author: filipchristiansen

src/gitingest/utils/query_parser_utils.py

@@ -60,22 +60,48 @@ def _is_valid_pattern(pattern: str) -> bool:
 
 def _validate_host(host: str) -> None:
  """
- Validate the given host against the known Git hosts.
+ Validate a hostname.
+
+ The host is accepted if it is either present in the hard-coded `KNOWN_GIT_HOSTS` list or if it satisfies the
+ simple heuristics in `_looks_like_git_host`, which try to recognise common self-hosted Git services (e.g. GitLab
+ instances on sub-domains such as `gitlab.example.com` or `git.example.com`).
 
  Parameters
  ----------
  host : str
- The host to validate.
+ Hostname (case-insensitive).
 
  Raises
  ------
  ValueError
- If the host is not a known Git host.
+ If the host cannot be recognised as a probable Git hosting domain.
  """
- if host not in KNOWN_GIT_HOSTS:
+ host = host.lower()
+ if host not in KNOWN_GIT_HOSTS and not _looks_like_git_host(host):
  raise ValueError(f"Unknown domain '{host}' in URL")
 
 
+def _looks_like_git_host(host: str) -> bool:
+ """
+ Check if the given host looks like a Git host.
+
+ The current heuristic returns `True` when the host starts with `git.` (e.g. `git.example.com`) or starts with
+ `gitlab.` (e.g. `gitlab.company.com`).
+
+ Parameters
+ ----------
+ host : str
+ Hostname (case-insensitive).
+
+ Returns
+ -------
+ bool
+ True if the host looks like a Git host, otherwise False.
+ """
+ host = host.lower()
+ return host.startswith(("git.", "gitlab."))
+
+
 def _validate_url_scheme(scheme: str) -> None:
  """
  Validate the given scheme against the known schemes.
@@ -90,6 +116,7 @@ def _validate_url_scheme(scheme: str) -> None:
  ValueError
  If the scheme is not 'http' or 'https'.
  """
+ scheme = scheme.lower()
  if scheme not in ("https", "http"):
  raise ValueError(f"Invalid URL scheme '{scheme}' in URL")
 

tests/query_parser/__init__.py

@@ -10,6 +10,7 @@
 import pytest
 
 from gitingest.query_parsing import parse_query
+from gitingest.utils.query_parser_utils import KNOWN_GIT_HOSTS
 
 # Repository matrix: (host, user, repo)
 _REPOS: List[Tuple[str, str, str]] = [
@@ -18,6 +19,8 @@
  ("bitbucket.org", "na-dna", "llm-knowledge-share"),
  ("gitea.com", "xorm", "xorm"),
  ("codeberg.org", "forgejo", "forgejo"),
+ ("git.rwth-aachen.de", "medialab", "19squared"),
+ ("gitlab.alpinelinux.org", "alpine", "apk-tools"),
 ]
 
 
@@ -43,6 +46,13 @@ async def test_parse_query_without_host(
 
  expected_url = f"https://{host}/{user}/{repo}"
 
+ # For slug form with a custom host (not in KNOWN_GIT_HOSTS) we expect a failure,
+ # because the parser cannot guess which domain to use.
+ if variant == "slug" and host not in KNOWN_GIT_HOSTS:
+ with pytest.raises(ValueError):
+ await parse_query(url, max_file_size=50, from_web=True)
+ return
+
  query = await parse_query(url, max_file_size=50, from_web=True)
 
  # Compare against the canonical dict while ignoring unpredictable fields.

tests/query_parser/test_git_host_agnostic.py

@@ -24,6 +24,9 @@
  "https://gitea.com/user/repo",
  "https://codeberg.org/user/repo",
  "https://gist.github.com/user/repo",
+ "https://git.example.com/user/repo",
+ "https://gitlab.example.com/user/repo",
+ "https://gitlab.example.se/user/repo",
 ]
 
 URLS_HTTP: List[str] = [url.replace("https://", "http://") for url in URLS_HTTPS]