Merge pull request #115 from nexB/auth-gh

Use authenticated requests for GitHub REST API calls

Author: keshav-space

.gitignore

@@ -72,3 +72,6 @@ tcl
 
 # Ignore Jupyter Notebook related temp files
 .ipynb_checkpoints/
+
+# Env Files
+.env

CHANGELOG.rst

@@ -2,6 +2,11 @@ Changelog
 =========
 
 
+v0.5.1
+-------
+- Use authenticated requests for GitHub REST API calls
+
+
 v0.5.0
 -------
 - FetchCode now supports retrieving package info for following generic packages: 

src/fetchcode/package.py

@@ -472,7 +472,9 @@ class UtilLinuxDirectoryListedSource(DirectoryListedSource):
  source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/"
  is_nested = True
  # Source archive ex: util-linux-1.2.3.tar.gz
- source_archive_regex = re.compile(r"^(util-linux-)(?P<version>[\w.-]*)(.tar.gz)$")
+ source_archive_regex = re.compile(
+ r"^(util-linux-|util-linux-ng-)(?P<version>[\w.-]*)(.tar.gz)$"
+ )
  ignored_files_and_dir = []
 
 
@@ -666,7 +668,7 @@ class BareboxDirectoryListedSource(DirectoryListedSource):
 
 
 class LinuxDirectoryListedSource(DirectoryListedSource):
- source_url = "https://cdn.kernel.org/pub/linux/kernel/"
+ source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/"
  # Source archive ex: linux-1.2.3.tar.gz
  source_archive_regex = re.compile(r"^(linux-)(?P<version>[\w.-]*)(.tar.gz)$")
  is_nested = True

src/fetchcode/package_util.py

@@ -55,7 +55,7 @@ def get_default_package(cls, purl):
  namespace = purl.namespace
  base_path = "https://api.github.com/repos"
  api_url = f"{base_path}/{namespace}/{name}"
- response = utils.get_response(api_url)
+ response = utils.get_github_rest(api_url)
  homepage_url = response.get("homepage")
  vcs_url = response.get("git_url")
  github_url = "https://github.com"
@@ -124,6 +124,12 @@ def _get_github_packages(purl, version_regex, ignored_tag_regex, default_package
  version = tag
 
  version = version.strip("Vv").strip()
+ if "+" in version:
+ first, last = version.split("+")
+ first.replace("_", ".")
+ version = f"{first}+{last}"
+ else:
+ version = version.replace("_", ".")
  if not version or not version[0].isdigit():
  continue
 
@@ -227,7 +233,7 @@ def get_package_info(cls, gh_purl):
  package_dict["type"] = "openssl"
  package_dict["namespace"] = None
  package_dict["name"] = "openssl"
- package_dict["version"] = package_dict["version"].replace("_", ".")
+ package_dict["version"] = package_dict["version"]
 
  yield package_from_dict(package_dict)
 
@@ -251,7 +257,7 @@ def get_package_info(cls, gh_purl):
  package_dict["type"] = "generic"
  package_dict["namespace"] = None
  package_dict["name"] = "erofs-utils"
- package_dict["version"] = package_dict["version"].replace("_", ".")
+ package_dict["version"] = package_dict["version"]
 
  yield package_from_dict(package_dict)
 
@@ -279,7 +285,7 @@ def get_package_info(cls, gh_purl, package_name):
  package_dict["type"] = "generic"
  package_dict["namespace"] = None
  package_dict["name"] = package_name
- package_dict["version"] = package_dict["version"].replace("_", ".")
+ package_dict["version"] = package_dict["version"]
 
  yield package_from_dict(package_dict)
 

src/fetchcode/utils.py

@@ -118,13 +118,18 @@ class GraphQLError(Exception):
  pass
 
 
-def github_response(graphql_query):
+def get_github_token():
  gh_token = os.environ.get("GH_TOKEN", None)
  if not gh_token:
  from dotenv import load_dotenv
 
  load_dotenv()
  gh_token = os.environ.get("GH_TOKEN", None)
+ return gh_token
+
+
+def github_response(graphql_query):
+ gh_token = get_github_token()
 
  if not gh_token:
  msg = (
@@ -149,11 +154,22 @@ def github_response(graphql_query):
  return response
 
 
-def get_response(url):
+def get_github_rest(url):
+ headers = None
+ gh_token = get_github_token()
+ if gh_token:
+ headers = {
+ "Authorization": f"Bearer {gh_token}",
+ }
+
+ return get_response(url, headers)
+
+
+def get_response(url, headers=None):
  """
  Generate `Package` object for a `url` string
  """
- resp = requests.get(url)
+ resp = requests.get(url, headers=headers)
  if resp.status_code == 200:
  return resp.json()