Add support for external licenses in scans

CHANGELOG.rst

@@ -46,14 +46,22 @@ License detection:
  matches in a larger license detecion. This remove a larger number of false
  positive or ambiguous license detections.
 
-
 - The data structure of the JSON output has changed for licenses. We now
  return match details once for each matched license expression rather than
  once for each license in a matched expression. There is a new top-level
  "license_references" attribute that contains the data details for each
  detected license only once. This data can contain the reference license text
  as an option.
 
+- We can now detect licenses using custom license texts and license rules.
+ These can be provided as a one off in a directory or packaged as a plugin
+ for consistent reuse and deployment.
+
+- There is a new "scancode-reindex-licenses" command that replace the
+ "scancode --reindex-licenses" command line option which has been
+ removed. This new command supports simpler reindexing using custom
+ license texts and license rules contributed by plugins or stored in an
+ additional directory. 
 v31.2.1 - 2022-10-05
 ----------------------------------
 

azure-pipelines.yml

@@ -33,6 +33,7 @@ jobs:
  --ignore=tests/licensedcode/test_detection_datadriven2.py \
  --ignore=tests/licensedcode/test_detection_datadriven3.py \
  --ignore=tests/licensedcode/test_detection_datadriven4.py \
+ --ignore=tests/licensedcode/test_additional_license.py \
  tests/licensedcode
 
  license_datadriven1_2: |
@@ -78,6 +79,18 @@ jobs:
  venv/bin/pytest -n 3 -vvs --test-suite=all \
  tests/licensedcode/test_zzzz_cache.py
 
+ # this test runs in isolation because it modifies the actual
+ # license index with additional licenses provided by a plugin
+ # and we use the special --test-suite=plugins marker for these
+ # tests
+ additional_license_combined: |
+ venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_1/
+ venv/bin/pip install tests/licensedcode/data/additional_licenses/additional_plugin_2/
+ venv/bin/scancode-reindex-licenses \
+ --additional-directory tests/licensedcode/data/additional_licenses/additional_dir/
+ venv/bin/pytest -vvs --test-suite=plugins \
+ tests/licensedcode/test_additional_license.py
+
  - template: etc/ci/azure-posix.yml
  parameters:
  job_name: ubuntu18_cpython

conftest.py

@@ -38,6 +38,7 @@
 ################################################################################
 SLOW_TEST = 'scanslow'
 VALIDATION_TEST = 'scanvalidate'
+PLUGINS_TEST = 'scanplugins'
 
 
 def pytest_configure(config):
@@ -53,8 +54,14 @@ def pytest_configure(config):
  ': Mark a ScanCode test as a validation test, super slow, long running test.',
  )
 
+ config.addinivalue_line(
+ 'markers',
+ PLUGINS_TEST +
+ ': Mark a ScanCode test as a special CI test to tests installing additional plugins.',
+ )
+
 
-TEST_SUITES = 'standard', 'all', 'validate'
+TEST_SUITES = ('standard', 'all', 'validate', 'plugins',)
 
 
 def pytest_addoption(parser):
@@ -72,9 +79,11 @@ def pytest_addoption(parser):
  help='Select which test suite to run: '
  '"standard" runs the standard test suite designed to run reasonably fast. '
  '"all" runs "standard" and "slow" (long running) tests. '
- '"validate" runs all the tests. '
+ '"validate" runs all the tests, except the "plugins" tests. '
+ '"plugins" runs special plugins tests. Needs extra setup, and is used only in the CI. '
  'Use the @pytest.mark.scanslow marker to mark a test as "slow" test. '
  'Use the @pytest.mark.scanvalidate marker to mark a test as a "validate" test.'
+ 'Use the @pytest.mark.scanplugins marker to mark a test as a "plugins" test.'
  )
 
 ################################################################################
@@ -87,13 +96,19 @@ def pytest_collection_modifyitems(config, items):
  test_suite = config.getvalue('test_suite')
  run_everything = test_suite == 'validate'
  run_slow_test = test_suite in ('all', 'validate')
+ run_only_plugins = test_suite == 'plugins'
 
  tests_to_run = []
  tests_to_skip = []
 
  for item in items:
  is_validate = bool(item.get_closest_marker(VALIDATION_TEST))
  is_slow = bool(item.get_closest_marker(SLOW_TEST))
+ is_plugins = bool(item.get_closest_marker(PLUGINS_TEST))
+
+ if is_plugins and not run_only_plugins:
+ tests_to_skip.append(item)
+ continue
 
  if is_validate and not run_everything:
  tests_to_skip.append(item)

docs/source/cli-reference/core-options.rst

@@ -69,22 +69,6 @@ Comparing Progress Message Options
 
 ----
 
-``--reindex-licenses`` Option
------------------------------
-
- ScanCode maintains a license index to search for and detect licenses. When Scancode is
- configured for the first time, a license index is built and used in every scan thereafter.
-
- This ``--reindex-licenses`` option rebuilds the license index. Running a scan with this option
- displays the following message to the terminal in addition to what it normally shows::
-
- Checking and rebuilding the license index...
-
- ..
- [ToDo] Research and Write Better
-
-----
-
 ``--from-json`` Option
 ----------------------
 

docs/source/cli-reference/index.rst

@@ -8,6 +8,7 @@
  help-text-options
  list-options
  simple-examples
+ other-commands
  basic-options
  core-options
  output-format

docs/source/cli-reference/list-options.rst

@@ -30,6 +30,10 @@ available in the command line.
 
 ----
 
+.. include:: /rst_snippets/scancode-reindex-licenses.rst
+
+----
+
 .. include:: /rst_snippets/core_options.rst
 
 ----

docs/source/cli-reference/other-commands.rst

@@ -0,0 +1,102 @@
+Other available CLIs
+====================
+
+.. _other_cli:
+
+----
+
+.. include:: /rst_snippets/scancode-reindex-licenses.rst
+
+----
+
+.. include:: /rst_snippets/extract.rst
+
+----
+
+``scancode-reindex-licenses`` command
+-------------------------------------
+
+ScanCode maintains a license index to search for and detect licenses. When Scancode is
+configured for the first time, a license index is built and used in every scan thereafter.
+
+This ``scancode-reindex-licenses`` command rebuilds the license index. Running this command
+displays the following message to the terminal::
+
+ Checking and rebuilding the license index...
+
+This has several CLI options as follows:
+
+``--additional-directory`` Option:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``--additional-directory`` option allows the user to include additional directories
+of licenses to use in license detection.
+
+This command only needs to be run once for each set of additional directories, in all subsequent
+runs of Scancode with the same directories all the licenses in the directories will be cached
+and used in License detection. But reindexing removes these directories, if they aren't
+reintroduced as additional directories.
+
+The directory structure should look something like this::
+
+ additional_license_directory/
+ ├── licenses/
+ │ ├── example-installed-1.LICENSE
+ │ └── example-installed-1.yaml
+ ├── rules/
+ │ ├── example-installed-1.RULE
+ │ └── example-installed-1.yaml
+
+Here is an example of reindexing the license cache using the ``--additional-directory PATH`` option
+with a single directory::
+
+ scancode-reindex-licenses --additional-directory tests/licensedcode/data/additional_licenses/additional_dir/
+
+You can also include multiple directories like so::
+
+ scancode-reindex-licenses --additional-directory /home/user/external_licenses/external1 --additional-directory /home/user/external_licenses/external2
+
+If you want to continue running scans with ``/home/user/external_licenses/external1`` and
+``/home/user/external_licenses/external2``, you can simply run scans after the command above
+reindexing with those directories and they will be included. ::
+
+ scancode -l --license-text --json-pp output.json samples
+
+However, if you wanted to run a scan with a new set of directories, such as
+``home/user/external_licenses/external1`` and ``home/user/external_licenses/external3``, you would
+need to reindex the license index with those directories as parameters::
+
+ scancode --additional-directory /home/user/external_licenses/external1 --additional-directory /home/user/external_licenses/external3
+
+.. include:: /rst_snippets/note_snippets/additional_directory_is_temp.rst
+
+
+.. note::
+
+ You can also install external licenses through a plugin for
+ better reproducibility and distribution of those license/rules
+ for use in conjunction with scancode-toolkit licenses.
+ See :ref:`install_new_license_plugin`
+
+
+``--only-builtin`` Option:
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Rebuild the license index excluding any additional license directory or additional
+license plugins which were added previously, i.e. with only builtin scancode license and rules.
+
+This is applicable when there are additional license plugins installed already and you want to
+reindex the licenses without these licenses from the additional plugins.
+
+.. note::
+
+ Running the ``--only-builtin`` command won't get rid of the installed license plugins, it
+ would just reindex without the licenses from these plugins for once. Another reindex afterwards
+ without this option would bring back the licenses from the plugins again in the index.
+
+
+``--all-languages`` Option:
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Rebuild the license index including texts all languages (and not only
+English) and exit. This is an EXPERIMENTAL option.

docs/source/cli-reference/output-format.rst

@@ -183,7 +183,7 @@ following options.
  "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
  "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit-old-style",
  "spdx_license_key": null,
- "spdx_url": "",
+ "spdx_url": null,
  "start_line": 9,
  "end_line": 15,
  "matched_rule": {

docs/source/cli-reference/synopsis.rst

@@ -234,7 +234,7 @@ A sample JSON output for an individual file will look like::
  "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
  "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:mit-old-style",
  "spdx_license_key": null,
- "spdx_url": "",
+ "spdx_url": null,
  "start_line": 9,
  "end_line": 15,
  "matched_rule": {

docs/source/how-to-guides/index.rst

@@ -8,3 +8,4 @@
 
  add_new_license
  add_new_license_detection_rule
+ install_new_license_plugin