diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..b5bc2d49cda 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,9 @@ Next release ``ValueError: did not find any 'pre_scan' hooks``. https://github.com/aboutcode-org/scancode-toolkit/issues/5363 +- Add a new CLI option for a compact JSON output with ``--compact-json`` + https://github.com/aboutcode-org/scancode-toolkit/issues/4555 + v33.0.0rc1 - 2026-05-14 ------------------------ diff --git a/configure b/configure index 8f2f685f10e..529373baf0f 100755 --- a/configure +++ b/configure @@ -161,7 +161,7 @@ FLOT_REQUIREMENTS="etc/thirdparty/flot-0.7.3-py3-none-any.whl" VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" +CLEANABLE="build tmp dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json .pytest_cache .ruff_cache src/scancode_toolkit.egg-info" # the URL to download virtualenv.pyz if needed diff --git a/docs/source/reference/scancode-cli/cli-help-text-options.rst b/docs/source/reference/scancode-cli/cli-help-text-options.rst index caa0a03636b..b09883adde0 100644 --- a/docs/source/reference/scancode-cli/cli-help-text-options.rst +++ b/docs/source/reference/scancode-cli/cli-help-text-options.rst @@ -46,11 +46,19 @@ The following help text is displayed for ScanCode version 32.0.0: Options: primary scans: - -l, --license Scan for licenses. - -p, --package Scan for application package and dependency - manifests, lockfiles and related data. - --system-package Scan for installed system package databases. - -c, --copyright Scan for copyrights. + -l, --license Scan for licenses. + -p, --package Scan for application package and dependency + manifests, lockfiles and related data. + --system-package Scan for installed system package databases. + --package-in-compiled Scan for package and dependency related data + in compiled binaries. Currently supported compiled + binaries: Go, Rust. + --package-only Scan for system and application package data and skip + license/copyright detection and top-level package + creation. + -c, --copyright Scan for copyrights. + --go-symbol Collect Go symbols. + --rust-symbol Collect Rust symbols from rust binaries. other scans: -i, --info Scan for file information (size, checksums, etc). @@ -59,133 +67,142 @@ The following help text is displayed for ScanCode version 32.0.0: -u, --url Scan for urls. scan options: - --license-diagnostics In license detections, include diagnostic details - to figure out the license detection post - processing steps applied. - --license-score INTEGER Do not return license matches with a score lower - than this score. A number between 0 and 100. - [default: 0] - --license-text Include the detected licenses matched text. - --license-text-diagnostics In the matched license text, include diagnostic + --license-diagnostics In license detections, include diagnostic + details to figure out the license detection + post processing steps applied. + --license-score INTEGER Do not return license matches with a score + lower than this score. A number between 0 and + 100. [default: 0] + --license-text Include the detected licenses matched text. + --license-text-diagnostics In the matched license text, include diagnostic highlights surrounding with square brackets [] words that are not matched. - --license-url-template URL Set the template URL used for the license - reference URLs. Curly braces ({}) are replaced by - the license key. [default: https://scancode- - licensedb.aboutcode.org/{}] - --max-email INT Report only up to INT emails found in a file. Use + --license-url-template TEXT Set the template URL used for the license + reference URLs. Curly braces ({}) are replaced + by the license key. [default: + https://scancode-licensedb.aboutcode.org/{}] + --max-email INT Report only up to INT emails found in a file. + Use 0 for no limit. [default: 50] + --max-url INT Report only up to INT urls found in a file. Use 0 for no limit. [default: 50] - --max-url INT Report only up to INT urls found in a file. Use 0 - for no limit. [default: 50] - --unknown-licenses [EXPERIMENTAL] Detect unknown licenses. + --unknown-licenses [EXPERIMENTAL] Detect unknown licenses. output formats: - --json FILE Write scan output as compact JSON to FILE. + --compact-json FILE Write scan output as compact JSON to FILE. + --json FILE Write scan output as JSON to FILE. --json-pp FILE Write scan output as pretty-printed JSON to FILE. --json-lines FILE Write scan output as JSON Lines to FILE. --yaml FILE Write scan output as YAML to FILE. --csv FILE [DEPRECATED] Write scan output as CSV to FILE. The - --csv option is deprecated and will be replaced by new - CSV and tabular output formats in the next ScanCode - release. Visit https://github.com/aboutcode-org/scancode- - toolkit/issues/3043 to provide inputs and feedback. + --csv option is deprecated and will be replaced by + new CSV and tabular output formats in the next + ScanCode release. Visit + https://github.com/nexB/scancode-toolkit/issues/3043 + to provide inputs and feedback. --html FILE Write scan output as HTML to FILE. --custom-output FILE Write scan output to FILE formatted with the custom Jinja template file. - --debian FILE Write scan output in machine-readable Debian copyright - format to FILE. + --debian FILE Write scan output in machine-readable Debian + copyright format to FILE. --custom-template FILE Use this Jinja template FILE as a custom template. --cyclonedx FILE Write scan output in CycloneDX JSON format to FILE. --cyclonedx-xml FILE Write scan output in CycloneDX XML format to FILE. --spdx-rdf FILE Write scan output as SPDX RDF to FILE. --spdx-tv FILE Write scan output as SPDX Tag/Value to FILE. - --html-app FILE (DEPRECATED: use the ScanCode Workbench app instead) - Write scan output as a mini HTML application to FILE. output filters: - --ignore-author Ignore a file (and all its findings) if an - author contains a match to the - regular expression. Note that this will ignore - a file even if it has other findings such as a - license or errors. - --ignore-copyright-holder Ignore a file (and all its findings) if a - copyright holder contains a match to the - regular expression. Note that this - will ignore a file even if it has other - scanned data such as a license or errors. - --only-findings Only return files or directories with findings - for the requested scans. Files and directories - without findings are omitted (file information - is not treated as findings). + --ignore-author Ignore a file (and all its findings) if an + author contains a match to the + regular expression. Note that this will + ignore a file even if it has other findings + such as a license or errors. + --ignore-copyright-holder + Ignore a file (and all its findings) if a + copyright holder contains a match to the + regular expression. Note that this + will ignore a file even if it has other + scanned data such as a license or errors. + --only-findings Only return files or directories with + findings for the requested scans. Files and + directories without findings are omitted + (file information is not treated as + findings). output control: --full-root Report full, absolute paths. - --strip-root Strip the root directory segment of all paths. The default is to - always include the last directory segment of the scanned path - such that all paths have a common root directory. + --strip-root Strip the root directory segment of all paths. The default is + to always include the last directory segment of the scanned + path such that all paths have a common root directory. pre-scan: - --classify Classify files with flags indicating whether the file is a - legal, readme, test or similar file. --facet = Add the to files with a path matching - . + . post-scan: - --consolidate Group resources by Packages or license and copyright - holder and return those groupings as a list of - consolidated packages and a list of consolidated - components. This requires the scan to have/be run - with the copyright, license, and package options - active + --classify Classify files with flags telling if the file is a + legal, or readme or test file, etc. + --consolidate Group resources by Packages or license and + copyright holder and return those groupings as a + list of consolidated packages and a list of + consolidated components. This requires the scan to + have/be run with the copyright, license, and + package options active --filter-clues Filter redundant duplicated clues already contained - in detected license and copyright texts and notices. + in detected license and copyright texts and + notices. --license-clarity-score Compute a summary license clarity score at the - codebase level. + codebase level. --license-policy FILE Load a License Policy file and apply it to the scan - at the Resource level. + at the Resource level. --license-references Return reference data for all licenses and license - rules present in detections. + rules present in detections. --mark-source Set the "is_source" to true for directories that - contain over 90% of source files as children and - descendants. Count the number of source files in a - directory as a new source_file_counts attribute + contain over 90% of source files as children and + descendants. Count the number of source files in a + directory as a new source_file_counts attribute --summary Summarize scans by providing declared origin - information and other detected origin info at the - codebase attribute level. + information and other detected origin info at the + codebase attribute level. --tallies Compute tallies for license, copyright and other - scans at the codebase level. + scans at the codebase level. --tallies-by-facet Compute tallies for license, copyright and other - scans and group the results by facet. + scans and group the results by facet. --tallies-key-files Compute tallies for license, copyright and other - scans for key, top-level files. Key files are top- - level codebase files such as COPYING, README and - package manifests as reported by the --classify - option "is_legal", "is_readme", "is_manifest" and - "is_top_level" flags. - --tallies-with-details Compute tallies of license, copyright and other scans - at the codebase level, keeping intermediate details - at the file and directory level. + scans for key, top-level files. Key files are top- + level codebase files such as COPYING, README and + package manifests as reported by the --classify + option "is_legal", "is_readme", "is_manifest" and + "is_top_level" flags. + --tallies-with-details Compute tallies of license, copyright and other + scans at the codebase level, keeping intermediate + details at the file and directory level. + --todo Summarize scans by providing all ambiguous + detections which are todo items and needs manual + review. core: - --ignore Ignore files matching . - --timeout Stop an unfinished file scan after a timeout in + --ignore Ignore files matching . + --timeout Stop an unfinished file scan after a timeout in seconds. [default: 120 seconds] - -n, --processes INT Set the number of parallel processes to use. Disable - parallel processing if 0. Also disable threading if - -1. [default: (number of CPUs)-1] - --config-file FILENAME Path to the configuration file. - -q, --quiet Do not print summary or progress. - -v, --verbose Print progress as file-by-file path instead of a + -n, --processes INT Set the number of parallel processes to use. + Disable parallel processing if 0. Also disable + threading if -1. [default: (number of CPUs)-1] + --config-file FILENAME Path to the configuration file. + -q, --quiet Do not print summary or progress. + -v, --verbose Print progress as file-by-file path instead of a progress bar. Print verbose scan counters. - --from-json Load codebase from one or more JSON scan + --from-json Load codebase from one or more JSON scan file(s). - --max-in-memory INTEGER Maximum number of files and directories scan details - kept in memory during a scan. Additional files and - directories scan details above this number are cached - on-disk rather than in memory. Use 0 to use unlimited - memory and disable on-disk caching. Use -1 to use - only on-disk caching. [default: 10000] - --max-depth INTEGER Maximum nesting depth of subdirectories to scan. + --use-cached-results (EXPERIMENTAL) Enables the creation, update, and + usage of cached results during scan time. + --max-in-memory INTEGER Maximum number of files and directories scan + details kept in memory during a scan. Additional + files and directories scan details above this + number are cached on-disk rather than in memory. + Use 0 to use unlimited memory and disable on-disk + caching. Use -1 to use only on-disk caching. + [default: 10000] + --max-depth INTEGER Maximum nesting depth of subdirectories to scan. Descend at most INTEGER levels of directories below and including the starting directory. Use 0 for no scan depth limit. @@ -195,7 +212,8 @@ The following help text is displayed for ScanCode version 32.0.0: -A, --about Show information about ScanCode and licensing and exit. -V, --version Show the version and exit. --examples Show command examples and exit. - --list-packages Show the list of supported package manifest parsers and exit. + --list-packages Show the list of supported package manifest parsers and + exit. --plugins Show the list of available ScanCode plugins and exit. --print-options Show the list of selected options and exit. @@ -212,8 +230,9 @@ The following help text is displayed for ScanCode version 32.0.0: scancode --json-pp - --license --package samples - Note: when you run ScanCode, a progress bar is displayed with a counter of the - number of files processed. Use --verbose to display file-by-file progress. + Note: when you run scancode, a progress bar is displayed with a counter of + the number of files processed. Use --verbose to display file-by-file + progress. .. _cli-examples-option: @@ -402,7 +421,7 @@ for ScanCode Version 32.0.0. -------------------------------------------- - Plugin: scancode_output:json class: formattedcode.output_json:JsonCompactOutput + Plugin: scancode_output:json class: formattedcode.output_json:JsonOutput codebase_attributes: resource_attributes: sort_order: 100 diff --git a/docs/source/reference/scancode-cli/cli-output-format-options.rst b/docs/source/reference/scancode-cli/cli-output-format-options.rst index 1f3a4bb852c..61da22a3bb6 100644 --- a/docs/source/reference/scancode-cli/cli-output-format-options.rst +++ b/docs/source/reference/scancode-cli/cli-output-format-options.rst @@ -221,6 +221,89 @@ Quick reference ---- +.. _cli-compact-json-option: + +``--compact-json FILE`` +----------------------- + + ``--compact-json`` is an output format option which outputs the resusts in the most + compact manner possible. To make the output results compact: + + * duplicate data is removed so we only have unique detection results only once + * only the most important data attributes are kept and the rest is removed + * longer text fields are either removed or CLI options adding longer/diangostic text + fields are made incompatible with the compact option + * empty data attributes are removed + + The following code performs a scan on the samples directory, and publishes the results in + ``compact-json`` format + + .. code-block:: shell + + scancode -clpieu --compact-json output.json samples + + **Example** + + .. code-block:: json + + { + "path": "samples/JGroups/LICENSE", + "type": "file", + "size": 26430, + "date": "2022-04-20", + "sha1": "e60c2e780886f95df9c9ee36992b8edabec00bcc", + "md5": "7fbc338309ac38fefcd64b04bb903e34", + "sha256": "a190dc9c8043755d90f8b0a75fa66b9e42d4af4c980bf5ddc633f0124db3cee7", + "sha1_git": "b1e3f5a2638797271cbc9b91b856c05ed6942c8f", + "mime_type": "text/plain", + "file_type": "ASCII text", + "is_text": true, + "detected_license_expression_spdx": "LGPL-2.1-only", + "percentage_of_license_text": 100.0, + "copyrights": [ + { + "copyright": "Copyright (c) 1991, 1999 Free Software Foundation, Inc.", + "start_line": 4, + "end_line": 4 + }, + { + "copyright": "copyrighted by the Free Software Foundation", + "start_line": 429, + "end_line": 429 + } + ], + "holders": [ + { + "holder": "Free Software Foundation, Inc.", + "start_line": 4, + "end_line": 4 + }, + { + "holder": "the Free Software Foundation", + "start_line": 429, + "end_line": 429 + } + ], + "license_detections": [ + { + "identifier": "lgpl_2_1-64dbac63-7b61-7151-023f-321235aeef62" + } + ] + }, + { + "path": "samples/JGroups/licenses", + "type": "directory", + "files_count": 5, + "size_count": 54552 + } + + + This is the recommended output option if you want the output filesize to be as minimal as possible + while retaining the most important information in ScanCode-Toolkit. + + +--- + .. _cli-json-lines-option: ``--json-lines FILE`` diff --git a/docs/source/rst-snippets/cli-output-format-options.rst b/docs/source/rst-snippets/cli-output-format-options.rst index b73f8cfbeed..0f59da0bf6d 100644 --- a/docs/source/rst-snippets/cli-output-format-options.rst +++ b/docs/source/rst-snippets/cli-output-format-options.rst @@ -1,7 +1,14 @@ **Output formats** ------------------ ---json FILE Write scan output as compact JSON to FILE. +--json FILE Write scan output as standard JSON to FILE. + +--compact-json FILE Write scan output as compact JSON to FILE. + This output format only keeps the information + in scans which are most important parts, + removes any redundant information or empty + attributes to make the output as compact as + possible. --json-pp FILE Write scan output as pretty-printed JSON to FILE. This is one of the recommended output diff --git a/docs/source/rst-snippets/cli-output-to-stdout.rst b/docs/source/rst-snippets/cli-output-to-stdout.rst index 778e63f3cc5..f0185302744 100644 --- a/docs/source/rst-snippets/cli-output-to-stdout.rst +++ b/docs/source/rst-snippets/cli-output-to-stdout.rst @@ -2,8 +2,8 @@ Print to ``stdout`` / terminal ------------------------------ If you want to format the output in JSON and print it at stdout, you can replace the JSON filename -with a "-", like ``--json-pp -`` instead of ``--json-pp output.json``. +with a "-", like ``--compact-json -`` instead of ``--compact-json output.json``. The following command will output the scan results in JSON format to ``stdout`` (in the terminal):: - ./scancode -clpieu --json-pp - samples/ + ./scancode -clpieu --compact-json - samples/ diff --git a/docs/source/tutorials/configuring-scan-output-formats.rst b/docs/source/tutorials/configuring-scan-output-formats.rst index 69bbde4849f..164b019ee24 100644 --- a/docs/source/tutorials/configuring-scan-output-formats.rst +++ b/docs/source/tutorials/configuring-scan-output-formats.rst @@ -15,6 +15,8 @@ The following commands will output scan results in a formatted json file: * ``scancode --json /path/to/output.json /path/to/target/dir`` +* ``scancode --compact-json /path/to/output.json /path/to/target/dir`` + * ``scancode --json-pp /path/to/output.json /path/to/target/dir`` * ``scancode --json-lines /path/to/output.json /path/to/target/dir`` diff --git a/pyproject-scancode-toolkit-mini.toml b/pyproject-scancode-toolkit-mini.toml index a5762e9c22f..eb24e90e367 100644 --- a/pyproject-scancode-toolkit-mini.toml +++ b/pyproject-scancode-toolkit-mini.toml @@ -306,7 +306,8 @@ ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights" [project.entry-points.scancode_output] html = "formattedcode.output_html:HtmlOutput" html-app = "formattedcode.output_html:HtmlAppOutput" -json = "formattedcode.output_json:JsonCompactOutput" +json = "formattedcode.output_json:JsonOutput" +compact-json = "formattedcode.output_json:JsonCompactOutput" json-pp = "formattedcode.output_json:JsonPrettyOutput" spdx-tv = "formattedcode.output_spdx:SpdxTvOutput" spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput" diff --git a/pyproject-scancode-toolkit.toml b/pyproject-scancode-toolkit.toml index e481874b662..845cf2842c6 100644 --- a/pyproject-scancode-toolkit.toml +++ b/pyproject-scancode-toolkit.toml @@ -307,7 +307,8 @@ ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights" [project.entry-points.scancode_output] html = "formattedcode.output_html:HtmlOutput" html-app = "formattedcode.output_html:HtmlAppOutput" -json = "formattedcode.output_json:JsonCompactOutput" +json = "formattedcode.output_json:JsonOutput" +compact-json = "formattedcode.output_json:JsonCompactOutput" json-pp = "formattedcode.output_json:JsonPrettyOutput" spdx-tv = "formattedcode.output_spdx:SpdxTvOutput" spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput" diff --git a/pyproject.toml b/pyproject.toml index 31a5711823c..c8627c1ba8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -311,7 +311,8 @@ ignore-copyrights = "cluecode.plugin_ignore_copyrights:IgnoreCopyrights" [project.entry-points.scancode_output] html = "formattedcode.output_html:HtmlOutput" html-app = "formattedcode.output_html:HtmlAppOutput" -json = "formattedcode.output_json:JsonCompactOutput" +json = "formattedcode.output_json:JsonOutput" +compact-json = "formattedcode.output_json:JsonCompactOutput" json-pp = "formattedcode.output_json:JsonPrettyOutput" spdx-tv = "formattedcode.output_spdx:SpdxTvOutput" spdx-rdf = "formattedcode.output_spdx:SpdxRdfOutput" diff --git a/src/formattedcode/output_compact.py b/src/formattedcode/output_compact.py new file mode 100644 index 00000000000..dd8c347e3f3 --- /dev/null +++ b/src/formattedcode/output_compact.py @@ -0,0 +1,233 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +""" +Helper functions to write scan results in a compact manner as JSON, preserving +only non-duplicated key detection information, and skipping attributes with +large values or more details. +""" + +# Tracing flags +TRACE = False + + +def logger_debug(*args): + pass + + +if TRACE: + import sys + import logging + + logger = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + def logger_debug(*args): + return logger.debug(' '.join(isinstance(a, str) + and a or repr(a) for a in args)) + + +COMPACT_CONFLICTING_OPTIONS = [ + 'summary', 'classify', 'consolidate', 'todo', 'mark_source', 'tallies', + 'tallies_by_facet', 'tallies_with_details', 'tallies_key_files', + 'license_references', 'license_text', 'license_diagnostics', 'license_text_diagnostics', + 'go_symbol', 'rust_symbol' +] + +PACKAGE_DETECTION_FIELDS = [ + # Detection fields have a lot of inlined duplicated data which needs to be skipped, + # just identifiers are enough, and this will be kept in the data mapping seperately + "license_detections", + "other_license_detections" +] + +COMPACT_PACKAGES_SKIP_ATTRIBUTES = [ + # These are PackageURL fields and are coontained in `package_uid` which is a purl + identifier + "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "purl", + # This usually has text which is too large and not useful for a compact output + "description", + # Just `declared_license_expression_spdx` is enough + "declared_license_expression", + # Just `other_license_expression_spdx` is enough + "other_license_expression_spdx", + # Already captured by license detection fields if present + "extracted_license_statement", + # This usually has text which is too large and not useful for a compact output + "notice_text", + "extra_data", + # This can be deduced from datafile_paths + "datasource_ids" +] + PACKAGE_DETECTION_FIELDS + + +COMPACT_DEPENDENCIES_SKIP_ATTRIBUTES = [ + "resolved_package", + # Not essential information + "extra_data", + # This can be deduced from `datafile_path` + "datasource_id" +] + +COMPACT_LICENSES_SKIP_ATTRIBUTES = [ + # Just keeping `license_expression_spdx` is enough + "license_expression", + # This needs to be compact further, so we can't have as-is + "reference_matches" +] + +COMPACT_LICENSE_MATCHES_SKIP_ATTRIBUTES = [ + # Just keeping `license_expression_spdx` is enough + "license_expression", + # We have the `rule_identifier` and this is derivative + "rule_url", + # We have the `rule_identifier` and this info can be fetched + "rule_relevance" +] + +COMPACT_RESOURCES_SKIP_ATTRIBUTES = [ + # Just keeping `detected_license_expression_spdx` is enough + "detected_license_expression", + # Detection fields have a lot of inlined duplicated data which needs to be skipped, + # just identifiers are enough, and this will be kept in the data mapping seperately + "license_detections", + "package_data", + # license clues are matches not aggregated in the top-level license detections, but + # these mappings need to be processed to make them more compact + "license_clues", + # `name`, `base_name` and `extension` info is also present in the full-path + "name", + "base_name", + "extension" +] + +def get_compact_packages(package_mapping): + compact_package_mapping = get_compact_mapping( + mapping=package_mapping, + keys_to_skip=COMPACT_PACKAGES_SKIP_ATTRIBUTES, + ) + + # Only keep the license detection identifiers from the detection mappings + for detection_attribute in PACKAGE_DETECTION_FIELDS: + detection_mapping = [ + {"identifier": detection.get("identifier")} + for detection in package_mapping.get(detection_attribute, []) + if detection + ] + if detection_mapping: + compact_package_mapping[detection_attribute] = detection_mapping + + return compact_package_mapping + + +def get_compact_dependencies(dependency_mapping): + return get_compact_mapping( + mapping=dependency_mapping, + keys_to_skip=COMPACT_DEPENDENCIES_SKIP_ATTRIBUTES, + ) + + +def get_compact_license_detections(license_mapping): + compact_license_mapping = get_compact_mapping( + mapping=license_mapping, + keys_to_skip=COMPACT_LICENSES_SKIP_ATTRIBUTES, + ) + compact_license_mapping["reference_matches"] = [ + get_compact_mapping( + mapping=match, + keys_to_skip=COMPACT_LICENSE_MATCHES_SKIP_ATTRIBUTES, + ) + for match in license_mapping.get("reference_matches", []) + ] + return compact_license_mapping + + +def get_compact_mapping(mapping, keys_to_skip=[]): + """ + Get a more compact version of `mapping` which doesn't have + empty key/value pairs or attributes which are explicitly skipped + with the `keys_to_skip` list. + + Also removes empty key/value pairs recursively. + """ + if type(mapping) is not dict: + return mapping + + return { + key: get_compact_mapping(value) + for key, value in mapping.items() + if value and key not in keys_to_skip + } + + +get_compact_func_by_key = { + "packages": get_compact_packages, + "dependencies": get_compact_dependencies, + "license_detections": get_compact_license_detections, +} + +def write_compact_codebase_attributes(stream, key, value): + if not key in get_compact_func_by_key: + return + + func = get_compact_func_by_key.get(key) + if func: + compact_output = [ + func(instance) + for instance in value + ] + if compact_output: + return stream.write(key, compact_output) + + +def get_compact_resource(resource_mapping): + compact_resource_mapping = get_compact_mapping( + mapping=resource_mapping, + keys_to_skip=COMPACT_RESOURCES_SKIP_ATTRIBUTES, + ) + + # Only keep detection identifiers for resource level package/license + # detections as these are assembled into top-level summaries and so + # these inlined data is duplicate information + package_data = [ + {"purl": detection.get("purl")} + for detection in resource_mapping.get("package_data", []) + if detection and detection.get("purl") + ] + if package_data: + compact_resource_mapping["package_data"] = package_data + + license_detections = [ + {"identifier": detection.get("identifier")} + for detection in resource_mapping.get("license_detections", []) + if detection + ] + if license_detections: + compact_resource_mapping["license_detections"] = license_detections + + # license clues are matches not aggregated in the top-level license detections, but + # these mappings need to be processed to make them more compact + license_clues = [ + get_compact_mapping( + mapping=clue_mapping, + keys_to_skip=COMPACT_LICENSE_MATCHES_SKIP_ATTRIBUTES, + ) + for clue_mapping in resource_mapping.get("license_clues", []) + if clue_mapping + ] + if license_clues: + compact_resource_mapping["license_clues"] = license_clues + + return compact_resource_mapping diff --git a/src/formattedcode/output_json.py b/src/formattedcode/output_json.py index d3ce570b83d..3419e6b308c 100644 --- a/src/formattedcode/output_json.py +++ b/src/formattedcode/output_json.py @@ -9,6 +9,9 @@ import jsonstreams from formattedcode import FileOptionType +from formattedcode.output_compact import COMPACT_CONFLICTING_OPTIONS +from formattedcode.output_compact import write_compact_codebase_attributes +from formattedcode.output_compact import get_compact_resource from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import OUTPUT_GROUP from plugincode.output import output_impl @@ -40,14 +43,14 @@ def logger_debug(*args): @output_impl -class JsonCompactOutput(OutputPlugin): +class JsonOutput(OutputPlugin): options = [ PluggableCommandLineOption(('--json', 'output_json',), type=FileOptionType(mode='w', encoding='utf-8', lazy=True), metavar='FILE', default=None, - help='Write scan output as compact JSON to FILE.', + help='Write scan output as JSON to FILE.', help_group=OUTPUT_GROUP, sort_order=10), ] @@ -59,6 +62,27 @@ def process_codebase(self, codebase, output_json, **kwargs): write_results(codebase, output_file=output_json, pretty=False, **kwargs) +@output_impl +class JsonCompactOutput(OutputPlugin): + + options = [ + PluggableCommandLineOption(('--compact-json', 'output_compact_json',), + type=FileOptionType(mode='w', encoding='utf-8', lazy=True), + metavar='FILE', + default=None, + conflicting_options=COMPACT_CONFLICTING_OPTIONS, + help='Write scan output as compact JSON to FILE.', + help_group=OUTPUT_GROUP, + sort_order=10), + ] + + def is_enabled(self, output_compact_json, **kwargs): + return output_compact_json + + def process_codebase(self, codebase, output_compact_json, **kwargs): + write_results(codebase, output_file=output_compact_json, pretty=True, compact=True, **kwargs) + + @output_impl class JsonPrettyOutput(OutputPlugin): @@ -79,11 +103,12 @@ def process_codebase(self, codebase, output_json_pp, **kwargs): write_results(codebase, output_file=output_json_pp, pretty=True, **kwargs) -def write_results(codebase, output_file, pretty=False, **kwargs): +def write_results(codebase, output_file, pretty=False, compact=False, **kwargs): """ Write headers, files, and other attributes from `codebase` to `output_file` Enable JSON indentation if `pretty` is True + Only return useful, non-duplicated """ # Set indentation for JSON output if `pretty` is True # We use a separate dict for jsonstream kwargs since we are passing @@ -114,12 +139,22 @@ def write_results(codebase, output_file, pretty=False, **kwargs): # Write attributes if codebase.attributes: for attribute_key, attribute_value in codebase.attributes.to_dict().items(): - s.write(attribute_key, attribute_value) + if compact: + write_compact_codebase_attributes(stream=s, key=attribute_key, value=attribute_value) + else: + s.write(attribute_key, attribute_value) # Write files codebase_files = OutputPlugin.get_files(codebase, **kwargs) # OutputPlugin.get_files() returns a generator, not JSON-serializable - codebase_files = list(codebase_files) + if compact: + codebase_files = list( + get_compact_resource(codebase_file) + for codebase_file in codebase_files + ) + else: + codebase_files = list(codebase_files) + s.write('files', codebase_files) diff --git a/src/licensedcode/data/rules/fsl-1.1_1.RULE b/src/licensedcode/data/rules/fsl-1.1_1.RULE index 004881548a3..236f5c904e2 100644 --- a/src/licensedcode/data/rules/fsl-1.1_1.RULE +++ b/src/licensedcode/data/rules/fsl-1.1_1.RULE @@ -4,7 +4,9 @@ is_license_reference: yes is_continuous: yes relevance: 100 minimum_coverage: 100 -notes: Rule based on the manifest file of the liquidbase source jar. +notes: Rule based on the manifest file of the liquidbase source jar. +ignorable_urls: + - https://fsl.software/FSL-1.1-ALv2.template.md --- -Bundle-License: https://fsl.software/FSL-1.1-ALv2.template.md +Bundle-License: https://fsl.software/FSL-1.1-ALv2.template.md \ No newline at end of file diff --git a/src/licensedcode/data/rules/fsl-1.1_2.RULE b/src/licensedcode/data/rules/fsl-1.1_2.RULE index 2e4d72ff45c..e7290e76dbe 100644 --- a/src/licensedcode/data/rules/fsl-1.1_2.RULE +++ b/src/licensedcode/data/rules/fsl-1.1_2.RULE @@ -4,7 +4,9 @@ is_license_reference: yes is_continuous: yes relevance: 100 minimum_coverage: 100 -notes: Rule based on the manifest file of the liquidbase source jar. +notes: Rule based on the manifest file of the liquidbase source jar. +ignorable_urls: + - https://fsl.software/FSL-1.1-ALv2.template.md --- -https://fsl.software/FSL-1.1-ALv2.template.md +https://fsl.software/FSL-1.1-ALv2.template.md \ No newline at end of file diff --git a/tests/formattedcode/data/common/manifests-expected.compact.json b/tests/formattedcode/data/common/manifests-expected.compact.json new file mode 100644 index 00000000000..7d8bfbde3ee --- /dev/null +++ b/tests/formattedcode/data/common/manifests-expected.compact.json @@ -0,0 +1,856 @@ +{ + "packages": [ + { + "primary_language": "Java", + "homepage_url": "http://www.jcp.org/en/jsr/detail?id=220", + "declared_license_expression_spdx": "CDDL-1.0", + "source_packages": [ + "pkg:maven/javax.persistence/persistence-api@1.0?classifier=sources" + ], + "repository_homepage_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/", + "repository_download_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.jar", + "api_data_url": "https://repo1.maven.org/maven2/javax/persistence/persistence-api/1.0/persistence-api-1.0.pom", + "package_uid": "pkg:maven/javax.persistence/persistence-api@1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "manifests/maven/persistence-api-1.0.pom" + ], + "license_detections": [ + { + "identifier": "cddl_1_0-b17acf03-1e4f-20e6-cbb8-1b6945ee4c4c" + } + ] + }, + { + "primary_language": "JavaScript", + "parties": [ + { + "type": "person", + "role": "author", + "name": "Spencer Alger", + "email": "spencer.alger@elasticsearch.com", + "url": null + } + ], + "keywords": [ + "gruntplugin" + ], + "homepage_url": "https://github.com/spenceralger/grunt-esvm", + "download_url": "https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz", + "bug_tracking_url": "https://github.com/spenceralger/grunt-esvm/issues", + "vcs_url": "git://github.com/spenceralger/grunt-esvm.git", + "declared_license_expression_spdx": "Apache-2.0", + "repository_homepage_url": "https://www.npmjs.com/package/grunt-esvm", + "repository_download_url": "https://registry.npmjs.org/grunt-esvm/-/grunt-esvm-3.2.8.tgz", + "api_data_url": "https://registry.npmjs.org/grunt-esvm/3.2.8", + "package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "manifests/npm-license-mapping/package.json" + ], + "license_detections": [ + { + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + }, + { + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8" + } + ] + }, + { + "primary_language": "JavaScript", + "parties": [ + { + "type": "person", + "role": "maintainer", + "name": "georgdangl", + "email": "npm@dan.gl", + "url": null + } + ], + "homepage_url": "https://github.com/GeorgDangl/angular-compare-validator#readme", + "download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz", + "sha1": "d35a0754c8587b0502874e3636cf0f19565d09b7", + "sha512": "8f70ed5e3513185ad58fb2a3114769ac93ddd68836ce89146e586fc03e03ac93dcfb1f11354ac26f408b75c0ebf51663d5e4e8e27c38752a3c06bdde749a7c03", + "bug_tracking_url": "https://github.com/GeorgDangl/angular-compare-validator/issues", + "vcs_url": "git+https://github.com/GeorgDangl/angular-compare-validator.git", + "declared_license_expression_spdx": "MIT", + "repository_homepage_url": "https://www.npmjs.com/package/angular-compare-validator", + "repository_download_url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz", + "api_data_url": "https://registry.npmjs.org/angular-compare-validator/0.1.1", + "package_uid": "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "manifests/npm-license-string/package.json" + ], + "license_detections": [ + { + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ] + }, + { + "primary_language": "Python", + "parties": [ + { + "type": "person", + "role": "author", + "name": "BlueBrain Project, EPFL", + "email": "werner.vangeit@epfl.ch", + "url": null + } + ], + "keywords": [ + "optimisation", + "neuroscience", + "BlueBrainProject", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Programming Language :: Python :: 3 :: Only", + "Operating System :: POSIX", + "Topic :: Scientific/Engineering", + "Topic :: Utilities" + ], + "homepage_url": "https://github.com/BlueBrain/BluePyOpt", + "declared_license_expression_spdx": "LGPL-3.0-only", + "repository_homepage_url": "https://pypi.org/project/bluepyopt", + "api_data_url": "https://pypi.org/pypi/bluepyopt/json", + "package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "manifests/pypi/bluepyopt_setup.py" + ], + "license_detections": [ + { + "identifier": "lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5" + }, + { + "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321" + } + ] + } + ], + "dependencies": [ + { + "purl": "pkg:npm/bluebird", + "extracted_requirement": "^2.9.30", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/bluebird?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/cli-color", + "extracted_requirement": "~0.3.2", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/cli-color?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/cli-table", + "extracted_requirement": "~0.3.0", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/cli-table?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/libesvm", + "extracted_requirement": "^3.8.2", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/libesvm?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": "^3.9.3", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/lodash?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/moment", + "extracted_requirement": "^2.10.3", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/moment?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/wreck", + "extracted_requirement": "^5.6.0", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/wreck?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/grunt", + "extracted_requirement": ">=0.4.0", + "scope": "devDependencies", + "is_optional": true, + "is_direct": true, + "dependency_uid": "pkg:npm/grunt?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/load-grunt-config", + "extracted_requirement": "~0.19.0", + "scope": "devDependencies", + "is_optional": true, + "is_direct": true, + "dependency_uid": "pkg:npm/load-grunt-config?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/grunt", + "extracted_requirement": ">=0.4.0", + "scope": "peerDependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/grunt?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-mapping/package.json" + }, + { + "purl": "pkg:npm/%40angular/core", + "extracted_requirement": ">=2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/%40angular/core?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-string/package.json" + }, + { + "purl": "pkg:npm/%40angular/forms", + "extracted_requirement": ">=2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/%40angular/forms?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-string/package.json" + }, + { + "purl": "pkg:npm/%40angular/common", + "extracted_requirement": ">=2.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/%40angular/common?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-string/package.json" + }, + { + "purl": "pkg:npm/rxjs", + "extracted_requirement": ">=5.0.0-beta.12", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/rxjs?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-string/package.json" + }, + { + "purl": "pkg:npm/zone.js", + "extracted_requirement": ">=0.6.21", + "scope": "dependencies", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:npm/zone.js?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/npm-license-string/package.json" + }, + { + "purl": "pkg:pypi/numpy", + "extracted_requirement": ">=1.6", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/numpy?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/pandas", + "extracted_requirement": ">=0.18", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/pandas?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/deap", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/deap?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/efel", + "extracted_requirement": ">=2.13", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/efel?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/ipyparallel", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/ipyparallel?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/pickleshare", + "extracted_requirement": ">=0.7.3", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/pickleshare?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/jinja2", + "extracted_requirement": ">=2.8", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/jinja2?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/future", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/future?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/pebble", + "extracted_requirement": ">=4.3.10", + "scope": "install", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/pebble?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/scoop", + "extracted_requirement": ">=0.7", + "scope": "all", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/scoop?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + }, + { + "purl": "pkg:pypi/scoop", + "extracted_requirement": ">=0.7", + "scope": "scoop", + "is_runtime": true, + "is_direct": true, + "dependency_uid": "pkg:pypi/scoop?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "manifests/pypi/bluepyopt_setup.py" + } + ], + "license_detections": [ + { + "identifier": "apache_2_0-d66ab77d-a5cc-7104-e702-dc7df61fe9e8", + "license_expression_spdx": "Apache-2.0", + "detection_count": 2, + "reference_matches": [ + { + "license_expression_spdx": "Apache-2.0", + "from_file": "manifests/npm-license-mapping/package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_identifier": "spdx_license_id_apache-2.0_for_apache-2.0.RULE" + } + ] + }, + { + "identifier": "apache_2_0-0d7a2023-aae9-2989-7f00-27b713b809bb", + "license_expression_spdx": "Apache-2.0", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "Apache-2.0", + "from_file": "manifests/npm-license-mapping/package.json", + "start_line": 18, + "end_line": 20, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_identifier": "apache-2.0_required_phrase_7.RULE" + } + ] + }, + { + "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0", + "license_expression_spdx": "Apache-2.0", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "Apache-2.0", + "from_file": "manifests/npm-license-mapping/package.json", + "start_line": 6, + "end_line": 6, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 4, + "match_coverage": 100.0, + "rule_identifier": "apache-2.0_65.RULE" + } + ] + }, + { + "identifier": "cddl_1_0-0507fc0c-7fd5-3ecd-8c7b-5ea542059d4f", + "license_expression_spdx": "CDDL-1.0", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "CDDL-1.0", + "from_file": "manifests/maven/persistence-api-1.0.pom", + "start_line": 17, + "end_line": 20, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 21, + "match_coverage": 100.0, + "rule_identifier": "cddl-1.0_95.RULE" + } + ] + }, + { + "identifier": "cddl_1_0-b17acf03-1e4f-20e6-cbb8-1b6945ee4c4c", + "license_expression_spdx": "CDDL-1.0", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "CDDL-1.0", + "from_file": "manifests/maven/persistence-api-1.0.pom", + "start_line": 1, + "end_line": 2, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 17, + "match_coverage": 100.0, + "rule_identifier": "cddl-1.0_98.RULE" + } + ] + }, + { + "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321", + "license_expression_spdx": "LGPL-3.0-only", + "detection_count": 2, + "reference_matches": [ + { + "license_expression_spdx": "LGPL-3.0-only", + "from_file": "manifests/pypi/bluepyopt_setup.py", + "start_line": 74, + "end_line": 75, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 10, + "match_coverage": 100.0, + "rule_identifier": "pypi_gnu_lesser_general_public_license_v3.RULE" + } + ] + }, + { + "identifier": "lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0", + "license_expression_spdx": "LGPL-3.0-only", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "LGPL-3.0-only", + "from_file": "manifests/pypi/bluepyopt_setup.py", + "start_line": 9, + "end_line": 20, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 106, + "match_coverage": 100.0, + "rule_identifier": "lgpl-3.0_276.RULE" + } + ] + }, + { + "identifier": "lgpl_3_0-2db87bcf-56b4-9d7d-7075-2effae31c631", + "license_expression_spdx": "LGPL-3.0-only", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "LGPL-3.0-only", + "from_file": "manifests/pypi/bluepyopt_setup.py", + "start_line": 65, + "end_line": 65, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_identifier": "lgpl-3.0_152.RULE" + } + ] + }, + { + "identifier": "lgpl_3_0-38174920-e8ed-7bda-41ec-94df7380b7d5", + "license_expression_spdx": "LGPL-3.0-only", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "LGPL-3.0-only", + "from_file": "manifests/pypi/bluepyopt_setup.py", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_identifier": "lgpl-3.0_29.RULE" + } + ] + }, + { + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "MIT", + "from_file": "manifests/npm-license-string/package.json", + "start_line": 4, + "end_line": 4, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_identifier": "mit_30.RULE" + } + ] + }, + { + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression_spdx": "MIT", + "from_file": "manifests/npm-license-string/package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06" + } + ] + } + ], + "files": [ + { + "path": "manifests", + "type": "directory", + "files_count": 4, + "dirs_count": 4, + "size_count": 6962 + }, + { + "path": "manifests/maven", + "type": "directory", + "for_packages": [ + "pkg:maven/javax.persistence/persistence-api@1.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "files_count": 1, + "size_count": 1307 + }, + { + "path": "manifests/maven/persistence-api-1.0.pom", + "type": "file", + "size": 1307, + "sha1": "82107f6781df8da9aa7cb60abc649208256af5cb", + "md5": "e79938fe4e630b02e3526c0195292cc6", + "sha256": "f94648bd92ddf2464e62c56c3833b36927fa8b9803e8ebf3e4cfa7f2d797f3ad", + "sha1_git": "614191600be349e61331a3a0eb2b883493f0db0f", + "mime_type": "text/xml", + "file_type": "XML 1.0 document, ASCII text", + "is_text": true, + "for_packages": [ + "pkg:maven/javax.persistence/persistence-api@1.0?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "detected_license_expression_spdx": "CDDL-1.0", + "percentage_of_license_text": 12.14, + "urls": [ + { + "url": "http://www.jcp.org/en/jsr/detail?id=220", + "start_line": 15, + "end_line": 15 + }, + { + "url": "http://www.sun.com/cddl/cddl.html", + "start_line": 20, + "end_line": 20 + } + ], + "package_data": [ + { + "purl": "pkg:maven/javax.persistence/persistence-api@1.0" + } + ], + "license_detections": [ + { + "identifier": "cddl_1_0-0507fc0c-7fd5-3ecd-8c7b-5ea542059d4f" + } + ] + }, + { + "path": "manifests/npm-license-mapping", + "type": "directory", + "files_count": 1, + "size_count": 1184 + }, + { + "path": "manifests/npm-license-mapping/package.json", + "type": "file", + "size": 1184, + "sha1": "0ed61bf6912407a143ce96c95bea56ad13463f5b", + "md5": "427e6d3b579b128fed50328ac65b698e", + "sha256": "d292aa647dc82e007ba319c878ec182d6f235f0a923cfe012ffc0ae8b7fa23be", + "sha1_git": "fee016dc24204e079cb9e0217d4439d73e1398ac", + "mime_type": "application/json", + "file_type": "JSON data", + "is_text": true, + "for_packages": [ + "pkg:npm/grunt-esvm@3.2.8?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "detected_license_expression_spdx": "Apache-2.0", + "percentage_of_license_text": 7.09, + "authors": [ + { + "author": "Spencer Alger", + "start_line": 7, + "end_line": 8 + } + ], + "emails": [ + { + "email": "spencer.alger@elasticsearch.com", + "start_line": 9, + "end_line": 9 + } + ], + "urls": [ + { + "url": "https://github.com/spenceralger/grunt-esvm", + "start_line": 5, + "end_line": 5 + }, + { + "url": "git://github.com/spenceralger/grunt-esvm.git", + "start_line": 13, + "end_line": 13 + }, + { + "url": "https://github.com/spenceralger/grunt-esvm/issues", + "start_line": 16, + "end_line": 16 + }, + { + "url": "https://github.com/spenceralger/grunt-esvm/blob/master/LICENSE.md", + "start_line": 21, + "end_line": 21 + } + ], + "package_data": [ + { + "purl": "pkg:npm/grunt-esvm@3.2.8" + } + ], + "license_detections": [ + { + "identifier": "apache_2_0-ec759ae0-ea5a-f138-793e-388520e080c0" + }, + { + "identifier": "apache_2_0-0d7a2023-aae9-2989-7f00-27b713b809bb" + } + ] + }, + { + "path": "manifests/npm-license-string", + "type": "directory", + "files_count": 1, + "size_count": 1544 + }, + { + "path": "manifests/npm-license-string/package.json", + "type": "file", + "size": 1544, + "sha1": "e6acddf907b7089d197d9d46879cd18ffb8f5198", + "md5": "5ba816691af915c02567f459ae26ebd6", + "sha256": "11129bc3bbf44b2567307ad13a31bc231f08416919e36bbdeabb9195b3e6ad58", + "sha1_git": "99128a9e08f5a23c650d83af5dcf40c3b976753d", + "mime_type": "application/json", + "file_type": "JSON data", + "is_text": true, + "for_packages": [ + "pkg:npm/angular-compare-validator@0.1.1?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "detected_license_expression_spdx": "MIT", + "percentage_of_license_text": 1.32, + "emails": [ + { + "email": "npm@dan.gl", + "start_line": 28, + "end_line": 28 + } + ], + "urls": [ + { + "url": "https://github.com/GeorgDangl/angular-compare-validator.git", + "start_line": 7, + "end_line": 7 + }, + { + "url": "https://github.com/GeorgDangl/angular-compare-validator/issues", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/GeorgDangl/angular-compare-validator#readme", + "start_line": 22, + "end_line": 22 + }, + { + "url": "https://registry.npmjs.org/angular-compare-validator/-/angular-compare-validator-0.1.1.tgz", + "start_line": 33, + "end_line": 33 + } + ], + "package_data": [ + { + "purl": "pkg:npm/angular-compare-validator@0.1.1" + } + ], + "license_detections": [ + { + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee" + } + ] + }, + { + "path": "manifests/pypi", + "type": "directory", + "files_count": 1, + "size_count": 2927 + }, + { + "path": "manifests/pypi/bluepyopt_setup.py", + "type": "file", + "size": 2927, + "sha1": "5507192502255517e006c531d07dd84dc2325be8", + "md5": "6795bed0c9c6b6cb915bfa959a2d1d4a", + "sha256": "4378d297df9fcb4f65fdcd1299e17eef7db86883baf149bc41b0335f943cf006", + "sha1_git": "887bd3bfe3bdba0c8f737a69df38454b3d489ac1", + "mime_type": "text/x-script.python", + "file_type": "Python script, ASCII text executable", + "programming_language": "Python", + "is_text": true, + "is_source": true, + "is_script": true, + "for_packages": [ + "pkg:pypi/bluepyopt?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "detected_license_expression_spdx": "LGPL-3.0-only", + "percentage_of_license_text": 34.71, + "copyrights": [ + { + "copyright": "Copyright (c) 2016-2020, EPFL/Blue Brain Project", + "start_line": 5, + "end_line": 5 + } + ], + "holders": [ + { + "holder": "EPFL/Blue Brain Project", + "start_line": 5, + "end_line": 5 + } + ], + "authors": [ + { + "author": "BlueBrain Project", + "start_line": 54, + "end_line": 54 + } + ], + "emails": [ + { + "email": "werner.vangeit@epfl.ch", + "start_line": 55, + "end_line": 55 + } + ], + "urls": [ + { + "url": "https://github.com/BlueBrain/BluePyOpt", + "start_line": 7, + "end_line": 7 + } + ], + "package_data": [ + { + "purl": "pkg:pypi/bluepyopt" + } + ], + "license_detections": [ + { + "identifier": "lgpl_3_0-121be3c2-9c80-df84-d3da-8f674e4125c0" + }, + { + "identifier": "lgpl_3_0-2db87bcf-56b4-9d7d-7075-2effae31c631" + }, + { + "identifier": "lgpl_3_0-272571eb-5e68-95b6-ddb0-71de2d8df321" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/formattedcode/test_output_json.py b/tests/formattedcode/test_output_json.py index b9d77046ffa..6c34d026065 100644 --- a/tests/formattedcode/test_output_json.py +++ b/tests/formattedcode/test_output_json.py @@ -33,7 +33,7 @@ def test_json_pretty_print(): check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES) -def test_json_compact(): +def test_json_output(): test_dir = test_env.get_test_loc('json/simple') result_file = test_env.get_temp_file('json') run_scan_click(['-clip', test_dir, '--json', result_file]) @@ -81,3 +81,11 @@ def test_scan_output_for_timestamp(): header = result_json['headers'][0] assert 'start_timestamp' in header assert 'end_timestamp' in header + + +def test_compact_json_with_all_scan_options(): + test_dir = test_env.get_test_loc('common/manifests') + result_file = test_env.get_temp_file('json') + run_scan_click(['-clipeu', test_dir, '--compact-json', result_file]) + expected = test_env.get_test_loc('common/manifests-expected.compact.json') + check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES) diff --git a/tests/formattedcode/test_reuse_output_plugins.py b/tests/formattedcode/test_reuse_output_plugins.py index 90236ae0450..a551105f86b 100644 --- a/tests/formattedcode/test_reuse_output_plugins.py +++ b/tests/formattedcode/test_reuse_output_plugins.py @@ -46,6 +46,11 @@ def check_plugin(plugin_class, test_file='reuse/vb.json', force_text=False): assert 'zlib' in inp.read() +def test_can_call_json_output_from_regular_code_with_virtualcodebase(): + from formattedcode.output_json import JsonOutput as plug + check_plugin(plug, 'reuse/vb.json') + + def test_can_call_json_output_from_regular_code_with_virtualcodebase(): from formattedcode.output_json import JsonCompactOutput as plug check_plugin(plug, 'reuse/vb.json') diff --git a/tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.py b/tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.py index 48f5e5f65f2..58ddbaece3d 100644 --- a/tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.py +++ b/tests/licensedcode/data/additional_licenses/additional_plugin_1/setup.py @@ -38,7 +38,7 @@ 'open source', 'scancode_licenses', ], install_requires=[ - 'scancode-toolkit', + #'scancode-toolkit', ], entry_points={ 'scancode_additional_license_location_provider': [ diff --git a/tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.py b/tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.py index b166e32c436..9b2ba931803 100644 --- a/tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.py +++ b/tests/licensedcode/data/additional_licenses/additional_plugin_2/setup.py @@ -35,7 +35,7 @@ 'open source', 'scancode_licenses', ], install_requires=[ - 'scancode-toolkit', + #'scancode-toolkit', ], entry_points={ 'scancode_additional_license_location_provider': [ diff --git a/tests/scancode/data/help/help.txt b/tests/scancode/data/help/help.txt index 39c233615e3..0539823cdc9 100644 --- a/tests/scancode/data/help/help.txt +++ b/tests/scancode/data/help/help.txt @@ -48,7 +48,8 @@ Options: --unknown-licenses [EXPERIMENTAL] Detect unknown licenses. output formats: - --json FILE Write scan output as compact JSON to FILE. + --compact-json FILE Write scan output as compact JSON to FILE. + --json FILE Write scan output as JSON to FILE. --json-pp FILE Write scan output as pretty-printed JSON to FILE. --json-lines FILE Write scan output as JSON Lines to FILE. --yaml FILE Write scan output as YAML to FILE. diff --git a/tests/scancode/data/help/help_linux.txt b/tests/scancode/data/help/help_linux.txt index f976d90fc52..b0e462bad98 100644 --- a/tests/scancode/data/help/help_linux.txt +++ b/tests/scancode/data/help/help_linux.txt @@ -50,7 +50,8 @@ Options: --unknown-licenses [EXPERIMENTAL] Detect unknown licenses. output formats: - --json FILE Write scan output as compact JSON to FILE. + --compact-json FILE Write scan output as compact JSON to FILE. + --json FILE Write scan output as JSON to FILE. --json-pp FILE Write scan output as pretty-printed JSON to FILE. --json-lines FILE Write scan output as JSON Lines to FILE. --yaml FILE Write scan output as YAML to FILE.