Support equivalent words in license detection #4190

Handle similar words in license detection by allowing multiple
"legalese words" to have the same token id.

Regenerate the tokens ids accordingly.

Convert Index.tokens_by_tid to a computed property, available on demand.
Convert tokens_by_tid to a dictionary from a list.
Ensure that all code relying on the tokens_by_tid is updated as needed.
All locations were used only for testing and debugging.

Deprecate all rules that are duplicated under this new regime, where
tokens like "license" and "licence" are not treated as identical.

Update test suite to test the detection of all deprecated licenses and
rules as a sanity check. A rule with "relevance" set to 0 is not tested
if deprecated, as some rules are deprecated because they are false
positive and should no longer be detected. Also improved the validation
and loading of rules relevance, including the case for zero relevance.

Update ambiguous or conflicting rules as needed.
In particular ensure that all rules in the style of "MIT or GPL"
without a GPL version are now reported consistently as:
"mit or gpl-1.0-plus"

Add new rules as needed to resolve failing tests and improve accuracy.

Improve deprecated support for rules and licenses, adding a new
"replaced_by" list attribute that lists the new expressions that must be
detected from scanning the deprecated license or rule text.

Reference: #4190
Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

etc/scripts/licenses/buildrules.py

@@ -149,7 +149,7 @@ def rule_exists(text):
 
 def all_rule_by_tokens():
     """
-    Return a mapping of {tuples of tokens: rule id}, with one item for each
+    Return a mapping of {(tuple of token id): rule id}, with one item for each
     existing and added rules. Used to avoid duplicates.
     """
     rule_tokens = {}
@@ -159,7 +159,7 @@ def all_rule_by_tokens():
         except Exception as e:
             rf = f"  file://{rule.rule_file()}"
             raise Exception(
-                f"Failed to to get tokens from rule:: {rule.identifier}\n" f"{rf}"
+                f"Failed to get tokens from rule:: {rule.identifier}\n" f"{rf}"
             ) from e
     return rule_tokens
 

src/formattedcode/output_cyclonedx.py

@@ -20,9 +20,9 @@
 from typing import List
 
 import attr
+from lxml import etree
 from commoncode.cliutils import OUTPUT_GROUP
 from commoncode.cliutils import PluggableCommandLineOption
-from lxml import etree
 from plugincode.output import OutputPlugin
 from plugincode.output import output_impl
 

src/licensedcode/cache.py

@@ -276,7 +276,7 @@ def build_licensing(licenses_db=None):
     from licensedcode.models import load_licenses
 
     licenses_db = licenses_db or load_licenses()
-    return Licensing((LicenseSymbolLike(lic) for lic in licenses_db.values()))
+    return Licensing(symbols=(LicenseSymbolLike(lic) for lic in licenses_db.values()))
 
 
 def build_spdx_symbols(licenses_db=None):
@@ -316,7 +316,6 @@ def get_licenses_by_spdx_key(
 
     Optionally include deprecated if ``include_deprecated`` is True.
 
-
     Optionally make the keys lowercase if ``lowercase_keys`` is True.
 
     Optionally include the license "other_spdx_license_keys" if present and

src/licensedcode/data/licenses/agpl-3.0-bacula.LICENSE

@@ -1,6 +1,14 @@
 ---
 key: agpl-3.0-bacula
 is_deprecated: yes
+replaced_by:
+    - bacula-exception
+    - bsd-simplified
+    - bsd-simplified
+    - bsd-simplified
+    - agpl-3.0-plus
+    - agpl-3.0-plus
+    - agpl-3.0
 short_name: AGPL 3.0 with Bacula exception
 name: AGPL 3.0 with Bacula exception
 category: Copyleft

src/licensedcode/data/licenses/agpl-3.0-linking-exception.LICENSE

@@ -1,13 +1,15 @@
 ---
 key: agpl-3.0-linking-exception
+is_deprecated: yes
+replaced_by:
+    - linking-exception-agpl-3.0
 short_name: AGPL 3.0 linking exception
 name: AGPL 3.0 linking exception
 category: Copyleft Limited
 owner: Unspecified
-is_exception: yes
 homepage_url: http://mo.morsi.org/blog/2009/08/13/lesser_affero_gplv3/
 notes: renamed to linking-exception-agpl-3.0
-is_deprecated: yes
+is_exception: yes
 ---
 
 Additional permission under the GNU Affero GPL version 3 section 7:

src/licensedcode/data/licenses/agpl-3.0-openssl.LICENSE

@@ -1,15 +1,16 @@
 ---
 key: agpl-3.0-openssl
+is_deprecated: yes
+replaced_by:
+    - openssl-exception-agpl-3.0
 short_name: AGPL 3.0 with OpenSSL exception
 name: AGPL 3.0 with OpenSSL exception
 category: Copyleft
 owner: MongoDB
-is_exception: yes
-is_deprecated: yes
 notes: replaced by openssl-exception-agpl-3.0
+is_exception: yes
 ---
 
-
 As a special exception, the copyright holders give permission to link the
 code of portions of this program with the OpenSSL library under certain
 conditions as described in each individual source file and distribute

src/licensedcode/data/licenses/aladdin-md5.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: aladdin-md5
 is_deprecated: yes
+replaced_by:
+    - zlib
 short_name: Aladdin MD5 License
 name: Aladdin MD5 License
 category: Permissive

src/licensedcode/data/licenses/aop-pd.LICENSE

@@ -1,8 +1,10 @@
 ---
 key: aop-pd
+is_deprecated: yes
+replaced_by:
+    - cc-pd
 short_name: AOP-PD
 name: AOP Public Domain License
-is_deprecated: yes
 category: Public Domain
 owner: AOP Alliance Project
 ---

src/licensedcode/data/licenses/apache-2.0-linking-exception.LICENSE

@@ -1,12 +1,14 @@
 ---
 key: apache-2.0-linking-exception
+is_deprecated: yes
+replaced_by:
+    - compuphase-linking-exception
 short_name: Apache 2.0 with Linking Exception
 name: Apache 2.0 with Linking Exception
 category: Permissive
 owner: compuphase
 homepage_url: https://github.com/compuphase/minIni/blob/master/LICENSE
 is_exception: yes
-is_deprecated: yes
 ---
 
 EXCEPTION TO THE APACHE 2.0 LICENSE

src/licensedcode/data/licenses/apache-2.0-runtime-library-exception.LICENSE

@@ -1,5 +1,8 @@
 ---
 key: apache-2.0-runtime-library-exception
+is_deprecated: yes
+replaced_by:
+    - apple-runtime-library-exception
 short_name: Apache 2.0 with Runtime Library Exception
 name: Apache 2.0 with Runtime Library Exception
 category: Permissive
@@ -8,7 +11,6 @@ homepage_url: https://github.com/apple/swift/blob/master/LICENSE.txt#L205
 is_exception: yes
 other_urls:
     - https://swift.org/
-is_deprecated: yes
 ---
 
 ## Runtime Library Exception to the Apache 2.0 License: ##

src/licensedcode/data/licenses/apache-due-credit.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: apache-due-credit
 is_deprecated: yes
+replaced_by:
+    - dom4j
 short_name: Apache Due Credit Variant
 name: Apache Due Credit Variant
 category: Permissive

src/licensedcode/data/licenses/apache-exception-llvm.LICENSE

@@ -1,13 +1,15 @@
 ---
 key: apache-exception-llvm
+is_deprecated: yes
+replaced_by:
+    - llvm-exception
 short_name: Apache-Exception-llvm
 name: Apache Exception LLVM
 category: Permissive
 owner: Apache Software Foundation
 homepage_url: https://lists.spdx.org
-is_exception: yes
-is_deprecated: yes
 notes: Replaced by llvm-exception
+is_exception: yes
 text_urls:
     - https://lists.spdx.org/pipermail/spdx-legal/2017-December/002421.html
 ---

src/licensedcode/data/licenses/apache-patent-provision-exception.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: apache-patent-provision-exception
 is_deprecated: yes
+replaced_by:
+    - apache-patent-exception
 short_name: Apache Patent Provision Exception Deprecated
 name: Apache Patent Provision Exception Deprecated
 category: Permissive

src/licensedcode/data/licenses/baekmuk-fonts.LICENSE

@@ -13,6 +13,7 @@ ignorable_copyrights:
     - Copyright (c) Kim Jeong-Hwan
 ignorable_holders:
     - Kim Jeong-Hwan
+minimum_coverage: 80
 ---
 
 Baekmuk Fonts License
@@ -26,4 +27,4 @@ derivative works or modified versions, and that the following
 acknowledgement appear in supporting documentation:
     Baekmuk Batang, Baekmuk Dotum, Baekmuk Gulim, and
     Baekmuk Headline are registered trademarks owned by
-    Kim Jeong-Hwan.
+    Kim Jeong-Hwan.

src/licensedcode/data/licenses/broadcom-dual.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: broadcom-dual
 is_deprecated: yes
+replaced_by:
+    - gpl-2.0 OR commercial-license
 short_name: Broadcom Dual GPL-Commercial
 name: Broadcom Dual GPL-Commercial
 category: Copyleft

src/licensedcode/data/licenses/broadcom-linking-unmodified.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: broadcom-linking-unmodified
 is_deprecated: yes
+replaced_by:
+    - broadcom-unmodified-exception
 short_name: Broadcom Linking Exception if unmodified
 name: Broadcom Linking Exception if unmodified
 category: Copyleft Limited

src/licensedcode/data/licenses/broadcom-unpublished-source.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: broadcom-unpublished-source
 is_deprecated: yes
+replaced_by:
+    - unpublished-source
 short_name: Broadcom Unpublished Source License
 name: Broadcom Unpublished Source License
 category: Commercial

src/licensedcode/data/licenses/bsd-2-clause-freebsd.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: bsd-2-clause-freebsd
 is_deprecated: yes
+replaced_by:
+    - bsd-2-clause-views
 short_name: BSD-2-Clause-FreeBSD
 name: BSD-2-Clause-FreeBSD License
 category: Permissive

src/licensedcode/data/licenses/bsd-2-clause-netbsd.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: bsd-2-clause-netbsd
 is_deprecated: yes
+replaced_by:
+    - bsd-simplified
 short_name: BSD-2-Clause-NetBSD
 name: BSD-2-Clause-NetBSD License
 category: Permissive

src/licensedcode/data/licenses/bsd-axis.LICENSE

@@ -1,12 +1,14 @@
 ---
 key: bsd-axis
+is_deprecated: yes
+replaced_by:
+    - bsd-source-code
 short_name: BSD-Axis
 name: BSD-Axis
 category: Permissive
 owner: Axis Communications
 notes: This is a variant composed of clause 1 and 3 of a BSD-Modified found in the Linux kernel
     This is now replaced by the bsd-source-code license.
-is_deprecated: yes
 ---
 
 Redistribution and use in source and binary forms, with or without

src/licensedcode/data/licenses/bsd-intel.LICENSE

@@ -1,10 +1,12 @@
 ---
 key: bsd-intel
+is_deprecated: yes
+replaced_by:
+    - bsd-new
 short_name: BSD Intel License
 name: BSD Intel License
 category: Permissive
 owner: Intel Corporation
-is_deprecated: yes
 ---
 
 Redistribution and use in source and binary forms, with or without modification,

src/licensedcode/data/licenses/bsd-new-far-manager.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: bsd-new-far-manager
 is_deprecated: yes
+replaced_by:
+    - bsd-new WITH far-manager-exception
 short_name: BSD-3-Clause with Far Manager exception
 name: BSD-3-Clause with Far Manager exception
 category: Permissive

src/licensedcode/data/licenses/bsd-original-uc-1990.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: bsd-original-uc-1990
 is_deprecated: yes
+replaced_by:
+    - bsla
 short_name: BSD-Original-UC-1990
 name: BSD-Original-UC-1990
 category: Permissive

src/licensedcode/data/licenses/bzip2-libbzip-1.0.5.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: bzip2-libbzip-1.0.5
 is_deprecated: yes
+replaced_by:
+    - bzip2-libbzip-2010
 short_name: bzip2 License
 name: bzip2 License
 category: Permissive

src/licensedcode/data/licenses/ccrc-1.0.LICENSE

@@ -1,19 +1,21 @@
 ---
 key: ccrc-1.0
+is_deprecated: yes
+replaced_by:
+    - gplcc-1.0
 short_name: Common Cure Rights Commitment v1.0
 name: Common Cure Rights Commitment v1.0
 category: Copyleft
 owner: Red Hat, Inc.
 homepage_url: https://www.redhat.com/en/about/press-releases/technology-industry-leaders-join-forces-increase-predictability-open-source-licensing
+notes: the text of the license itself is under the CC-BY-SA-4.0 license. And this license has
+    been renamed to gplcc-1.0
 text_urls:
     - http://git.gluster.org/cgit/glusterfs.git/tree/COMMITMENT
     - https://raw.githubusercontent.com/wildfly/wildfly/master/COMMITMENT
 other_urls:
     - https://www.redhat.com/en/about/press-releases/technology-industry-leaders-join-forces-increase-predictability-open-source-licensing
     - https://www.fsf.org/blogs/licensing/red-hat-leads-coalition-supporting-key-part-of-principles-of-community-oriented-gpl-enforcement
-notes: the text of the license itself is under the CC-BY-SA-4.0 license. And this license has
-    been renamed to gplcc-1.0
-is_deprecated: yes
 ---
 
 Common Cure Rights Commitment

src/licensedcode/data/licenses/classworlds.LICENSE

@@ -1,6 +1,8 @@
 ---
 key: classworlds
 is_deprecated: yes
+replaced_by:
+    - dom4j
 short_name: Classworlds License
 name: Classworlds License
 category: Permissive

src/licensedcode/data/licenses/cmr-no.LICENSE

@@ -1,10 +1,12 @@
 ---
 key: cmr-no
+is_deprecated: yes
+replaced_by:
+    - mit-old-style
 short_name: CMR License
 name: Christian Michelsen Research AS License
 category: Permissive
 owner: CMR - Christian Michelsen Research AS
-is_deprecated: yes
 notes: replaced by mit-old-style
 ---
 

src/licensedcode/data/licenses/commercial-option.LICENSE

@@ -1,12 +1,14 @@
 ---
 key: commercial-option
 is_deprecated: yes
+replaced_by:
+    - commercial-license
 short_name: Commercial Option
 name: Commercial Option
 category: Commercial
 owner: Unspecified
-is_generic: yes
 notes: replaced by commercial-license
+is_generic: yes
 ---
 
 This component may be licensed under a commercial contract from the supplier.

src/licensedcode/data/licenses/dejavu-font.LICENSE

@@ -1,6 +1,11 @@
 ---
 key: dejavu-font
 is_deprecated: yes
+replaced_by:
+    - bitstream AND public-domain
+    - bitstream
+    - bitstream
+    - bitstream
 short_name: DejaVu Font License
 name: DejaVu Font License
 category: Permissive

src/licensedcode/data/licenses/digia-qt-exception-lgpl-2.1.LICENSE

@@ -1,10 +1,12 @@
 ---
 key: digia-qt-exception-lgpl-2.1
+is_deprecated: yes
+replaced_by:
+    - qt-lgpl-exception-1.1
 short_name: Digia Qt Exception to LGPL 2.1
 name: Digia Qt Exception to LGPL 2.1
 category: Copyleft Limited
 owner: Digia
-is_deprecated: yes
 is_exception: yes
 other_urls:
     - http://www.gnu.org/licenses/lgpl-2.1.txt