Rollup merge of rust-lang#133453 - ferrocene:check-license-metadata, r=Kobzol

matthiaskrgr · web-flow · commit 04d633366d16 · 2024-11-27T08:13:48.000+01:00
Commit license-metadata.json to git and check it's correct in CI

This PR adds `license-metadata.json` to the root of the git repo, and changes `mingw-check` to check that the file is still up-to-date.

By committing this file, we remove the need for developers to a) have reuse installed or b) run an expensive ~90 second analysis of the files on disk when they want generate the COPYRIGHT.html files which depend on this license metadata.

The file will need updating whenever `REUSE.toml` changes, or when git submodules are added, or when git submodules change their license information (as detected by REUSE).

You can now run:

* `./x run collect-license-metadata` to update the `./license-metadata.json` file
* `./x test collect-license-metadata` to test the `./license-metadata.json` file for correctness

The comparison is done with two `serde_json::Value` objects, so the map objects they contain should ignore differences in ordering.
diff --git a/REUSE.toml b/REUSE.toml
@@ -28,6 +28,7 @@ path = [
     "COPYRIGHT",
     "INSTALL.md",
     "LICENSE-APACHE",
+    "license-metadata.json",
     "LICENSE-MIT",
     "README.md",
     "RELEASES.md",
diff --git a/license-metadata.json b/license-metadata.json
@@ -0,0 +1,269 @@
+{
+  "files": {
+    "children": [
+      {
+        "children": [
+          {
+            "children": [
+              {
+                "children": [
+                  {
+                    "children": [
+                      {
+                        "directories": [],
+                        "files": [
+                          "analyzer-decls.h",
+                          "malloc-macro.h"
+                        ],
+                        "license": {
+                          "copyright": [
+                            "2000-2024 Free Software Foundation, Inc"
+                          ],
+                          "spdx": "GPL-2.0-only"
+                        },
+                        "type": "group"
+                      }
+                    ],
+                    "license": {
+                      "copyright": [
+                        "2007-2011 Atheros Communications Inc",
+                        "2011-2012,2017 Qualcomm Atheros, Inc",
+                        "2016-2017 Erik Stromdahl <erik.stromdahl@gmail.com>"
+                      ],
+                      "spdx": "ISC"
+                    },
+                    "name": "c-c++-common/analyzer",
+                    "type": "directory"
+                  }
+                ],
+                "license": {
+                  "copyright": [
+                    "2000-2024 Free Software Foundation, Inc"
+                  ],
+                  "spdx": "GPL-2.0-only"
+                },
+                "name": "gcc/testsuite",
+                "type": "directory"
+              },
+              {
+                "license": {
+                  "copyright": [
+                    "2000-2024 Free Software Foundation, Inc"
+                  ],
+                  "spdx": "GCC-exception-3.1"
+                },
+                "name": "libstdc++-v3/config/os/aix/os_defines.h",
+                "type": "file"
+              }
+            ],
+            "license": {
+              "copyright": [
+                "1997-2024 Free Software Foundation, Inc"
+              ],
+              "spdx": "GPL-3.0-or-later"
+            },
+            "name": "src/gcc",
+            "type": "directory"
+          },
+          {
+            "children": [
+              {
+                "license": {
+                  "copyright": [
+                    "The Rust Project Developers (see https://thanks.rust-lang.org)"
+                  ],
+                  "spdx": "Apache-2.0 OR MIT"
+                },
+                "name": "noscript.css",
+                "type": "file"
+              },
+              {
+                "license": {
+                  "copyright": [
+                    "Nicolas Gallagher and Jonathan Neal"
+                  ],
+                  "spdx": "MIT"
+                },
+                "name": "normalize.css",
+                "type": "file"
+              }
+            ],
+            "license": {
+              "copyright": [
+                "2016 Ike Ku, Jessica Stokes and Leon Guan",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "src/librustdoc/html/static/css",
+            "type": "directory"
+          },
+          {
+            "children": [
+              {
+                "license": {
+                  "copyright": [
+                    "The Rust Project Developers (see https://thanks.rust-lang.org)"
+                  ],
+                  "spdx": "Apache-2.0 OR MIT"
+                },
+                "name": "README.txt",
+                "type": "file"
+              },
+              {
+                "directories": [],
+                "files": [
+                  "FiraSans-LICENSE.txt",
+                  "FiraSans-Medium.woff2",
+                  "FiraSans-Regular.woff2"
+                ],
+                "license": {
+                  "copyright": [
+                    "2014, Mozilla Foundation",
+                    "2014, Telefonica S.A"
+                  ],
+                  "spdx": "OFL-1.1"
+                },
+                "type": "group"
+              },
+              {
+                "directories": [],
+                "files": [
+                  "NanumBarunGothic-LICENSE.txt",
+                  "NanumBarunGothic.ttf.woff2"
+                ],
+                "license": {
+                  "copyright": [
+                    "2010 NAVER Corporation"
+                  ],
+                  "spdx": "OFL-1.1"
+                },
+                "type": "group"
+              }
+            ],
+            "license": {
+              "copyright": [
+                "2010, 2012, 2014-2023, Adobe Systems Incorporated"
+              ],
+              "spdx": "OFL-1.1"
+            },
+            "name": "src/librustdoc/html/static/fonts",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "2003-2019 University of Illinois at Urbana-Champaign",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 WITH LLVM-exception AND (Apache-2.0 OR MIT)"
+            },
+            "name": "compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2014 Alex Crichton",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "library/backtrace",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "1991-2024 Unicode, Inc"
+              ],
+              "spdx": "Unicode-3.0"
+            },
+            "name": "library/core/src/unicode/unicode_data.rs",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2019 The Crossbeam Project Developers",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "library/std/src/sync/mpmc",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "2016 The Fuchsia Authors",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "BSD-2-Clause AND (Apache-2.0 OR MIT)"
+            },
+            "name": "library/std/src/sys/sync/mutex/fuchsia.rs",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "Rust on Embedded Devices Working Group",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR CC-BY-SA-4.0 OR MIT"
+            },
+            "name": "src/doc/embedded-book",
+            "type": "directory"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2014 Jorge Aparicio",
+                "The Rust Project Developers (see https://thanks.rust-lang.org)"
+              ],
+              "spdx": "Apache-2.0 OR MIT"
+            },
+            "name": "src/doc/rust-by-example",
+            "type": "directory"
+          },
+          {
+            "license": {
+              "copyright": [
+                "2014-2021 Knut Sveidqvist"
+              ],
+              "spdx": "MIT"
+            },
+            "name": "src/doc/rustc-dev-guide/mermaid.min.js",
+            "type": "file"
+          },
+          {
+            "children": [],
+            "license": {
+              "copyright": [
+                "2003-2019 University of Illinois at Urbana-Champaign",
+                "2003-2019 by the contributors listed in CREDITS.TXT (https://github.com/rust-lang/llvm-project/blob/7738295178045041669876bf32b0543ec8319a5c/llvm/CREDITS.TXT)",
+                "2010 Apple Inc"
+              ],
+              "spdx": "Apache-2.0 WITH LLVM-exception AND NCSA"
+            },
+            "name": "src/llvm-project",
+            "type": "directory"
+          }
+        ],
+        "license": {
+          "copyright": [
+            "The Rust Project Developers (see https://thanks.rust-lang.org)"
+          ],
+          "spdx": "Apache-2.0 OR MIT"
+        },
+        "name": ".",
+        "type": "directory"
+      }
+    ],
+    "type": "root"
+  }
+}
diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs
@@ -181,8 +181,7 @@ impl Step for CollectLicenseMetadata {
             panic!("REUSE is required to collect the license metadata");
         };
 
-        // Temporary location, it will be moved to src/etc once it's accurate.
-        let dest = builder.out.join("license-metadata.json");
+        let dest = builder.src.join("license-metadata.json");
 
         let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata);
         cmd.env("REUSE_EXE", reuse);
@@ -209,8 +208,7 @@ impl Step for GenerateCopyright {
     }
 
     fn run(self, builder: &Builder<'_>) -> Self::Output {
-        let license_metadata = builder.ensure(CollectLicenseMetadata);
-
+        let license_metadata = builder.src.join("license-metadata.json");
         let dest = builder.out.join("COPYRIGHT.html");
         let dest_libstd = builder.out.join("COPYRIGHT-library.html");
 
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
@@ -3670,3 +3670,35 @@ impl Step for TestFloatParse {
         cargo_run.into_cmd().run(builder);
     }
 }
+
+#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
+pub struct CollectLicenseMetadata;
+
+impl Step for CollectLicenseMetadata {
+    type Output = PathBuf;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/collect-license-metadata")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(CollectLicenseMetadata);
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let Some(reuse) = &builder.config.reuse else {
+            panic!("REUSE is required to collect the license metadata");
+        };
+
+        let dest = builder.src.join("license-metadata.json");
+
+        let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata);
+        cmd.env("REUSE_EXE", reuse);
+        cmd.env("DEST", &dest);
+        cmd.env("ONLY_CHECK", "1");
+        cmd.run(builder);
+
+        dest
+    }
+}
diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs
@@ -915,6 +915,7 @@ impl<'a> Builder<'a> {
                 test::HtmlCheck,
                 test::RustInstaller,
                 test::TestFloatParse,
+                test::CollectLicenseMetadata,
                 // Run bootstrap close to the end as it's unlikely to fail
                 test::Bootstrap,
                 // Run run-make last, since these won't pass without make on Windows
diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile
@@ -63,6 +63,7 @@ ENV SCRIPT \
            /scripts/validate-toolstate.sh && \
            /scripts/validate-error-codes.sh && \
            reuse --include-submodules lint && \
+           python3 ../x.py test collect-license-metadata && \
            # Runs checks to ensure that there are no issues in our JS code.
            es-check es2019 ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \
diff --git a/src/tools/collect-license-metadata/src/main.rs b/src/tools/collect-license-metadata/src/main.rs
diff --git a/src/tools/collect-license-metadata/src/reuse.rs b/src/tools/collect-license-metadata/src/reuse.rs
diff --git a/src/tools/generate-copyright/src/main.rs b/src/tools/generate-copyright/src/main.rs

-Original file line number
+Diff line change
 use std::path::PathBuf;
 -use anyhow::Error;
 +use anyhow::{Context, Error};
 use crate::licenses::LicensesInterner;
 /// The entry point to the binary.
 ///
 /// You should probably let `bootstrap` execute this program instead of running it directly.
 ///
 -/// Run `x.py run collect-license-metadata`
 +/// * Run `x.py run collect-license-metadata` to re-regenerate the file.
 +/// * Run `x.py test collect-license-metadata` to check if the file you have is correct.
 fn main() -> Result<(), Error> {
     let reuse_exe: PathBuf = std::env::var_os("REUSE_EXE").expect("Missing REUSE_EXE").into();
     let dest: PathBuf = std::env::var_os("DEST").expect("Missing DEST").into();
 +    let only_check = std::env::var_os("ONLY_CHECK").is_some();
     let mut interner = LicensesInterner::new();
     let paths = crate::reuse::collect(&reuse_exe, &mut interner)?;
     let mut tree = crate::path_tree::build(paths);
     tree.simplify();
 -    if let Some(parent) = dest.parent() {
 -        std::fs::create_dir_all(parent)?;
 +    let output = serde_json::json!({
 +        "files": crate::path_tree::expand_interned_licenses(tree, &interner)
 +    });
++
 +    if only_check {
 +        println!("loading existing license information");
 +        let existing = std::fs::read_to_string(&dest).with_context(|| {
 +            format!("Failed to read existing license JSON at {}", dest.display())
 +        })?;
 +        let existing_json: serde_json::Value =
 +            serde_json::from_str(&existing).with_context(|| {
 +                format!("Failed to read existing license JSON at {}", dest.display())
 +            })?;
 +        if existing_json != output {
 +            eprintln!("The existing {} file is out of date.", dest.display());
 +            eprintln!("Run ./x run collect-license-metadata to update it.");
 +            anyhow::bail!("The existing {} file doesn't match what REUSE reports.", dest.display());
 +        }
 +        println!("license information matches");
 +    } else {
 +        if let Some(parent) = dest.parent() {
 +            std::fs::create_dir_all(parent)?;
 +        }
 +        std::fs::write(&dest, &serde_json::to_vec_pretty(&output)?)?;
 +        println!("license information written to {}", dest.display());
+    }
 -    std::fs::write(
 -        &dest,
 -        &serde_json::to_vec_pretty(&serde_json::json!({
 -            "files": crate::path_tree::expand_interned_licenses(tree, &interner),
 -        }))?,
 -    )?;
     Ok(())
+}
-Original file line number
+Diff line change
     reuse_exe: &Path,
     interner: &mut LicensesInterner,
 ) -> Result<Vec<(PathBuf, LicenseId)>, Error> {
 -    eprintln!("gathering license information from REUSE");
 +    println!("gathering license information from REUSE (this might take a minute...)");
     let start = Instant::now();
     let raw = &obtain_spdx_document(reuse_exe)?;
 -    eprintln!("finished gathering the license information from REUSE in {:.2?}", start.elapsed());
 +    println!("finished gathering the license information from REUSE in {:.2?}", start.elapsed());
     let document = spdx_rs::parsers::spdx_from_tag_value(&raw)?;
-Original file line number
+Diff line change
         dependencies: collected_cargo_metadata,
     };
     let output = template.render()?;
 +    // Git stores text files with \n, but this file may contain \r\n in files
 +    // copied from dependencies. Normalise them before we write them out, for
 +    // consistency.
 +    let output = output.replace("\r\n", "\n");
     std::fs::write(&dest_file, output)?;
     // Output libstd subset file
         dependencies: library_collected_cargo_metadata,
     };
     let output = template.render()?;
 +    // Git stores text files with \n, but this file may contain \r\n in files
 +    // copied from dependencies. Normalise them before we write them out, for
 +    // consistency.
 +    let output = output.replace("\r\n", "\n");
     std::fs::write(&libstd_dest_file, output)?;
     Ok(())