rust: generate target specification files on the fly

Now that we have host programs support, add a "Rust script"
to generate the `rustc` target spec file.

The script has been written so that its output matches exactly
the current `arch/*/rust/*.json` static target files we have
at the moment, to have a baseline and reduce the changes done
by this commit. Thus the static files are now gone.

The build system is set up in a way that we do not need to rebuild
any Rust code as long as the `target.json` contents do not change,
even if the kernel configuration changes for something else.
Furthermore, the script does not need to be rebuilt even if
the kernel configuration changes.

With this, `KBUILD_RUST_TARGET` is also gone -- one should modify
the script instead of trying to override the target file.

After this, the script should be cleaned (e.g. remove unneeded
keys) and fixed to configure the architectures properly (e.g.
enable CPU features as needed, etc.).

There were several design choices:

  - The language: this could have been done in a shell script,
    C or Perl. But as soon as Rust is an option, it is more
    convenient than any of those. And it is an ideal showcase for
    the Rust host programs support.

    Python could have also been a natural option for the script,
    though it cannot be used as part as the kernel compilation
    because it is not required (as far as I can see in the docs).
    Even if it were, Rust offers some advantages anyway (though
    the `json` module would have been fairly useful here).

    Compiling the Rust host program takes a lot of time compared
    to other languages, at least currently (e.g. 300ms), but there
    is no need to recompile it on config changes, so it is fine.
    And, of course, running it is very fast.

  - Whether or not to use conditional compilation.

    Initially I took advantage of the `rustc_cfg` flags, to make
    the script look closer to the actual Rust kernel code.

    However, that means the script depends on the config, which
    means it is not an independent tool anymore (though there is
    at least one other host program that gets passed the flags,
    but it is not common), plus requires recompiling it every
    time the configuration changes just to generate the actual
    file. It also made things more tricky on the build system.

  - Splitting the logic into `arch/*` or not.

    In order to have arch maintainers work independently, we could
    move part of the logic into their trees.

    However, given how small the script is and that some logic
    is shared anyway, it looks a little bit odd. So I decided
    to avoid it. We can revisit this if needed.

  - Whether to use a proper JSON generator or not, whether to
    use a `HashMap`, whether to set up a type that knows about
    all the keys that can be generated (or maybe even import
    part of the `rustc` code to be in sync), etc.

    The script has been written to be as simple as possible
    while making it easy to change the actual business logic.
    We should reevaluate as it gets developed.

Signed-off-by: Miguel Ojeda <[email protected]>

Author: ojeda

Documentation/kbuild/makefiles.rst

@@ -1205,6 +1205,9 @@ When kbuild executes, the following steps are followed (roughly):
 
 	Often, the KBUILD_RUSTFLAGS variable depends on the configuration.
 
+	Note that target specification file generation (for ``--target``)
+	is handled in ``scripts/generate_rust_target.rs``.
+
     KBUILD_AFLAGS_KERNEL
 	Assembler options specific for built-in
 

Makefile

@@ -551,7 +551,6 @@ KBUILD_CFLAGS   := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \
 		   -Werror=return-type -Wno-format-security \
 		   -std=gnu89
 KBUILD_CPPFLAGS := -D__KERNEL__
-KBUILD_RUST_TARGET := $(srctree)/arch/$(SRCARCH)/rust/target.json
 KBUILD_RUSTFLAGS := $(rust_common_flags) \
 		    -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \
 		    -Cforce-unwind-tables=n -Ccodegen-units=1 \
@@ -592,7 +591,7 @@ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
 
 export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
 export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
-export KBUILD_RUST_TARGET KBUILD_RUSTFLAGS RUSTFLAGS_KERNEL RUSTFLAGS_MODULE
+export KBUILD_RUSTFLAGS RUSTFLAGS_KERNEL RUSTFLAGS_MODULE
 export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE
 export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_RUSTFLAGS_MODULE KBUILD_LDFLAGS_MODULE
 export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL KBUILD_RUSTFLAGS_KERNEL
@@ -1578,7 +1577,7 @@ MRPROPER_FILES += include/config include/generated          \
 		  certs/x509.genkey \
 		  vmlinux-gdb.py \
 		  *.spec \
-		  rust/libmacros.so
+		  rust/target.json rust/libmacros.so
 
 # clean - Delete most, but leave enough to build external modules
 #

arch/arm/rust/target.json

@@ -58,7 +58,6 @@ riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei
 
 KBUILD_CFLAGS += -march=$(subst fd,,$(riscv-march-y))
 KBUILD_AFLAGS += -march=$(riscv-march-y)
-KBUILD_RUST_TARGET := $(srctree)/arch/riscv/rust/$(subst fd,,$(riscv-march-y)).json
 
 KBUILD_CFLAGS += -mno-save-restore
 KBUILD_CFLAGS += -DCONFIG_PAGE_OFFSET=$(CONFIG_PAGE_OFFSET)

arch/arm64/rust/target.json

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
+target.json
 bindings_generated.rs
 bindings_helpers_generated.rs
 exports_*_generated.h

arch/powerpc/rust/target.json

@@ -1,5 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
+always-$(CONFIG_RUST) += target.json
+no-clean-files += target.json
+
 obj-$(CONFIG_RUST) += core.o compiler_builtins.o
 always-$(CONFIG_RUST) += exports_core_generated.h
 
@@ -206,6 +209,11 @@ rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttest-prepare \
 	$(call if_changed,rustc_test_library)
 	$(call if_changed,rustdoc_test)
 
+filechk_rust_target = $(objtree)/scripts/generate_rust_target < $<
+
+$(objtree)/rust/target.json: $(objtree)/include/config/auto.conf FORCE
+	$(call filechk,rust_target)
+
 ifdef CONFIG_CC_IS_CLANG
 bindgen_c_flags = $(c_flags)
 else
@@ -364,7 +372,8 @@ $(objtree)/rust/kernel.o: $(srctree)/rust/kernel/lib.rs $(objtree)/rust/alloc.o
 $(objtree)/rust/core.o: private skip_clippy = 1
 $(objtree)/rust/core.o: private skip_flags = -Dunreachable_pub --edition=2021
 $(objtree)/rust/core.o: private rustc_target_flags = $(core-cfgs) --edition=2018
-$(objtree)/rust/core.o: $(RUST_LIB_SRC)/core/src/lib.rs FORCE
+$(objtree)/rust/core.o: $(RUST_LIB_SRC)/core/src/lib.rs \
+    $(objtree)/rust/target.json FORCE
 	$(call if_changed_dep,rustc_library)
 
 rustdoc-core: private rustc_target_flags = $(core-cfgs)

arch/riscv/Makefile

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 /asn1_compiler
 /bin2c
+/generate_rust_target
 /insert-sys-cert
 /kallsyms
 /module.lds

arch/riscv/rust/rv32ima.json

@@ -10,6 +10,9 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT)		+= sorttable
 hostprogs-always-$(CONFIG_ASN1)				+= asn1_compiler
 hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT)		+= sign-file
 hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE)	+= insert-sys-cert
+hostprogs-always-$(CONFIG_RUST)				+= generate_rust_target
+
+generate_rust_target-rust := y
 
 HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include
 HOSTLDLIBS_sorttable = -lpthread

arch/riscv/rust/rv32imac.json

@@ -328,7 +328,7 @@ $(obj)/%.lst: $(src)/%.c FORCE
 # Compile Rust sources (.rs)
 # ---------------------------------------------------------------------------
 
-rust_cross_flags := --target=$(KBUILD_RUST_TARGET)
+rust_cross_flags := --target=$(objtree)/rust/target.json
 rust_allowed_features := allocator_api,bench_black_box,concat_idents,generic_associated_types
 
 rust_common_cmd = \