Skip to content

Commit 38a10b4

Browse files
ojedaojeda
committed
rust: compile alloc from the in-tree sources
Includes: - Cross-building a new sysroot with our custom `alloc` in order to support `alloc` tests. - Documentation generation for in-tree `alloc`. - Split `RUSTDOC` command in two for clarity, since now we have to also handle `sysroot` differently. - Proper handling of both silent and verbose mode for tests and `cargo`. - Deny missing docs globally `alloc`. - Allow broken intra docs for `alloc`. - Do not format `rust/alloc` nor `rust/test`, including handling the case where `srctree` is a prent folder. - `make clean` now cleans `rust/doc` and `rust/test`. - Move allocs out of the second expansion section since it is in-tree now. - Other small cleanups. I want to clean this up further and factor a few things, but let's move ahead. Signed-off-by: Miguel Ojeda <[email protected]>
1 parent 1f1b619 commit 38a10b4

File tree

2 files changed

+125
-37
lines changed

2 files changed

+125
-37
lines changed

‎Makefile

+20-9
Original file line numberDiff line numberDiff line change
@@ -461,9 +461,11 @@ STRIP = $(CROSS_COMPILE)strip
461461
endif
462462
RUSTC = rustc
463463
RUSTC_BOOTSTRAP = 1
464+
RUSTDOC = rustdoc
464465
RUSTFMT = rustfmt
465466
CLIPPY_DRIVER = clippy-driver
466467
BINDGEN = bindgen
468+
CARGO = cargo
467469
PAHOLE = pahole
468470
RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids
469471
LEX = flex
@@ -525,7 +527,7 @@ KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \
525527
-Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \
526528
-Cforce-unwind-tables=n -Ccodegen-units=1 \
527529
-Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0 \
528-
-D unsafe_op_in_unsafe_fn
530+
-Dunsafe_op_in_unsafe_fn -Wmissing_docs
529531
KBUILD_AFLAGS_KERNEL :=
530532
KBUILD_CFLAGS_KERNEL :=
531533
KBUILD_RUSTCFLAGS_KERNEL :=
@@ -543,14 +545,13 @@ else
543545
RUSTC_OR_CLIPPY_QUIET := RUSTC
544546
RUSTC_OR_CLIPPY = $(RUSTC)
545547
endif
546-
export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY
547548

548549
ifdef RUST_LIB_SRC
549550
export RUST_LIB_SRC
550551
endif
551552

552553
export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
553-
export RUSTC RUSTC_BOOTSTRAP BINDGEN
554+
export RUSTC RUSTC_BOOTSTRAP RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO
554555
export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
555556
export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
556557
export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
@@ -1598,7 +1599,7 @@ endif # CONFIG_MODULES
15981599
# Directories & files removed with 'make clean'
15991600
CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \
16001601
modules.builtin modules.builtin.modinfo modules.nsdeps \
1601-
compile_commands.json .thinlto-cache
1602+
compile_commands.json .thinlto-cache rust/test rust/doc
16021603

16031604
# Directories & files removed with 'make mrproper'
16041605
MRPROPER_FILES += include/config include/generated \
@@ -1728,7 +1729,7 @@ help:
17281729
@echo ' rustdoc - Generate Rust documentation'
17291730
@echo ' (requires kernel .config)'
17301731
@echo ' rusttest - Runs the Rust tests'
1731-
@echo ' (requires kernel .config)'
1732+
@echo ' (requires kernel .config; downloads external repos)'
17321733
@echo ' rust-analyzer - Generate rust-project.json rust-analyzer support file'
17331734
@echo ' (requires kernel .config)'
17341735
@echo ''
@@ -1822,11 +1823,21 @@ rusttest: prepare0
18221823
# Formatting targets
18231824
PHONY += rustfmt rustfmtcheck
18241825

1826+
# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream.
1827+
#
1828+
# We match using absolute paths since `find` does not resolve them
1829+
# when matching, which is a problem when e.g. `srctree` is `..`.
1830+
# We `grep` afterwards in order to remove the directory entry itself.
18251831
rustfmt:
1826-
find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT)
1827-
1828-
rustfmtcheck:
1829-
find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT) --check
1832+
$(Q)find $(abs_srctree) -type f -name '*.rs' \
1833+
-o -path $(abs_srctree)/rust/alloc -prune \
1834+
-o -path $(abs_objtree)/rust/test -prune \
1835+
| grep -Fv $(abs_srctree)/rust/alloc \
1836+
| grep -Fv $(abs_objtree)/rust/test \
1837+
| xargs $(RUSTFMT) $(rustfmt_flags)
1838+
1839+
rustfmtcheck: rustfmt_flags = --check
1840+
rustfmtcheck: rustfmt
18301841

18311842
# IDE support targets
18321843
PHONY += rust-analyzer

‎rust/Makefile

+105-28
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,27 @@ endif
1717

1818
obj-$(CONFIG_RUST) += exports.o
1919

20-
RUSTDOC = rustdoc
20+
ifeq ($(quiet),silent_)
21+
cargo_quiet=-q
22+
rust_test_quiet=-q
23+
rustdoc_test_quiet=--test-args -q
24+
else ifeq ($(quiet),quiet_)
25+
rust_test_quiet=-q
26+
rustdoc_test_quiet=--test-args -q
27+
else
28+
cargo_quiet=--verbose
29+
endif
2130

22-
quiet_cmd_rustdoc = RUSTDOC $(if $(filter --test,$(rustdoc_target_flags)),T, ) $(if $(rustdoc_host),H, ) $<
31+
quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $<
2332
cmd_rustdoc = \
2433
RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
2534
$(RUSTDOC) $(if $(rustdoc_host),,$(rustc_cross_flags)) \
26-
$(filter-out --emit=%, $(rustc_flags)) $(rustc_target_flags) $(rustdoc_target_flags) \
27-
-L $(objtree)/rust/$(if $(filter --test,$(rustdoc_target_flags)),test/) \
28-
--output $(objtree)/rust/doc --crate-name $(subst rusttest-,,$(subst rustdoc-,,$@)) \
29-
-Fmissing-docs @$(objtree)/include/generated/rustc_cfg $<
35+
$(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \
36+
$(rustc_target_flags) -L $(objtree)/rust \
37+
--output $(objtree)/rust/doc --crate-name $(subst rustdoc-,,$@) \
38+
@$(objtree)/include/generated/rustc_cfg $<
3039

31-
rustdoc: rustdoc-macros rustdoc-compiler_builtins rustdoc-kernel
40+
rustdoc: rustdoc-macros rustdoc-compiler_builtins rustdoc-alloc rustdoc-kernel
3241

3342
rustdoc-macros: private rustdoc_host = yes
3443
rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \
@@ -39,6 +48,15 @@ rustdoc-macros: $(srctree)/rust/macros/lib.rs FORCE
3948
rustdoc-compiler_builtins: $(srctree)/rust/compiler_builtins.rs FORCE
4049
$(call if_changed,rustdoc)
4150

51+
# We need to allow `broken_intra_doc_links` because some
52+
# `no_global_oom_handling` functions refer to non-`no_global_oom_handling`
53+
# functions. Ideally `rustdoc` would have a way to distinguish broken links
54+
# due to things that are "configured out" vs. entirely non-existing ones.
55+
rustdoc-alloc: private rustc_target_flags = \
56+
-Abroken_intra_doc_links
57+
rustdoc-alloc: $(srctree)/rust/alloc/lib.rs FORCE
58+
$(call if_changed,rustdoc)
59+
4260
rustdoc-kernel: private rustc_target_flags = --extern alloc \
4361
--extern build_error \
4462
--extern macros=$(objtree)/rust/libmacros.so
@@ -49,50 +67,108 @@ rustdoc-kernel: $(srctree)/rust/kernel/lib.rs rustdoc-macros \
4967
quiet_cmd_rustc_test_library = RUSTC TL $<
5068
cmd_rustc_test_library = \
5169
RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
52-
$(RUSTC) $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \
70+
$(RUSTC) $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \
5371
$(rustc_target_flags) --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \
5472
--out-dir $(objtree)/rust/test/ --cfg testlib \
73+
--sysroot $(objtree)/rust/test/sysroot \
5574
-L $(objtree)/rust/test/ --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $<
5675

57-
rusttestlib-build_error: $(srctree)/rust/build_error.rs FORCE
76+
rusttestlib-build_error: $(srctree)/rust/build_error.rs rusttest-prepare FORCE
5877
$(call if_changed,rustc_test_library)
5978

6079
rusttestlib-macros: private rustc_target_flags = --extern proc_macro
6180
rusttestlib-macros: private rustc_test_library_proc = yes
62-
rusttestlib-macros: $(srctree)/rust/macros/lib.rs FORCE
81+
rusttestlib-macros: $(srctree)/rust/macros/lib.rs rusttest-prepare FORCE
6382
$(call if_changed,rustc_test_library)
6483

84+
quiet_cmd_rustdoc_test = RUSTDOC T $<
85+
cmd_rustdoc_test = \
86+
RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
87+
$(RUSTDOC) --test $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \
88+
$(rustc_target_flags) $(rustdoc_test_target_flags) \
89+
--sysroot $(objtree)/rust/test/sysroot $(rustdoc_test_quiet) \
90+
-L $(objtree)/rust/test \
91+
--output $(objtree)/rust/doc --crate-name $(subst rusttest-,,$@) \
92+
@$(objtree)/include/generated/rustc_cfg $<
93+
6594
# We cannot use `-Zpanic-abort-tests` because some tests are dynamic,
6695
# so for the moment we skip `-Cpanic=abort`.
6796
quiet_cmd_rustc_test = RUSTC T $<
6897
cmd_rustc_test = \
6998
RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
70-
$(RUSTC) --test $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags))) \
99+
$(RUSTC) --test $(filter-out --sysroot=%, $(filter-out -Cpanic=abort, $(filter-out --emit=%, $(rustc_flags)))) \
71100
$(rustc_target_flags) --out-dir $(objtree)/rust/test \
101+
--sysroot $(objtree)/rust/test/sysroot \
72102
-L $(objtree)/rust/test/ --crate-name $(subst rusttest-,,$@) $<; \
73-
$(objtree)/rust/test/$(subst rusttest-,,$@) $(rustc_test_run_flags)
103+
$(objtree)/rust/test/$(subst rusttest-,,$@) $(rust_test_quiet) \
104+
$(rustc_test_run_flags)
74105

75106
rusttest: rusttest-macros rusttest-kernel
76107

108+
# This prepares a custom sysroot with our custom `alloc` instead of
109+
# the standard one.
110+
#
111+
# This requires several hacks:
112+
# - Unlike `core` and `alloc`, `std` depends on more than a dozen crates,
113+
# including third-party crates that need to be downloaded, plus custom
114+
# `build.rs` steps. Thus hardcoding things here is not maintainable.
115+
# - `cargo` knows how to build the standard library, but it is an unstable
116+
# feature so far (`-Zbuild-std`).
117+
# - `cargo` only considers the use case of building the standard library
118+
# to use it in a given package. Thus we need to create a dummy package
119+
# and pick the generated libraries from there.
120+
# - Since we only keep a subset of upstream `alloc` in-tree, we need
121+
# to recreate it on the fly by putting our sources on top.
122+
# - The usual ways of modifying the dependency graph in `cargo` do not seem
123+
# to apply for the `-Zbuild-std` steps, thus we have to mislead it
124+
# by modifying the sources in the sysroot.
125+
# - To avoid messing with the user's Rust installation, we create a clone
126+
# of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std`
127+
# steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag.
128+
#
129+
# In the future, we hope to avoid the whole ordeal by either:
130+
# - Making the `test` crate not depend on `std` (either improving upstream
131+
# or having our own custom crate).
132+
# - Making the tests run in kernel space (requires the previous point).
133+
# - Making `std` and friends be more like a "normal" crate, so that
134+
# `-Zbuild-std` and related hacks are not needed.
135+
quiet_cmd_rustsysroot = RUSTSYSROOT
136+
cmd_rustsysroot = \
137+
rm -rf $(objtree)/rust/test; \
138+
mkdir -p $(objtree)/rust/test; \
139+
cp -a $(rustc_sysroot) $(objtree)/rust/test/sysroot; \
140+
cp -r $(srctree)/rust/alloc/* \
141+
$(objtree)/rust/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \
142+
echo '\#!/bin/sh' > $(objtree)/rust/test/rustc_sysroot; \
143+
echo "$(RUSTC) --sysroot=$(abspath $(objtree)/rust/test/sysroot) \"\$$@\"" \
144+
>> $(objtree)/rust/test/rustc_sysroot; \
145+
chmod u+x $(objtree)/rust/test/rustc_sysroot; \
146+
$(CARGO) -q new $(objtree)/rust/test/dummy; \
147+
RUSTC=$(objtree)/rust/test/rustc_sysroot $(CARGO) $(cargo_quiet) \
148+
test -Zbuild-std --target $(rustc_host_target) \
149+
--manifest-path $(objtree)/rust/test/dummy/Cargo.toml; \
150+
rm $(objtree)/rust/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \
151+
cp $(objtree)/rust/test/dummy/target/$(rustc_host_target)/debug/deps/* \
152+
$(objtree)/rust/test/sysroot/lib/rustlib/$(rustc_host_target)/lib
153+
154+
rusttest-prepare: FORCE
155+
$(call if_changed,rustsysroot)
156+
77157
rusttest-macros: private rustc_target_flags = --extern proc_macro
78-
rusttest-macros: private rustdoc_host = yes
79-
rusttest-macros: private rustdoc_target_flags = --test --crate-type proc-macro
80-
rusttest-macros: $(srctree)/rust/macros/lib.rs FORCE
158+
rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro
159+
rusttest-macros: $(srctree)/rust/macros/lib.rs rusttest-prepare FORCE
81160
$(call if_changed,rustc_test)
82-
$(call if_changed,rustdoc)
161+
$(call if_changed,rustdoc_test)
83162

84163
rusttest-kernel: private rustc_target_flags = --extern alloc \
85-
--extern build_error \
86-
--extern macros=$(objtree)/rust/test/libmacros.so
164+
--extern build_error --extern macros
87165
rusttest-kernel: private rustc_test_run_flags = \
88166
--skip bindgen_test_layout_
89-
rusttest-kernel: private rustdoc_host = yes
90-
rusttest-kernel: private rustdoc_target_flags = --test
91-
rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttestlib-build_error \
92-
rusttestlib-macros FORCE
167+
rusttest-kernel: $(srctree)/rust/kernel/lib.rs rusttest-prepare \
168+
rusttestlib-build_error rusttestlib-macros FORCE
93169
$(call if_changed,rustc_test)
94170
$(call if_changed,rustc_test_library)
95-
$(call if_changed,rustdoc)
171+
$(call if_changed,rustdoc_test)
96172

97173
ifdef CONFIG_CC_IS_CLANG
98174
bindgen_c_flags = $(c_flags)
@@ -188,6 +264,7 @@ quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L
188264

189265
# `$(rustc_flags)` is passed in case the user added `--sysroot`.
190266
rustc_sysroot = $(shell $(RUSTC) $(rustc_flags) --print sysroot)
267+
rustc_host_target = $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2)
191268
RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library
192269

193270
rust-analyzer:
@@ -198,6 +275,11 @@ $(objtree)/rust/compiler_builtins.o: $(srctree)/rust/compiler_builtins.rs \
198275
$(objtree)/rust/core.o FORCE
199276
$(call if_changed_dep,rustc_library)
200277

278+
$(objtree)/rust/alloc.o: private skip_clippy = 1
279+
$(objtree)/rust/alloc.o: $(srctree)/rust/alloc/lib.rs \
280+
$(objtree)/rust/compiler_builtins.o FORCE
281+
$(call if_changed_dep,rustc_library)
282+
201283
$(objtree)/rust/build_error.o: $(srctree)/rust/build_error.rs \
202284
$(objtree)/rust/compiler_builtins.o FORCE
203285
$(call if_changed_dep,rustc_library)
@@ -216,8 +298,3 @@ $(objtree)/rust/kernel.o: $(srctree)/rust/kernel/lib.rs $(objtree)/rust/alloc.o
216298
$(objtree)/rust/core.o: private skip_clippy = 1
217299
$(objtree)/rust/core.o: $$(RUST_LIB_SRC)/core/src/lib.rs FORCE
218300
$(call if_changed_dep,rustc_library)
219-
220-
$(objtree)/rust/alloc.o: private skip_clippy = 1
221-
$(objtree)/rust/alloc.o: $$(RUST_LIB_SRC)/alloc/src/lib.rs \
222-
$(objtree)/rust/compiler_builtins.o FORCE
223-
$(call if_changed_dep,rustc_library)

0 commit comments

Comments
 (0)