proc_macro: Add an expand_expr method to TokenStream

This feature is aimed at giving proc macros access to powers similar to
those used by builtin macros such as `format_args!` or `concat!`. These
macros are able to accept macros in place of string literal parameters,
such as the format string, as they perform recursive macro expansion
while being expanded.

This can be especially useful in many cases thanks to helper macros like
`concat!`, `stringify!` and `include_str!` which are often used to
construct string literals at compile-time in user code.

For now, this method only allows expanding macros which produce
literals, although more expresisons will be supported before the method
is stabilized.

Author: mystor

compiler/rustc_expand/src/proc_macro.rs

@@ -24,8 +24,9 @@ impl base::ProcMacro for BangProcMacro {
         span: Span,
         input: TokenStream,
     ) -> Result<TokenStream, ErrorReported> {
+        let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
         let server = proc_macro_server::Rustc::new(ecx);
-        self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace).map_err(|e| {
+        self.client.run(&EXEC_STRATEGY, server, input, proc_macro_backtrace).map_err(|e| {
             let mut err = ecx.struct_span_err(span, "proc macro panicked");
             if let Some(s) = e.as_str() {
                 err.help(&format!("message: {}", s));
@@ -48,9 +49,10 @@ impl base::AttrProcMacro for AttrProcMacro {
         annotation: TokenStream,
         annotated: TokenStream,
     ) -> Result<TokenStream, ErrorReported> {
+        let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
         let server = proc_macro_server::Rustc::new(ecx);
         self.client
-            .run(&EXEC_STRATEGY, server, annotation, annotated, ecx.ecfg.proc_macro_backtrace)
+            .run(&EXEC_STRATEGY, server, annotation, annotated, proc_macro_backtrace)
             .map_err(|e| {
                 let mut err = ecx.struct_span_err(span, "custom attribute panicked");
                 if let Some(s) = e.as_str() {
@@ -97,19 +99,19 @@ impl MultiItemModifier for ProcMacroDerive {
             nt_to_tokenstream(&item, &ecx.sess.parse_sess, CanSynthesizeMissingTokens::No)
         };
 
+        let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
         let server = proc_macro_server::Rustc::new(ecx);
-        let stream =
-            match self.client.run(&EXEC_STRATEGY, server, input, ecx.ecfg.proc_macro_backtrace) {
-                Ok(stream) => stream,
-                Err(e) => {
-                    let mut err = ecx.struct_span_err(span, "proc-macro derive panicked");
-                    if let Some(s) = e.as_str() {
-                        err.help(&format!("message: {}", s));
-                    }
-                    err.emit();
-                    return ExpandResult::Ready(vec![]);
+        let stream = match self.client.run(&EXEC_STRATEGY, server, input, proc_macro_backtrace) {
+            Ok(stream) => stream,
+            Err(e) => {
+                let mut err = ecx.struct_span_err(span, "proc-macro derive panicked");
+                if let Some(s) = e.as_str() {
+                    err.help(&format!("message: {}", s));
                 }
-            };
+                err.emit();
+                return ExpandResult::Ready(vec![]);
+            }
+        };
 
         let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count();
         let mut parser =

compiler/rustc_expand/src/proc_macro_server.rs

@@ -62,6 +62,7 @@ macro_rules! with_api {
                 fn clone($self: &$S::TokenStream) -> $S::TokenStream;
                 fn new() -> $S::TokenStream;
                 fn is_empty($self: &$S::TokenStream) -> bool;
+                fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
                 fn from_str(src: &str) -> $S::TokenStream;
                 fn to_string($self: &$S::TokenStream) -> String;
                 fn from_token_tree(

library/proc_macro/src/bridge/mod.rs

@@ -88,12 +88,6 @@ impl !Sync for TokenStream {}
 #[derive(Debug)]
 pub struct LexError;
 
-impl LexError {
-    fn new() -> Self {
-        LexError
-    }
-}
-
 #[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")]
 impl fmt::Display for LexError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -109,6 +103,28 @@ impl !Send for LexError {}
 #[stable(feature = "proc_macro_lib", since = "1.15.0")]
 impl !Sync for LexError {}
 
+/// Error returned from `TokenStream::expand_expr`.
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+#[non_exhaustive]
+#[derive(Debug)]
+pub struct ExpandError;
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl fmt::Display for ExpandError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str("macro expansion failed")
+    }
+}
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl error::Error for ExpandError {}
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl !Send for ExpandError {}
+
+#[unstable(feature = "proc_macro_expand", issue = "90765")]
+impl !Sync for ExpandError {}
+
 impl TokenStream {
     /// Returns an empty `TokenStream` containing no token trees.
     #[stable(feature = "proc_macro_lib2", since = "1.29.0")]
@@ -121,6 +137,24 @@ impl TokenStream {
     pub fn is_empty(&self) -> bool {
         self.0.is_empty()
     }
+
+    /// Parses this `TokenStream` as an expression and attempts to expand any
+    /// macros within it. Returns the expanded `TokenStream`.
+    ///
+    /// Currently only expressions expanding to literals will succeed, although
+    /// this may be relaxed in the future.
+    ///
+    /// NOTE: In error conditions, `expand_expr` may leave macros unexpanded,
+    /// report an error, failing compilation, and/or return an `Err(..)`. The
+    /// specific behavior for any error condition, and what conditions are
+    /// considered errors, is unspecified and may change in the future.
+    #[unstable(feature = "proc_macro_expand", issue = "90765")]
+    pub fn expand_expr(&self) -> Result<TokenStream, ExpandError> {
+        match bridge::client::TokenStream::expand_expr(&self.0) {
+            Ok(stream) => Ok(TokenStream(stream)),
+            Err(_) => Err(ExpandError),
+        }
+    }
 }
 
 /// Attempts to break the string into tokens and parse those tokens into a token stream.
@@ -1211,7 +1245,7 @@ impl FromStr for Literal {
     fn from_str(src: &str) -> Result<Self, LexError> {
         match bridge::client::Literal::from_str(src) {
             Ok(literal) => Ok(Literal(literal)),
-            Err(()) => Err(LexError::new()),
+            Err(()) => Err(LexError),
         }
     }
 }

library/proc_macro/src/lib.rs

@@ -0,0 +1,80 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+#![deny(warnings)]
+#![feature(proc_macro_expand, proc_macro_span)]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+use std::str::FromStr;
+
+#[proc_macro]
+pub fn expand_expr_is(input: TokenStream) -> TokenStream {
+    let mut iter = input.into_iter();
+    let mut expected_tts = Vec::new();
+    loop {
+        match iter.next() {
+            Some(TokenTree::Punct(ref p)) if p.as_char() == ',' => break,
+            Some(tt) => expected_tts.push(tt),
+            None => panic!("expected comma"),
+        }
+    }
+
+    let expected = expected_tts.into_iter().collect::<TokenStream>();
+    let expanded = iter.collect::<TokenStream>().expand_expr().expect("expand_expr failed");
+    assert!(
+        expected.to_string() == expanded.to_string(),
+        "assert failed\nexpected: `{}`\nexpanded: `{}`",
+        expected.to_string(),
+        expanded.to_string()
+    );
+
+    TokenStream::new()
+}
+
+#[proc_macro]
+pub fn expand_expr_fail(input: TokenStream) -> TokenStream {
+    match input.expand_expr() {
+        Ok(ts) => panic!("expand_expr unexpectedly succeeded: `{}`", ts),
+        Err(_) => TokenStream::new(),
+    }
+}
+
+#[proc_macro]
+pub fn check_expand_expr_file(ts: TokenStream) -> TokenStream {
+    // Check that the passed in `file!()` invocation and a parsed `file!`
+    // invocation expand to the same literal.
+    let input_t = ts.expand_expr().expect("expand_expr failed on macro input").to_string();
+    let parse_t = TokenStream::from_str("file!{}")
+    .unwrap()
+        .expand_expr()
+        .expect("expand_expr failed on internal macro")
+        .to_string();
+    assert_eq!(input_t, parse_t);
+
+    // Check that the literal matches `Span::call_site().source_file().path()`
+    let expect_t =
+        Literal::string(&Span::call_site().source_file().path().to_string_lossy()).to_string();
+    assert_eq!(input_t, expect_t);
+
+    TokenStream::new()
+}
+
+#[proc_macro]
+pub fn recursive_expand(_: TokenStream) -> TokenStream {
+    // Recursively call until we hit the recursion limit and get an error.
+    //
+    // NOTE: This doesn't panic if expansion fails because that'll cause a very
+    // large number of errors to fill the output.
+    TokenStream::from_str("recursive_expand!{}")
+        .unwrap()
+        .expand_expr()
+        .unwrap_or(std::iter::once(TokenTree::Literal(Literal::u32_suffixed(0))).collect())
+}
+
+#[proc_macro]
+pub fn echo_pm(input: TokenStream) -> TokenStream {
+    input
+}

src/test/ui/proc-macro/auxiliary/expand-expr.rs

@@ -0,0 +1 @@
+Included file contents

src/test/ui/proc-macro/auxiliary/included-file.txt

@@ -0,0 +1,121 @@
+// aux-build:expand-expr.rs
+
+extern crate expand_expr;
+
+use expand_expr::{
+    check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is, recursive_expand,
+};
+
+// Check builtin macros can be expanded.
+
+expand_expr_is!(11u32, line!());
+expand_expr_is!(24u32, column!());
+
+expand_expr_is!("Hello, World!", concat!("Hello, ", "World", "!"));
+expand_expr_is!("int10floats5.3booltrue", concat!("int", 10, "floats", 5.3, "bool", true));
+expand_expr_is!("Hello", concat!(r##"Hello"##));
+
+expand_expr_is!("Included file contents\n", include_str!("auxiliary/included-file.txt"));
+expand_expr_is!(b"Included file contents\n", include_bytes!("auxiliary/included-file.txt"));
+
+expand_expr_is!(
+    "contents: Included file contents\n",
+    concat!("contents: ", include_str!("auxiliary/included-file.txt"))
+);
+
+// Correct value is checked for multiple sources.
+check_expand_expr_file!(file!());
+
+expand_expr_is!("hello", stringify!(hello));
+expand_expr_is!("10 + 20", stringify!(10 + 20));
+
+macro_rules! echo_tts {
+    ($($t:tt)*) => { $($t)* };  //~ ERROR: expected expression, found `$`
+}
+
+macro_rules! echo_lit {
+    ($l:literal) => {
+        $l
+    };
+}
+
+macro_rules! echo_expr {
+    ($e:expr) => {
+        $e
+    };
+}
+
+macro_rules! simple_lit {
+    ($l:literal) => {
+        expand_expr_is!($l, $l);
+        expand_expr_is!($l, echo_lit!($l));
+        expand_expr_is!($l, echo_expr!($l));
+        expand_expr_is!($l, echo_tts!($l));
+        expand_expr_is!($l, echo_pm!($l));
+        const _: () = {
+            macro_rules! mac {
+                () => {
+                    $l
+                };
+            }
+            expand_expr_is!($l, mac!());
+            expand_expr_is!($l, echo_expr!(mac!()));
+            expand_expr_is!($l, echo_tts!(mac!()));
+            expand_expr_is!($l, echo_pm!(mac!()));
+        };
+    };
+}
+
+simple_lit!("Hello, World");
+simple_lit!('c');
+simple_lit!(b'c');
+simple_lit!(10);
+simple_lit!(10.0);
+simple_lit!(10.0f64);
+simple_lit!(-3.14159);
+simple_lit!(-3.5e10);
+simple_lit!(0xFEED);
+simple_lit!(-0xFEED);
+simple_lit!(0b0100);
+simple_lit!(-0b0100);
+simple_lit!("string");
+simple_lit!(r##"raw string"##);
+simple_lit!(b"byte string");
+simple_lit!(br##"raw byte string"##);
+simple_lit!(true);
+simple_lit!(false);
+
+// Ensure char escapes aren't normalized by expansion
+simple_lit!("\u{0}");
+simple_lit!("\0");
+simple_lit!("\x00");
+simple_lit!('\u{0}');
+simple_lit!('\0');
+simple_lit!('\x00');
+simple_lit!(b"\x00");
+simple_lit!(b"\0");
+simple_lit!(b'\x00');
+simple_lit!(b'\0');
+
+// Extra tokens after the string literal aren't ignored
+expand_expr_fail!("string"; hello); //~ ERROR: expected one of `.`, `?`, or an operator, found `;`
+
+// Invalid expressions produce errors in addition to returning `Err(())`.
+expand_expr_fail!($); //~ ERROR: expected expression, found `$`
+expand_expr_fail!(echo_tts!($));
+expand_expr_fail!(echo_pm!($)); //~ ERROR: expected expression, found `$`
+
+// We get errors reported and recover during macro expansion if the macro
+// doesn't produce a valid expression.
+expand_expr_is!("string", echo_tts!("string"; hello)); //~ ERROR: macro expansion ignores token `hello` and any following
+expand_expr_is!("string", echo_pm!("string"; hello)); //~ ERROR: macro expansion ignores token `;` and any following
+
+// For now, fail if a non-literal expression is expanded.
+expand_expr_fail!(arbitrary_expression() + "etc");
+expand_expr_fail!(echo_tts!(arbitrary_expression() + "etc"));
+expand_expr_fail!(echo_expr!(arbitrary_expression() + "etc"));
+expand_expr_fail!(echo_pm!(arbitrary_expression() + "etc"));
+
+const _: u32 = recursive_expand!(); //~ ERROR: recursion limit reached while expanding `recursive_expand!`
+
+fn main() {}

src/test/ui/proc-macro/expand-expr.rs

@@ -0,0 +1,55 @@
+error: expected one of `.`, `?`, or an operator, found `;`
+  --> $DIR/expand-expr.rs:101:27
+   |
+LL | expand_expr_fail!("string"; hello);
+   |                           ^ expected one of `.`, `?`, or an operator
+
+error: expected expression, found `$`
+  --> $DIR/expand-expr.rs:104:19
+   |
+LL | expand_expr_fail!($);
+   |                   ^ expected expression
+
+error: expected expression, found `$`
+  --> $DIR/expand-expr.rs:33:23
+   |
+LL |     ($($t:tt)*) => { $($t)* };
+   |                       ^^^^ expected expression
+
+error: expected expression, found `$`
+  --> $DIR/expand-expr.rs:106:28
+   |
+LL | expand_expr_fail!(echo_pm!($));
+   |                            ^ expected expression
+
+error: macro expansion ignores token `hello` and any following
+  --> $DIR/expand-expr.rs:110:47
+   |
+LL | expand_expr_is!("string", echo_tts!("string"; hello));
+   |                           --------------------^^^^^-- help: you might be missing a semicolon here: `;`
+   |                           |
+   |                           caused by the macro expansion here
+   |
+   = note: the usage of `echo_tts!` is likely invalid in expression context
+
+error: macro expansion ignores token `;` and any following
+  --> $DIR/expand-expr.rs:111:44
+   |
+LL | expand_expr_is!("string", echo_pm!("string"; hello));
+   |                           -----------------^-------- help: you might be missing a semicolon here: `;`
+   |                           |
+   |                           caused by the macro expansion here
+   |
+   = note: the usage of `echo_pm!` is likely invalid in expression context
+
+error: recursion limit reached while expanding `recursive_expand!`
+  --> $DIR/expand-expr.rs:119:16
+   |
+LL | const _: u32 = recursive_expand!();
+   |                ^^^^^^^^^^^^^^^^^^^
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`expand_expr`)
+   = note: this error originates in the macro `recursive_expand` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 7 previous errors
+