Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit b3330f6

Browse files
SpriteOvOSpriteOvO
committedOct 9, 2024·
Add a new trait proc_macro::ToTokens 
1 parent 9e3e517 commit b3330f6

14 files changed

+359
-45
lines changed
 

‎library/proc_macro/src/lib.rs

+15-33
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#![feature(rustc_attrs)]
3434
#![feature(min_specialization)]
3535
#![feature(strict_provenance)]
36+
#![feature(extend_one)]
3637
#![recursion_limit = "256"]
3738
#![allow(internal_features)]
3839
#![deny(ffi_unwind_calls)]
@@ -44,6 +45,7 @@ pub mod bridge;
4445

4546
mod diagnostic;
4647
mod escape;
48+
mod to_tokens;
4749

4850
use std::ffi::CStr;
4951
use std::ops::{Range, RangeBounds};
@@ -53,6 +55,8 @@ use std::{error, fmt};
5355

5456
#[unstable(feature = "proc_macro_diagnostic", issue = "54140")]
5557
pub use diagnostic::{Diagnostic, Level, MultiSpan};
58+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
59+
pub use to_tokens::ToTokens;
5660

5761
use crate::escape::{EscapeOptions, escape_bytes};
5862

@@ -279,6 +283,7 @@ impl ConcatTreesHelper {
279283
}
280284
}
281285

286+
#[allow(dead_code)]
282287
fn append_to(self, stream: &mut TokenStream) {
283288
if self.trees.is_empty() {
284289
return;
@@ -325,45 +330,22 @@ impl ConcatStreamsHelper {
325330
}
326331
}
327332

328-
/// Collects a number of token trees into a single stream.
329-
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
330-
impl FromIterator<TokenTree> for TokenStream {
331-
fn from_iter<I: IntoIterator<Item = TokenTree>>(trees: I) -> Self {
332-
let iter = trees.into_iter();
333-
let mut builder = ConcatTreesHelper::new(iter.size_hint().0);
334-
iter.for_each(|tree| builder.push(tree));
335-
builder.build()
336-
}
337-
}
338-
339-
/// A "flattening" operation on token streams, collects token trees
340-
/// from multiple token streams into a single stream.
341-
#[stable(feature = "proc_macro_lib", since = "1.15.0")]
342-
impl FromIterator<TokenStream> for TokenStream {
343-
fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
344-
let iter = streams.into_iter();
333+
#[stable(feature = "proc_macro_totokens_migration", since = "CURRENT_RUSTC_VERSION")]
334+
impl<T: ToTokens> FromIterator<T> for TokenStream {
335+
fn from_iter<I: IntoIterator<Item = T>>(t: I) -> Self {
336+
let iter = t.into_iter();
345337
let mut builder = ConcatStreamsHelper::new(iter.size_hint().0);
346-
iter.for_each(|stream| builder.push(stream));
338+
iter.for_each(|t| builder.push(t.into_token_stream()));
347339
builder.build()
348340
}
349341
}
350342

351-
#[stable(feature = "token_stream_extend", since = "1.30.0")]
352-
impl Extend<TokenTree> for TokenStream {
353-
fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, trees: I) {
354-
let iter = trees.into_iter();
355-
let mut builder = ConcatTreesHelper::new(iter.size_hint().0);
356-
iter.for_each(|tree| builder.push(tree));
357-
builder.append_to(self);
358-
}
359-
}
360-
361-
#[stable(feature = "token_stream_extend", since = "1.30.0")]
362-
impl Extend<TokenStream> for TokenStream {
363-
fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
364-
let iter = streams.into_iter();
343+
#[stable(feature = "proc_macro_totokens_migration", since = "CURRENT_RUSTC_VERSION")]
344+
impl<T: ToTokens> Extend<T> for TokenStream {
345+
fn extend<I: IntoIterator<Item = T>>(&mut self, t: I) {
346+
let iter = t.into_iter();
365347
let mut builder = ConcatStreamsHelper::new(iter.size_hint().0);
366-
iter.for_each(|stream| builder.push(stream));
348+
iter.for_each(|t| builder.push(t.into_token_stream()));
367349
builder.append_to(self);
368350
}
369351
}

‎library/proc_macro/src/quote.rs

+5-4
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
//! This quasiquoter uses macros 2.0 hygiene to reliably access
55
//! items from `proc_macro`, to build a `proc_macro::TokenStream`.
66
7-
use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
7+
use crate::{
8+
Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
9+
};
810

911
macro_rules! quote_tt {
1012
(($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, quote!($($t)*)) };
@@ -50,7 +52,7 @@ macro_rules! quote {
5052
() => { TokenStream::new() };
5153
($($t:tt)*) => {
5254
[
53-
$(TokenStream::from(quote_ts!($t)),)*
55+
$(ToTokens::into_token_stream(quote_ts!($t)),)*
5456
].iter().cloned().collect::<TokenStream>()
5557
};
5658
}
@@ -73,8 +75,7 @@ pub fn quote(stream: TokenStream) -> TokenStream {
7375
after_dollar = false;
7476
match tree {
7577
TokenTree::Ident(_) => {
76-
return Some(quote!(Into::<crate::TokenStream>::into(
77-
Clone::clone(&(@ tree))),));
78+
return Some(quote!(ToTokens::into_token_stream(Clone::clone(&(@ tree))),));
7879
}
7980
TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
8081
_ => panic!("`$` must be followed by an ident or `$` in `quote!`"),

‎library/proc_macro/src/to_tokens.rs

+310
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,310 @@
1+
use std::borrow::Cow;
2+
use std::ffi::{CStr, CString};
3+
use std::rc::Rc;
4+
5+
use crate::{ConcatTreesHelper, Group, Ident, Literal, Punct, Span, TokenStream, TokenTree};
6+
7+
/// Types that can be interpolated inside a [`quote!`] invocation.
8+
///
9+
/// [`quote!`]: crate::quote!
10+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
11+
pub trait ToTokens {
12+
/// Write `self` to the given `TokenStream`.
13+
///
14+
/// # Example
15+
///
16+
/// Example implementation for a struct representing Rust paths like
17+
/// `std::cmp::PartialEq`:
18+
///
19+
/// ```
20+
/// #![feature(proc_macro_totokens)]
21+
///
22+
/// use std::iter;
23+
/// use proc_macro::{Spacing, Punct, TokenStream, ToTokens};
24+
///
25+
/// pub struct Path {
26+
/// pub global: bool,
27+
/// pub segments: Vec<PathSegment>,
28+
/// }
29+
///
30+
/// impl ToTokens for Path {
31+
/// fn to_tokens(&self, tokens: &mut TokenStream) {
32+
/// for (i, segment) in self.segments.iter().enumerate() {
33+
/// if i > 0 || self.global {
34+
/// // Double colon `::`
35+
/// tokens.extend(iter::once(Punct::new(':', Spacing::Joint)));
36+
/// tokens.extend(iter::once(Punct::new(':', Spacing::Alone)));
37+
/// }
38+
/// segment.to_tokens(tokens);
39+
/// }
40+
/// }
41+
/// }
42+
/// #
43+
/// # pub struct PathSegment;
44+
/// #
45+
/// # impl ToTokens for PathSegment {
46+
/// # fn to_tokens(&self, tokens: &mut TokenStream) {
47+
/// # unimplemented!()
48+
/// # }
49+
/// # }
50+
/// ```
51+
fn to_tokens(&self, tokens: &mut TokenStream);
52+
53+
/// Convert `self` directly into a `TokenStream` object.
54+
///
55+
/// This method is implicitly implemented using `to_tokens`, and acts as a
56+
/// convenience method for consumers of the `ToTokens` trait.
57+
fn to_token_stream(&self) -> TokenStream {
58+
let mut tokens = TokenStream::new();
59+
self.to_tokens(&mut tokens);
60+
tokens
61+
}
62+
63+
/// Convert `self` directly into a `TokenStream` object.
64+
///
65+
/// This method is implicitly implemented using `to_tokens`, and acts as a
66+
/// convenience method for consumers of the `ToTokens` trait.
67+
fn into_token_stream(self) -> TokenStream
68+
where
69+
Self: Sized,
70+
{
71+
self.to_token_stream()
72+
}
73+
}
74+
75+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
76+
impl ToTokens for TokenTree {
77+
fn to_tokens(&self, tokens: &mut TokenStream) {
78+
tokens.extend_one(self.clone());
79+
}
80+
81+
fn into_token_stream(self) -> TokenStream {
82+
let mut builder = ConcatTreesHelper::new(1);
83+
builder.push(self);
84+
builder.build()
85+
}
86+
}
87+
88+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
89+
impl ToTokens for TokenStream {
90+
fn to_tokens(&self, tokens: &mut TokenStream) {
91+
tokens.extend(self.clone());
92+
}
93+
94+
fn into_token_stream(self) -> TokenStream {
95+
self
96+
}
97+
}
98+
99+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
100+
impl ToTokens for Literal {
101+
fn to_tokens(&self, tokens: &mut TokenStream) {
102+
tokens.extend_one(TokenTree::from(self.clone()));
103+
}
104+
}
105+
106+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
107+
impl ToTokens for Ident {
108+
fn to_tokens(&self, tokens: &mut TokenStream) {
109+
tokens.extend_one(TokenTree::from(self.clone()));
110+
}
111+
}
112+
113+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
114+
impl ToTokens for Punct {
115+
fn to_tokens(&self, tokens: &mut TokenStream) {
116+
tokens.extend_one(TokenTree::from(self.clone()));
117+
}
118+
}
119+
120+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
121+
impl ToTokens for Group {
122+
fn to_tokens(&self, tokens: &mut TokenStream) {
123+
tokens.extend_one(TokenTree::from(self.clone()));
124+
}
125+
}
126+
127+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
128+
impl<T: ToTokens + ?Sized> ToTokens for &T {
129+
fn to_tokens(&self, tokens: &mut TokenStream) {
130+
(**self).to_tokens(tokens)
131+
}
132+
}
133+
134+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
135+
impl<T: ToTokens + ?Sized> ToTokens for &mut T {
136+
fn to_tokens(&self, tokens: &mut TokenStream) {
137+
(**self).to_tokens(tokens)
138+
}
139+
}
140+
141+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
142+
impl<T: ToTokens + ?Sized> ToTokens for Box<T> {
143+
fn to_tokens(&self, tokens: &mut TokenStream) {
144+
(**self).to_tokens(tokens)
145+
}
146+
}
147+
148+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
149+
impl<T: ToTokens + ?Sized> ToTokens for Rc<T> {
150+
fn to_tokens(&self, tokens: &mut TokenStream) {
151+
(**self).to_tokens(tokens)
152+
}
153+
}
154+
155+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
156+
impl<T: ToTokens + ToOwned + ?Sized> ToTokens for Cow<'_, T> {
157+
fn to_tokens(&self, tokens: &mut TokenStream) {
158+
(**self).to_tokens(tokens)
159+
}
160+
}
161+
162+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
163+
impl<T: ToTokens> ToTokens for Option<T> {
164+
fn to_tokens(&self, tokens: &mut TokenStream) {
165+
if let Some(t) = self {
166+
t.to_tokens(tokens);
167+
}
168+
}
169+
}
170+
171+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
172+
impl ToTokens for u8 {
173+
fn to_tokens(&self, tokens: &mut TokenStream) {
174+
Literal::u8_suffixed(*self).to_tokens(tokens)
175+
}
176+
}
177+
178+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
179+
impl ToTokens for u16 {
180+
fn to_tokens(&self, tokens: &mut TokenStream) {
181+
Literal::u16_suffixed(*self).to_tokens(tokens)
182+
}
183+
}
184+
185+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
186+
impl ToTokens for u32 {
187+
fn to_tokens(&self, tokens: &mut TokenStream) {
188+
Literal::u32_suffixed(*self).to_tokens(tokens)
189+
}
190+
}
191+
192+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
193+
impl ToTokens for u64 {
194+
fn to_tokens(&self, tokens: &mut TokenStream) {
195+
Literal::u64_suffixed(*self).to_tokens(tokens)
196+
}
197+
}
198+
199+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
200+
impl ToTokens for u128 {
201+
fn to_tokens(&self, tokens: &mut TokenStream) {
202+
Literal::u128_suffixed(*self).to_tokens(tokens)
203+
}
204+
}
205+
206+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
207+
impl ToTokens for i8 {
208+
fn to_tokens(&self, tokens: &mut TokenStream) {
209+
Literal::i8_suffixed(*self).to_tokens(tokens)
210+
}
211+
}
212+
213+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
214+
impl ToTokens for i16 {
215+
fn to_tokens(&self, tokens: &mut TokenStream) {
216+
Literal::i16_suffixed(*self).to_tokens(tokens)
217+
}
218+
}
219+
220+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
221+
impl ToTokens for i32 {
222+
fn to_tokens(&self, tokens: &mut TokenStream) {
223+
Literal::i32_suffixed(*self).to_tokens(tokens)
224+
}
225+
}
226+
227+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
228+
impl ToTokens for i64 {
229+
fn to_tokens(&self, tokens: &mut TokenStream) {
230+
Literal::i64_suffixed(*self).to_tokens(tokens)
231+
}
232+
}
233+
234+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
235+
impl ToTokens for i128 {
236+
fn to_tokens(&self, tokens: &mut TokenStream) {
237+
Literal::i128_suffixed(*self).to_tokens(tokens)
238+
}
239+
}
240+
241+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
242+
impl ToTokens for f32 {
243+
fn to_tokens(&self, tokens: &mut TokenStream) {
244+
Literal::f32_suffixed(*self).to_tokens(tokens)
245+
}
246+
}
247+
248+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
249+
impl ToTokens for f64 {
250+
fn to_tokens(&self, tokens: &mut TokenStream) {
251+
Literal::f64_suffixed(*self).to_tokens(tokens)
252+
}
253+
}
254+
255+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
256+
impl ToTokens for usize {
257+
fn to_tokens(&self, tokens: &mut TokenStream) {
258+
Literal::usize_suffixed(*self).to_tokens(tokens)
259+
}
260+
}
261+
262+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
263+
impl ToTokens for isize {
264+
fn to_tokens(&self, tokens: &mut TokenStream) {
265+
Literal::isize_suffixed(*self).to_tokens(tokens)
266+
}
267+
}
268+
269+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
270+
impl ToTokens for bool {
271+
fn to_tokens(&self, tokens: &mut TokenStream) {
272+
let word = if *self { "true" } else { "false" };
273+
Ident::new(word, Span::call_site()).to_tokens(tokens)
274+
}
275+
}
276+
277+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
278+
impl ToTokens for char {
279+
fn to_tokens(&self, tokens: &mut TokenStream) {
280+
Literal::character(*self).to_tokens(tokens)
281+
}
282+
}
283+
284+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
285+
impl ToTokens for str {
286+
fn to_tokens(&self, tokens: &mut TokenStream) {
287+
Literal::string(self).to_tokens(tokens)
288+
}
289+
}
290+
291+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
292+
impl ToTokens for String {
293+
fn to_tokens(&self, tokens: &mut TokenStream) {
294+
Literal::string(self).to_tokens(tokens)
295+
}
296+
}
297+
298+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
299+
impl ToTokens for CStr {
300+
fn to_tokens(&self, tokens: &mut TokenStream) {
301+
Literal::c_string(self).to_tokens(tokens)
302+
}
303+
}
304+
305+
#[unstable(feature = "proc_macro_totokens", issue = "130977")]
306+
impl ToTokens for CString {
307+
fn to_tokens(&self, tokens: &mut TokenStream) {
308+
Literal::c_string(self).to_tokens(tokens)
309+
}
310+
}

‎tests/ui/macros/auxiliary/issue-100199.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33

44
#![crate_type = "proc-macro"]
55
#![feature(proc_macro_quote)]
6+
#![feature(proc_macro_totokens)]
67

78
extern crate proc_macro;
89

9-
use proc_macro::{quote, Ident, Span, TokenStream, TokenTree};
10+
use proc_macro::{Ident, Span, TokenStream, TokenTree, quote};
1011

1112
#[proc_macro_attribute]
1213
pub fn struct_with_bound(_: TokenStream, _: TokenStream) -> TokenStream {

‎tests/ui/macros/auxiliary/proc_macro_def.rs

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#![crate_type = "proc-macro"]
55
#![feature(proc_macro_quote)]
6+
#![feature(proc_macro_totokens)]
67

78
extern crate proc_macro;
89

‎tests/ui/macros/proc_macro.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
extern crate proc_macro_def;
55

6-
use proc_macro_def::{attr_tru, attr_identity, identity, ret_tru, tru};
6+
use proc_macro_def::{attr_identity, attr_tru, identity, ret_tru, tru};
77

88
#[attr_tru]
99
fn f1() -> bool {

‎tests/ui/proc-macro/auxiliary/cond_plugin.rs

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#![crate_type = "proc-macro"]
55
#![feature(proc_macro_quote)]
6+
#![feature(proc_macro_totokens)]
67

78
extern crate proc_macro;
89

‎tests/ui/proc-macro/auxiliary/double.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
//@ no-prefer-dynamic
33

44
#![feature(proc_macro_quote)]
5-
5+
#![feature(proc_macro_totokens)]
66
#![crate_type = "proc-macro"]
77

88
extern crate proc_macro;

‎tests/ui/proc-macro/auxiliary/generate-dollar-ident.rs

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
//@ no-prefer-dynamic
33

44
#![feature(proc_macro_quote)]
5+
#![feature(proc_macro_totokens)]
56
#![crate_type = "proc-macro"]
67

78
extern crate proc_macro;

‎tests/ui/proc-macro/auxiliary/hygiene_example_codegen.rs

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
//@ no-prefer-dynamic
33

44
#![feature(proc_macro_quote)]
5+
#![feature(proc_macro_totokens)]
56
#![crate_type = "proc-macro"]
67

78
extern crate proc_macro as proc_macro_renamed; // This does not break `quote!`

‎tests/ui/proc-macro/auxiliary/mixed-site-span.rs

+3-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
//@ no-prefer-dynamic
33

44
#![feature(proc_macro_quote)]
5-
5+
#![feature(proc_macro_totokens)]
66
#![crate_type = "proc-macro"]
77

88
extern crate proc_macro;
@@ -18,10 +18,8 @@ pub fn proc_macro_rules(input: TokenStream) -> TokenStream {
1818
let local_use = id("local_use");
1919
let mut single_quote = Punct::new('\'', Spacing::Joint);
2020
single_quote.set_span(Span::mixed_site());
21-
let label_use: TokenStream = [
22-
TokenTree::from(single_quote),
23-
id("label_use"),
24-
].iter().cloned().collect();
21+
let label_use: TokenStream =
22+
[TokenTree::from(single_quote), id("label_use")].iter().cloned().collect();
2523
quote!(
2624
struct $item_def;
2725
let $local_def = 0;

‎tests/ui/proc-macro/auxiliary/nonterminal-recollect-attr.rs

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
#![crate_type = "proc-macro"]
55
#![feature(proc_macro_quote)]
6+
#![feature(proc_macro_totokens)]
67

78
extern crate proc_macro;
89
use proc_macro::{TokenStream, quote};

‎tests/ui/proc-macro/auxiliary/resolved-located-at.rs

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#![feature(proc_macro_def_site)]
55
#![feature(proc_macro_diagnostic)]
66
#![feature(proc_macro_quote)]
7+
#![feature(proc_macro_totokens)]
78
#![crate_type = "proc-macro"]
89

910
extern crate proc_macro;

‎tests/ui/proc-macro/quote-interpolation.rs

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
//@ check-pass
2+
3+
#![feature(proc_macro_quote)]
4+
#![feature(proc_macro_totokens)]
5+
#![crate_type = "proc-macro"]
6+
7+
extern crate proc_macro;
8+
9+
use proc_macro::*;
10+
11+
fn main() {
12+
let x = Ident::new("foo", Span::call_site());
13+
let _ = quote! {
14+
let $x = 199;
15+
};
16+
}

0 commit comments

Comments
 (0)
Please sign in to comment.