Skip to content

Commit 3e39180

Browse files
authored
Rollup merge of rust-lang#47603 - mark-i-m:markim_comments_0000, r=jseyfried
Run rustfmt and add doc comments to libsyntax/ext/tt/quoted.rs I was already going to try to understand this code to try to implement rust-lang/rfcs#2298. I figured I would put that knowledge into comments and share... This PR contains _no changes_ to the code itself -- just formatting and comments. I'm not sure what the best way to do this is. I plan to make more such PR for other files, but I figured it would have fewer conflicts if I do it file by file...
2 parents 19aac09 + 5762942 commit 3e39180

File tree

1 file changed

+173
-39
lines changed

1 file changed

+173
-39
lines changed

src/libsyntax/ext/tt/quoted.rs

+173-39
Original file line numberDiff line numberDiff line change
@@ -10,29 +10,34 @@
1010

1111
use ast;
1212
use ext::tt::macro_parser;
13-
use parse::{ParseSess, token};
13+
use parse::{token, ParseSess};
1414
use print::pprust;
1515
use symbol::keywords;
16-
use syntax_pos::{DUMMY_SP, Span, BytePos};
16+
use syntax_pos::{BytePos, Span, DUMMY_SP};
1717
use tokenstream;
1818

1919
use std::rc::Rc;
2020

21+
/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
22+
/// that the delimiter itself might be `NoDelim`.
2123
#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)]
2224
pub struct Delimited {
2325
pub delim: token::DelimToken,
2426
pub tts: Vec<TokenTree>,
2527
}
2628

2729
impl Delimited {
30+
/// Return the opening delimiter (possibly `NoDelim`).
2831
pub fn open_token(&self) -> token::Token {
2932
token::OpenDelim(self.delim)
3033
}
3134

35+
/// Return the closing delimiter (possibly `NoDelim`).
3236
pub fn close_token(&self) -> token::Token {
3337
token::CloseDelim(self.delim)
3438
}
3539

40+
/// Return a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
3641
pub fn open_tt(&self, span: Span) -> TokenTree {
3742
let open_span = if span == DUMMY_SP {
3843
DUMMY_SP
@@ -42,6 +47,7 @@ impl Delimited {
4247
TokenTree::Token(open_span, self.open_token())
4348
}
4449

50+
/// Return a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
4551
pub fn close_tt(&self, span: Span) -> TokenTree {
4652
let close_span = if span == DUMMY_SP {
4753
DUMMY_SP
@@ -68,12 +74,14 @@ pub struct SequenceRepetition {
6874
/// for token sequences.
6975
#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
7076
pub enum KleeneOp {
77+
/// Kleene star (`*`) for zero or more repetitions
7178
ZeroOrMore,
79+
/// Kleene plus (`+`) for one or more repetitions
7280
OneOrMore,
7381
}
7482

7583
/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
76-
/// are "first-class" token trees.
84+
/// are "first-class" token trees. Useful for parsing macros.
7785
#[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)]
7886
pub enum TokenTree {
7987
Token(Span, token::Token),
@@ -83,10 +91,15 @@ pub enum TokenTree {
8391
/// E.g. `$var`
8492
MetaVar(Span, ast::Ident),
8593
/// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
86-
MetaVarDecl(Span, ast::Ident /* name to bind */, ast::Ident /* kind of nonterminal */),
94+
MetaVarDecl(
95+
Span,
96+
ast::Ident, /* name to bind */
97+
ast::Ident, /* kind of nonterminal */
98+
),
8799
}
88100

89101
impl TokenTree {
102+
/// Return the number of tokens in the tree.
90103
pub fn len(&self) -> usize {
91104
match *self {
92105
TokenTree::Delimited(_, ref delimed) => match delimed.delim {
@@ -98,6 +111,8 @@ impl TokenTree {
98111
}
99112
}
100113

114+
/// Returns true if the given token tree contains no other tokens. This is vacuously true for
115+
/// single tokens or metavar/decls, but may be false for delimited trees or sequences.
101116
pub fn is_empty(&self) -> bool {
102117
match *self {
103118
TokenTree::Delimited(_, ref delimed) => match delimed.delim {
@@ -109,6 +124,7 @@ impl TokenTree {
109124
}
110125
}
111126

127+
/// Get the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
112128
pub fn get_tt(&self, index: usize) -> TokenTree {
113129
match (self, index) {
114130
(&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
@@ -131,21 +147,48 @@ impl TokenTree {
131147
/// Retrieve the `TokenTree`'s span.
132148
pub fn span(&self) -> Span {
133149
match *self {
134-
TokenTree::Token(sp, _) |
135-
TokenTree::MetaVar(sp, _) |
136-
TokenTree::MetaVarDecl(sp, _, _) |
137-
TokenTree::Delimited(sp, _) |
138-
TokenTree::Sequence(sp, _) => sp,
150+
TokenTree::Token(sp, _)
151+
| TokenTree::MetaVar(sp, _)
152+
| TokenTree::MetaVarDecl(sp, _, _)
153+
| TokenTree::Delimited(sp, _)
154+
| TokenTree::Sequence(sp, _) => sp,
139155
}
140156
}
141157
}
142158

143-
pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &ParseSess)
144-
-> Vec<TokenTree> {
159+
/// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this
160+
/// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
161+
/// collection of `TokenTree` for use in parsing a macro.
162+
///
163+
/// # Parameters
164+
///
165+
/// - `input`: a token stream to read from, the contents of which we are parsing.
166+
/// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a
167+
/// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with
168+
/// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and
169+
/// `ident` are "matchers". They are not present in the body of a macro rule -- just in the
170+
/// pattern, so we pass a parameter to indicate whether to expect them or not.
171+
/// - `sess`: the parsing session. Any errors will be emitted to this session.
172+
///
173+
/// # Returns
174+
///
175+
/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
176+
pub fn parse(
177+
input: tokenstream::TokenStream,
178+
expect_matchers: bool,
179+
sess: &ParseSess,
180+
) -> Vec<TokenTree> {
181+
// Will contain the final collection of `self::TokenTree`
145182
let mut result = Vec::new();
183+
184+
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
185+
// additional trees if need be.
146186
let mut trees = input.trees();
147187
while let Some(tree) = trees.next() {
148188
let tree = parse_tree(tree, &mut trees, expect_matchers, sess);
189+
190+
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually
191+
// parse out the matcher (i.e. in `$id:ident` this would parse the `:` and `ident`).
149192
match tree {
150193
TokenTree::MetaVar(start_sp, ident) if expect_matchers => {
151194
let span = match trees.next() {
@@ -154,78 +197,149 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars
154197
Some(kind) => {
155198
let span = end_sp.with_lo(start_sp.lo());
156199
result.push(TokenTree::MetaVarDecl(span, ident, kind));
157-
continue
200+
continue;
158201
}
159202
_ => end_sp,
160203
},
161-
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
204+
tree => tree.as_ref()
205+
.map(tokenstream::TokenTree::span)
206+
.unwrap_or(span),
162207
},
163-
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp),
208+
tree => tree.as_ref()
209+
.map(tokenstream::TokenTree::span)
210+
.unwrap_or(start_sp),
164211
};
165212
sess.missing_fragment_specifiers.borrow_mut().insert(span);
166-
result.push(TokenTree::MetaVarDecl(span, ident, keywords::Invalid.ident()));
213+
result.push(TokenTree::MetaVarDecl(
214+
span,
215+
ident,
216+
keywords::Invalid.ident(),
217+
));
167218
}
219+
220+
// Not a metavar or no matchers allowed, so just return the tree
168221
_ => result.push(tree),
169222
}
170223
}
171224
result
172225
}
173226

174-
fn parse_tree<I>(tree: tokenstream::TokenTree,
175-
trees: &mut I,
176-
expect_matchers: bool,
177-
sess: &ParseSess)
178-
-> TokenTree
179-
where I: Iterator<Item = tokenstream::TokenTree>,
227+
/// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a
228+
/// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree`
229+
/// for use in parsing a macro.
230+
///
231+
/// Converting the given tree may involve reading more tokens.
232+
///
233+
/// # Parameters
234+
///
235+
/// - `tree`: the tree we wish to convert.
236+
/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish
237+
/// converting `tree`
238+
/// - `expect_matchers`: same as for `parse` (see above).
239+
/// - `sess`: the parsing session. Any errors will be emitted to this session.
240+
fn parse_tree<I>(
241+
tree: tokenstream::TokenTree,
242+
trees: &mut I,
243+
expect_matchers: bool,
244+
sess: &ParseSess,
245+
) -> TokenTree
246+
where
247+
I: Iterator<Item = tokenstream::TokenTree>,
180248
{
249+
// Depending on what `tree` is, we could be parsing different parts of a macro
181250
match tree {
251+
// `tree` is a `$` token. Look at the next token in `trees`
182252
tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
253+
// `tree` is followed by a delimited set of token trees. This indicates the beginning
254+
// of a repetition sequence in the macro (e.g. `$(pat)*`).
183255
Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
256+
// Must have `(` not `{` or `[`
184257
if delimited.delim != token::Paren {
185258
let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
186259
let msg = format!("expected `(`, found `{}`", tok);
187260
sess.span_diagnostic.span_err(span, &msg);
188261
}
262+
// Parse the contents of the sequence itself
189263
let sequence = parse(delimited.tts.into(), expect_matchers, sess);
264+
// Get the Kleene operator and optional separator
190265
let (separator, op) = parse_sep_and_kleene_op(trees, span, sess);
266+
// Count the number of captured "names" (i.e. named metavars)
191267
let name_captures = macro_parser::count_names(&sequence);
192-
TokenTree::Sequence(span, Rc::new(SequenceRepetition {
193-
tts: sequence,
194-
separator,
195-
op,
196-
num_captures: name_captures,
197-
}))
268+
TokenTree::Sequence(
269+
span,
270+
Rc::new(SequenceRepetition {
271+
tts: sequence,
272+
separator,
273+
op,
274+
num_captures: name_captures,
275+
}),
276+
)
198277
}
278+
279+
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
280+
// metavariable that names the crate of the invokation.
199281
Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
200282
let ident = token.ident().unwrap();
201283
let span = ident_span.with_lo(span.lo());
202284
if ident.name == keywords::Crate.name() {
203-
let ident = ast::Ident { name: keywords::DollarCrate.name(), ..ident };
285+
let ident = ast::Ident {
286+
name: keywords::DollarCrate.name(),
287+
..ident
288+
};
204289
TokenTree::Token(span, token::Ident(ident))
205290
} else {
206291
TokenTree::MetaVar(span, ident)
207292
}
208293
}
294+
295+
// `tree` is followed by a random token. This is an error.
209296
Some(tokenstream::TokenTree::Token(span, tok)) => {
210-
let msg = format!("expected identifier, found `{}`", pprust::token_to_string(&tok));
297+
let msg = format!(
298+
"expected identifier, found `{}`",
299+
pprust::token_to_string(&tok)
300+
);
211301
sess.span_diagnostic.span_err(span, &msg);
212302
TokenTree::MetaVar(span, keywords::Invalid.ident())
213303
}
304+
305+
// There are no more tokens. Just return the `$` we already have.
214306
None => TokenTree::Token(span, token::Dollar),
215307
},
308+
309+
// `tree` is an arbitrary token. Keep it.
216310
tokenstream::TokenTree::Token(span, tok) => TokenTree::Token(span, tok),
217-
tokenstream::TokenTree::Delimited(span, delimited) => {
218-
TokenTree::Delimited(span, Rc::new(Delimited {
311+
312+
// `tree` is the beginning of a delimited set of tokens (e.g. `(` or `{`). We need to
313+
// descend into the delimited set and further parse it.
314+
tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
315+
span,
316+
Rc::new(Delimited {
219317
delim: delimited.delim,
220318
tts: parse(delimited.tts.into(), expect_matchers, sess),
221-
}))
222-
}
319+
}),
320+
),
223321
}
224322
}
225323

226-
fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
227-
-> (Option<token::Token>, KleeneOp)
228-
where I: Iterator<Item = tokenstream::TokenTree>,
324+
/// Attempt to parse a single Kleene star, possibly with a separator.
325+
///
326+
/// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the
327+
/// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing
328+
/// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator
329+
/// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some
330+
/// stream of tokens in an invocation of a macro.
331+
///
332+
/// This function will take some input iterator `input` corresponding to `span` and a parsing
333+
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
334+
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
335+
/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
336+
fn parse_sep_and_kleene_op<I>(
337+
input: &mut I,
338+
span: Span,
339+
sess: &ParseSess,
340+
) -> (Option<token::Token>, KleeneOp)
341+
where
342+
I: Iterator<Item = tokenstream::TokenTree>,
229343
{
230344
fn kleene_op(token: &token::Token) -> Option<KleeneOp> {
231345
match *token {
@@ -235,20 +349,40 @@ fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
235349
}
236350
}
237351

352+
// We attempt to look at the next two token trees in `input`. I will call the first #1 and the
353+
// second #2. If #1 and #2 don't match a valid KleeneOp with/without separator, that is an
354+
// error, and we should emit an error on the most specific span possible.
238355
let span = match input.next() {
356+
// #1 is a token
239357
Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) {
358+
// #1 is a KleeneOp with no separator
240359
Some(op) => return (None, op),
360+
361+
// #1 is not a KleeneOp, but may be a separator... need to look at #2
241362
None => match input.next() {
363+
// #2 is a token
242364
Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) {
365+
// #2 is a KleeneOp, so #1 must be a separator
243366
Some(op) => return (Some(tok), op),
367+
368+
// #2 is not a KleeneOp... error
244369
None => span,
245370
},
246-
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
247-
}
371+
372+
// #2 is not a token at all... error
373+
tree => tree.as_ref()
374+
.map(tokenstream::TokenTree::span)
375+
.unwrap_or(span),
376+
},
248377
},
249-
tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span),
378+
379+
// #1 is not a token at all... error
380+
tree => tree.as_ref()
381+
.map(tokenstream::TokenTree::span)
382+
.unwrap_or(span),
250383
};
251384

385+
// Error...
252386
sess.span_diagnostic.span_err(span, "expected `*` or `+`");
253387
(None, KleeneOp::ZeroOrMore)
254388
}

0 commit comments

Comments
 (0)