10
10
11
11
use ast;
12
12
use ext:: tt:: macro_parser;
13
- use parse:: { ParseSess , token } ;
13
+ use parse:: { token , ParseSess } ;
14
14
use print:: pprust;
15
15
use symbol:: keywords;
16
- use syntax_pos:: { DUMMY_SP , Span , BytePos } ;
16
+ use syntax_pos:: { BytePos , Span , DUMMY_SP } ;
17
17
use tokenstream;
18
18
19
19
use std:: rc:: Rc ;
20
20
21
+ /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
22
+ /// that the delimiter itself might be `NoDelim`.
21
23
#[ derive( Clone , PartialEq , Eq , RustcEncodable , RustcDecodable , Hash , Debug ) ]
22
24
pub struct Delimited {
23
25
pub delim : token:: DelimToken ,
24
26
pub tts : Vec < TokenTree > ,
25
27
}
26
28
27
29
impl Delimited {
30
+ /// Return the opening delimiter (possibly `NoDelim`).
28
31
pub fn open_token ( & self ) -> token:: Token {
29
32
token:: OpenDelim ( self . delim )
30
33
}
31
34
35
+ /// Return the closing delimiter (possibly `NoDelim`).
32
36
pub fn close_token ( & self ) -> token:: Token {
33
37
token:: CloseDelim ( self . delim )
34
38
}
35
39
40
+ /// Return a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
36
41
pub fn open_tt ( & self , span : Span ) -> TokenTree {
37
42
let open_span = if span == DUMMY_SP {
38
43
DUMMY_SP
@@ -42,6 +47,7 @@ impl Delimited {
42
47
TokenTree :: Token ( open_span, self . open_token ( ) )
43
48
}
44
49
50
+ /// Return a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
45
51
pub fn close_tt ( & self , span : Span ) -> TokenTree {
46
52
let close_span = if span == DUMMY_SP {
47
53
DUMMY_SP
@@ -68,12 +74,14 @@ pub struct SequenceRepetition {
68
74
/// for token sequences.
69
75
#[ derive( Clone , PartialEq , Eq , RustcEncodable , RustcDecodable , Hash , Debug , Copy ) ]
70
76
pub enum KleeneOp {
77
+ /// Kleene star (`*`) for zero or more repetitions
71
78
ZeroOrMore ,
79
+ /// Kleene plus (`+`) for one or more repetitions
72
80
OneOrMore ,
73
81
}
74
82
75
83
/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
76
- /// are "first-class" token trees.
84
+ /// are "first-class" token trees. Useful for parsing macros.
77
85
#[ derive( Debug , Clone , PartialEq , Eq , RustcEncodable , RustcDecodable , Hash ) ]
78
86
pub enum TokenTree {
79
87
Token ( Span , token:: Token ) ,
@@ -83,10 +91,15 @@ pub enum TokenTree {
83
91
/// E.g. `$var`
84
92
MetaVar ( Span , ast:: Ident ) ,
85
93
/// E.g. `$var:expr`. This is only used in the left hand side of MBE macros.
86
- MetaVarDecl ( Span , ast:: Ident /* name to bind */ , ast:: Ident /* kind of nonterminal */ ) ,
94
+ MetaVarDecl (
95
+ Span ,
96
+ ast:: Ident , /* name to bind */
97
+ ast:: Ident , /* kind of nonterminal */
98
+ ) ,
87
99
}
88
100
89
101
impl TokenTree {
102
+ /// Return the number of tokens in the tree.
90
103
pub fn len ( & self ) -> usize {
91
104
match * self {
92
105
TokenTree :: Delimited ( _, ref delimed) => match delimed. delim {
@@ -98,6 +111,8 @@ impl TokenTree {
98
111
}
99
112
}
100
113
114
+ /// Returns true if the given token tree contains no other tokens. This is vacuously true for
115
+ /// single tokens or metavar/decls, but may be false for delimited trees or sequences.
101
116
pub fn is_empty ( & self ) -> bool {
102
117
match * self {
103
118
TokenTree :: Delimited ( _, ref delimed) => match delimed. delim {
@@ -109,6 +124,7 @@ impl TokenTree {
109
124
}
110
125
}
111
126
127
+ /// Get the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
112
128
pub fn get_tt ( & self , index : usize ) -> TokenTree {
113
129
match ( self , index) {
114
130
( & TokenTree :: Delimited ( _, ref delimed) , _) if delimed. delim == token:: NoDelim => {
@@ -131,21 +147,48 @@ impl TokenTree {
131
147
/// Retrieve the `TokenTree`'s span.
132
148
pub fn span ( & self ) -> Span {
133
149
match * self {
134
- TokenTree :: Token ( sp, _) |
135
- TokenTree :: MetaVar ( sp, _) |
136
- TokenTree :: MetaVarDecl ( sp, _, _) |
137
- TokenTree :: Delimited ( sp, _) |
138
- TokenTree :: Sequence ( sp, _) => sp,
150
+ TokenTree :: Token ( sp, _)
151
+ | TokenTree :: MetaVar ( sp, _)
152
+ | TokenTree :: MetaVarDecl ( sp, _, _)
153
+ | TokenTree :: Delimited ( sp, _)
154
+ | TokenTree :: Sequence ( sp, _) => sp,
139
155
}
140
156
}
141
157
}
142
158
143
- pub fn parse ( input : tokenstream:: TokenStream , expect_matchers : bool , sess : & ParseSess )
144
- -> Vec < TokenTree > {
159
+ /// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this
160
+ /// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
161
+ /// collection of `TokenTree` for use in parsing a macro.
162
+ ///
163
+ /// # Parameters
164
+ ///
165
+ /// - `input`: a token stream to read from, the contents of which we are parsing.
166
+ /// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a
167
+ /// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with
168
+ /// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and
169
+ /// `ident` are "matchers". They are not present in the body of a macro rule -- just in the
170
+ /// pattern, so we pass a parameter to indicate whether to expect them or not.
171
+ /// - `sess`: the parsing session. Any errors will be emitted to this session.
172
+ ///
173
+ /// # Returns
174
+ ///
175
+ /// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
176
+ pub fn parse (
177
+ input : tokenstream:: TokenStream ,
178
+ expect_matchers : bool ,
179
+ sess : & ParseSess ,
180
+ ) -> Vec < TokenTree > {
181
+ // Will contain the final collection of `self::TokenTree`
145
182
let mut result = Vec :: new ( ) ;
183
+
184
+ // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
185
+ // additional trees if need be.
146
186
let mut trees = input. trees ( ) ;
147
187
while let Some ( tree) = trees. next ( ) {
148
188
let tree = parse_tree ( tree, & mut trees, expect_matchers, sess) ;
189
+
190
+ // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
191
+ // parse out the matcher (i.e. in `$id:ident` this would parse the `:` and `ident`).
149
192
match tree {
150
193
TokenTree :: MetaVar ( start_sp, ident) if expect_matchers => {
151
194
let span = match trees. next ( ) {
@@ -154,78 +197,149 @@ pub fn parse(input: tokenstream::TokenStream, expect_matchers: bool, sess: &Pars
154
197
Some ( kind) => {
155
198
let span = end_sp. with_lo ( start_sp. lo ( ) ) ;
156
199
result. push ( TokenTree :: MetaVarDecl ( span, ident, kind) ) ;
157
- continue
200
+ continue ;
158
201
}
159
202
_ => end_sp,
160
203
} ,
161
- tree => tree. as_ref ( ) . map ( tokenstream:: TokenTree :: span) . unwrap_or ( span) ,
204
+ tree => tree. as_ref ( )
205
+ . map ( tokenstream:: TokenTree :: span)
206
+ . unwrap_or ( span) ,
162
207
} ,
163
- tree => tree. as_ref ( ) . map ( tokenstream:: TokenTree :: span) . unwrap_or ( start_sp) ,
208
+ tree => tree. as_ref ( )
209
+ . map ( tokenstream:: TokenTree :: span)
210
+ . unwrap_or ( start_sp) ,
164
211
} ;
165
212
sess. missing_fragment_specifiers . borrow_mut ( ) . insert ( span) ;
166
- result. push ( TokenTree :: MetaVarDecl ( span, ident, keywords:: Invalid . ident ( ) ) ) ;
213
+ result. push ( TokenTree :: MetaVarDecl (
214
+ span,
215
+ ident,
216
+ keywords:: Invalid . ident ( ) ,
217
+ ) ) ;
167
218
}
219
+
220
+ // Not a metavar or no matchers allowed, so just return the tree
168
221
_ => result. push ( tree) ,
169
222
}
170
223
}
171
224
result
172
225
}
173
226
174
- fn parse_tree < I > ( tree : tokenstream:: TokenTree ,
175
- trees : & mut I ,
176
- expect_matchers : bool ,
177
- sess : & ParseSess )
178
- -> TokenTree
179
- where I : Iterator < Item = tokenstream:: TokenTree > ,
227
+ /// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a
228
+ /// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree`
229
+ /// for use in parsing a macro.
230
+ ///
231
+ /// Converting the given tree may involve reading more tokens.
232
+ ///
233
+ /// # Parameters
234
+ ///
235
+ /// - `tree`: the tree we wish to convert.
236
+ /// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish
237
+ /// converting `tree`
238
+ /// - `expect_matchers`: same as for `parse` (see above).
239
+ /// - `sess`: the parsing session. Any errors will be emitted to this session.
240
+ fn parse_tree < I > (
241
+ tree : tokenstream:: TokenTree ,
242
+ trees : & mut I ,
243
+ expect_matchers : bool ,
244
+ sess : & ParseSess ,
245
+ ) -> TokenTree
246
+ where
247
+ I : Iterator < Item = tokenstream:: TokenTree > ,
180
248
{
249
+ // Depending on what `tree` is, we could be parsing different parts of a macro
181
250
match tree {
251
+ // `tree` is a `$` token. Look at the next token in `trees`
182
252
tokenstream:: TokenTree :: Token ( span, token:: Dollar ) => match trees. next ( ) {
253
+ // `tree` is followed by a delimited set of token trees. This indicates the beginning
254
+ // of a repetition sequence in the macro (e.g. `$(pat)*`).
183
255
Some ( tokenstream:: TokenTree :: Delimited ( span, delimited) ) => {
256
+ // Must have `(` not `{` or `[`
184
257
if delimited. delim != token:: Paren {
185
258
let tok = pprust:: token_to_string ( & token:: OpenDelim ( delimited. delim ) ) ;
186
259
let msg = format ! ( "expected `(`, found `{}`" , tok) ;
187
260
sess. span_diagnostic . span_err ( span, & msg) ;
188
261
}
262
+ // Parse the contents of the sequence itself
189
263
let sequence = parse ( delimited. tts . into ( ) , expect_matchers, sess) ;
264
+ // Get the Kleene operator and optional separator
190
265
let ( separator, op) = parse_sep_and_kleene_op ( trees, span, sess) ;
266
+ // Count the number of captured "names" (i.e. named metavars)
191
267
let name_captures = macro_parser:: count_names ( & sequence) ;
192
- TokenTree :: Sequence ( span, Rc :: new ( SequenceRepetition {
193
- tts : sequence,
194
- separator,
195
- op,
196
- num_captures : name_captures,
197
- } ) )
268
+ TokenTree :: Sequence (
269
+ span,
270
+ Rc :: new ( SequenceRepetition {
271
+ tts : sequence,
272
+ separator,
273
+ op,
274
+ num_captures : name_captures,
275
+ } ) ,
276
+ )
198
277
}
278
+
279
+ // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
280
+ // metavariable that names the crate of the invokation.
199
281
Some ( tokenstream:: TokenTree :: Token ( ident_span, ref token) ) if token. is_ident ( ) => {
200
282
let ident = token. ident ( ) . unwrap ( ) ;
201
283
let span = ident_span. with_lo ( span. lo ( ) ) ;
202
284
if ident. name == keywords:: Crate . name ( ) {
203
- let ident = ast:: Ident { name : keywords:: DollarCrate . name ( ) , ..ident } ;
285
+ let ident = ast:: Ident {
286
+ name : keywords:: DollarCrate . name ( ) ,
287
+ ..ident
288
+ } ;
204
289
TokenTree :: Token ( span, token:: Ident ( ident) )
205
290
} else {
206
291
TokenTree :: MetaVar ( span, ident)
207
292
}
208
293
}
294
+
295
+ // `tree` is followed by a random token. This is an error.
209
296
Some ( tokenstream:: TokenTree :: Token ( span, tok) ) => {
210
- let msg = format ! ( "expected identifier, found `{}`" , pprust:: token_to_string( & tok) ) ;
297
+ let msg = format ! (
298
+ "expected identifier, found `{}`" ,
299
+ pprust:: token_to_string( & tok)
300
+ ) ;
211
301
sess. span_diagnostic . span_err ( span, & msg) ;
212
302
TokenTree :: MetaVar ( span, keywords:: Invalid . ident ( ) )
213
303
}
304
+
305
+ // There are no more tokens. Just return the `$` we already have.
214
306
None => TokenTree :: Token ( span, token:: Dollar ) ,
215
307
} ,
308
+
309
+ // `tree` is an arbitrary token. Keep it.
216
310
tokenstream:: TokenTree :: Token ( span, tok) => TokenTree :: Token ( span, tok) ,
217
- tokenstream:: TokenTree :: Delimited ( span, delimited) => {
218
- TokenTree :: Delimited ( span, Rc :: new ( Delimited {
311
+
312
+ // `tree` is the beginning of a delimited set of tokens (e.g. `(` or `{`). We need to
313
+ // descend into the delimited set and further parse it.
314
+ tokenstream:: TokenTree :: Delimited ( span, delimited) => TokenTree :: Delimited (
315
+ span,
316
+ Rc :: new ( Delimited {
219
317
delim : delimited. delim ,
220
318
tts : parse ( delimited. tts . into ( ) , expect_matchers, sess) ,
221
- } ) )
222
- }
319
+ } ) ,
320
+ ) ,
223
321
}
224
322
}
225
323
226
- fn parse_sep_and_kleene_op < I > ( input : & mut I , span : Span , sess : & ParseSess )
227
- -> ( Option < token:: Token > , KleeneOp )
228
- where I : Iterator < Item = tokenstream:: TokenTree > ,
324
+ /// Attempt to parse a single Kleene star, possibly with a separator.
325
+ ///
326
+ /// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the
327
+ /// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing
328
+ /// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator
329
+ /// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some
330
+ /// stream of tokens in an invocation of a macro.
331
+ ///
332
+ /// This function will take some input iterator `input` corresponding to `span` and a parsing
333
+ /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
334
+ /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
335
+ /// error with the appropriate span is emitted to `sess` and a dummy value is returned.
336
+ fn parse_sep_and_kleene_op < I > (
337
+ input : & mut I ,
338
+ span : Span ,
339
+ sess : & ParseSess ,
340
+ ) -> ( Option < token:: Token > , KleeneOp )
341
+ where
342
+ I : Iterator < Item = tokenstream:: TokenTree > ,
229
343
{
230
344
fn kleene_op ( token : & token:: Token ) -> Option < KleeneOp > {
231
345
match * token {
@@ -235,20 +349,40 @@ fn parse_sep_and_kleene_op<I>(input: &mut I, span: Span, sess: &ParseSess)
235
349
}
236
350
}
237
351
352
+ // We attempt to look at the next two token trees in `input`. I will call the first #1 and the
353
+ // second #2. If #1 and #2 don't match a valid KleeneOp with/without separator, that is an
354
+ // error, and we should emit an error on the most specific span possible.
238
355
let span = match input. next ( ) {
356
+ // #1 is a token
239
357
Some ( tokenstream:: TokenTree :: Token ( span, tok) ) => match kleene_op ( & tok) {
358
+ // #1 is a KleeneOp with no separator
240
359
Some ( op) => return ( None , op) ,
360
+
361
+ // #1 is not a KleeneOp, but may be a separator... need to look at #2
241
362
None => match input. next ( ) {
363
+ // #2 is a token
242
364
Some ( tokenstream:: TokenTree :: Token ( span, tok2) ) => match kleene_op ( & tok2) {
365
+ // #2 is a KleeneOp, so #1 must be a separator
243
366
Some ( op) => return ( Some ( tok) , op) ,
367
+
368
+ // #2 is not a KleeneOp... error
244
369
None => span,
245
370
} ,
246
- tree => tree. as_ref ( ) . map ( tokenstream:: TokenTree :: span) . unwrap_or ( span) ,
247
- }
371
+
372
+ // #2 is not a token at all... error
373
+ tree => tree. as_ref ( )
374
+ . map ( tokenstream:: TokenTree :: span)
375
+ . unwrap_or ( span) ,
376
+ } ,
248
377
} ,
249
- tree => tree. as_ref ( ) . map ( tokenstream:: TokenTree :: span) . unwrap_or ( span) ,
378
+
379
+ // #1 is not a token at all... error
380
+ tree => tree. as_ref ( )
381
+ . map ( tokenstream:: TokenTree :: span)
382
+ . unwrap_or ( span) ,
250
383
} ;
251
384
385
+ // Error...
252
386
sess. span_diagnostic . span_err ( span, "expected `*` or `+`" ) ;
253
387
( None , KleeneOp :: ZeroOrMore )
254
388
}
0 commit comments