|
| 1 | +/* This Source Code Form is subject to the terms of the Mozilla Public |
| 2 | + * License, v. 2.0. If a copy of the MPL was not distributed with this |
| 3 | + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
| 4 | + |
| 5 | +use quote::{ToTokens, Tokens}; |
| 6 | +use std::fs::File; |
| 7 | +use std::io::{Read, Write}; |
| 8 | +use std::path::Path; |
| 9 | +use std::vec; |
| 10 | +use std::iter; |
| 11 | +use syn; |
| 12 | + |
| 13 | +pub fn expand(from: &Path, to: &Path) { |
| 14 | + let mut source = String::new(); |
| 15 | + File::open(from).unwrap().read_to_string(&mut source).unwrap(); |
| 16 | + let tts = syn::parse_token_trees(&source).expect("Parsing rules.rs module"); |
| 17 | + let mut tokens = Tokens::new(); |
| 18 | + tokens.append_all(expand_tts(tts)); |
| 19 | + |
| 20 | + let code = tokens.to_string().replace("{ ", "{\n").replace(" }", "\n}"); |
| 21 | + File::create(to).unwrap().write_all(code.as_bytes()).unwrap(); |
| 22 | +} |
| 23 | + |
| 24 | +fn expand_tts(tts: Vec<syn::TokenTree>) -> Vec<syn::TokenTree> { |
| 25 | + use syn::*; |
| 26 | + let mut expanded = Vec::new(); |
| 27 | + let mut tts = tts.into_iter(); |
| 28 | + while let Some(tt) = tts.next() { |
| 29 | + match tt { |
| 30 | + TokenTree::Token(Token::Ident(ident)) => { |
| 31 | + if ident != "match_byte" { |
| 32 | + expanded.push(TokenTree::Token(Token::Ident(ident))); |
| 33 | + continue; |
| 34 | + } |
| 35 | + |
| 36 | + match tts.next() { |
| 37 | + Some(TokenTree::Token(Token::Not)) => {}, |
| 38 | + other => { |
| 39 | + expanded.push(TokenTree::Token(Token::Ident(ident))); |
| 40 | + if let Some(other) = other { |
| 41 | + expanded.push(other); |
| 42 | + } |
| 43 | + continue; |
| 44 | + } |
| 45 | + } |
| 46 | + |
| 47 | + let tts = match tts.next() { |
| 48 | + Some(TokenTree::Delimited(Delimited { tts, .. })) => tts, |
| 49 | + other => { |
| 50 | + expanded.push(TokenTree::Token(Token::Ident(ident))); |
| 51 | + expanded.push(TokenTree::Token(Token::Not)); |
| 52 | + if let Some(other) = other { |
| 53 | + expanded.push(other); |
| 54 | + } |
| 55 | + continue; |
| 56 | + } |
| 57 | + }; |
| 58 | + |
| 59 | + let (to_be_matched, table, cases, wildcard_binding) = parse_match_bytes_macro(tts); |
| 60 | + let expr = expand_match_bytes_macro(to_be_matched, |
| 61 | + &table, |
| 62 | + cases, |
| 63 | + wildcard_binding); |
| 64 | + |
| 65 | + let tts = syn::parse_token_trees(&expr) |
| 66 | + .expect("parsing macro expansion as token trees"); |
| 67 | + expanded.extend(expand_tts(tts)); |
| 68 | + } |
| 69 | + TokenTree::Delimited(Delimited { delim, tts }) => { |
| 70 | + expanded.push(TokenTree::Delimited(Delimited { |
| 71 | + delim: delim, |
| 72 | + tts: expand_tts(tts), |
| 73 | + })) |
| 74 | + } |
| 75 | + other => expanded.push(other), |
| 76 | + } |
| 77 | + } |
| 78 | + expanded |
| 79 | +} |
| 80 | + |
| 81 | +/// Parses a token tree corresponding to the `match_byte` macro. |
| 82 | +/// |
| 83 | +/// ## Example |
| 84 | +/// |
| 85 | +/// ```rust |
| 86 | +/// match_byte! { tokenizer.next_byte_unchecked(), |
| 87 | +/// b'a'..b'z' => { ... } |
| 88 | +/// b'0'..b'9' => { ... } |
| 89 | +/// b'\n' | b'\\' => { ... } |
| 90 | +/// foo => { ... } |
| 91 | +/// } |
| 92 | +/// |
| 93 | +/// Returns: |
| 94 | +/// * The token tree that contains the expression to be matched (in this case |
| 95 | +/// `tokenizer.next_byte_unchecked()`. |
| 96 | +/// |
| 97 | +/// * The table with the different cases per byte, each entry in the table |
| 98 | +/// contains a non-zero integer representing a different arm of the |
| 99 | +/// match expression. |
| 100 | +/// |
| 101 | +/// * The list of cases containing the expansion of the arms of the match |
| 102 | +/// expression. |
| 103 | +/// |
| 104 | +/// * An optional identifier to which the wildcard pattern is matched (`foo` in |
| 105 | +/// this case). |
| 106 | +/// |
| 107 | +fn parse_match_bytes_macro(tts: Vec<syn::TokenTree>) -> (Vec<syn::TokenTree>, [u8; 256], Vec<Case>, Option<syn::Ident>) { |
| 108 | + let mut tts = tts.into_iter(); |
| 109 | + |
| 110 | + // Grab the thing we're matching, until we find a comma. |
| 111 | + let mut left_hand_side = vec![]; |
| 112 | + loop { |
| 113 | + match tts.next() { |
| 114 | + Some(syn::TokenTree::Token(syn::Token::Comma)) => break, |
| 115 | + Some(other) => left_hand_side.push(other), |
| 116 | + None => panic!("Expected not to run out of tokens looking for a comma"), |
| 117 | + } |
| 118 | + } |
| 119 | + |
| 120 | + let mut cases = vec![]; |
| 121 | + let mut table = [0u8; 256]; |
| 122 | + |
| 123 | + let mut tts = tts.peekable(); |
| 124 | + let mut case_id: u8 = 1; |
| 125 | + let mut binding = None; |
| 126 | + while tts.len() > 0 { |
| 127 | + cases.push(parse_case(&mut tts, &mut table, &mut binding, case_id)); |
| 128 | + |
| 129 | + // Allow an optional comma between cases. |
| 130 | + match tts.peek() { |
| 131 | + Some(&syn::TokenTree::Token(syn::Token::Comma)) => { |
| 132 | + tts.next(); |
| 133 | + }, |
| 134 | + _ => {}, |
| 135 | + } |
| 136 | + |
| 137 | + case_id += 1; |
| 138 | + } |
| 139 | + |
| 140 | + (left_hand_side, table, cases, binding) |
| 141 | +} |
| 142 | + |
| 143 | +#[derive(Debug)] |
| 144 | +struct Case(Vec<syn::TokenTree>); |
| 145 | + |
| 146 | +/// Parses a single pattern => expression, and returns the case, filling in the |
| 147 | +/// table with the case id for every byte that matched. |
| 148 | +/// |
| 149 | +/// The `binding` parameter is the identifier that is used by the wildcard |
| 150 | +/// pattern. |
| 151 | +fn parse_case(tts: &mut iter::Peekable<vec::IntoIter<syn::TokenTree>>, |
| 152 | + table: &mut [u8; 256], |
| 153 | + binding: &mut Option<syn::Ident>, |
| 154 | + case_id: u8) |
| 155 | + -> Case { |
| 156 | + // The last byte checked, as part of this pattern, to properly detect |
| 157 | + // ranges. |
| 158 | + let mut last_byte: Option<u8> = None; |
| 159 | + |
| 160 | + // Loop through the pattern filling with bytes the table. |
| 161 | + loop { |
| 162 | + match tts.next() { |
| 163 | + Some(syn::TokenTree::Token(syn::Token::Literal(syn::Lit::Byte(byte)))) => { |
| 164 | + table[byte as usize] = case_id; |
| 165 | + last_byte = Some(byte); |
| 166 | + } |
| 167 | + Some(syn::TokenTree::Token(syn::Token::BinOp(syn::BinOpToken::Or))) => { |
| 168 | + last_byte = None; // This pattern is over. |
| 169 | + }, |
| 170 | + Some(syn::TokenTree::Token(syn::Token::DotDotDot)) => { |
| 171 | + assert!(last_byte.is_some(), "Expected closed range!"); |
| 172 | + match tts.next() { |
| 173 | + Some(syn::TokenTree::Token(syn::Token::Literal(syn::Lit::Byte(byte)))) => { |
| 174 | + for b in last_byte.take().unwrap()..byte { |
| 175 | + if table[b as usize] == 0 { |
| 176 | + table[b as usize] = case_id; |
| 177 | + } |
| 178 | + } |
| 179 | + if table[byte as usize] == 0 { |
| 180 | + table[byte as usize] = case_id; |
| 181 | + } |
| 182 | + } |
| 183 | + other => panic!("Expected closed range, got: {:?}", other), |
| 184 | + } |
| 185 | + }, |
| 186 | + Some(syn::TokenTree::Token(syn::Token::FatArrow)) => break, |
| 187 | + Some(syn::TokenTree::Token(syn::Token::Ident(ident))) => { |
| 188 | + assert_eq!(last_byte, None, "I don't support ranges with identifiers!"); |
| 189 | + assert_eq!(*binding, None); |
| 190 | + for mut byte in table.iter_mut() { |
| 191 | + if *byte == 0 { |
| 192 | + *byte = case_id; |
| 193 | + } |
| 194 | + } |
| 195 | + *binding = Some(ident) |
| 196 | + } |
| 197 | + Some(syn::TokenTree::Token(syn::Token::Underscore)) => { |
| 198 | + assert_eq!(last_byte, None); |
| 199 | + for mut byte in table.iter_mut() { |
| 200 | + if *byte == 0 { |
| 201 | + *byte = case_id; |
| 202 | + } |
| 203 | + } |
| 204 | + }, |
| 205 | + other => panic!("Expected literal byte, got: {:?}", other), |
| 206 | + } |
| 207 | + } |
| 208 | + |
| 209 | + match tts.next() { |
| 210 | + Some(syn::TokenTree::Delimited(syn::Delimited { delim: syn::DelimToken::Brace, tts })) => { |
| 211 | + Case(tts) |
| 212 | + } |
| 213 | + other => panic!("Expected case with braces after fat arrow, got: {:?}", other), |
| 214 | + } |
| 215 | +} |
| 216 | + |
| 217 | +fn expand_match_bytes_macro(to_be_matched: Vec<syn::TokenTree>, |
| 218 | + table: &[u8; 256], |
| 219 | + cases: Vec<Case>, |
| 220 | + binding: Option<syn::Ident>) |
| 221 | + -> String { |
| 222 | + use std::fmt::Write; |
| 223 | + |
| 224 | + assert!(!to_be_matched.is_empty()); |
| 225 | + assert!(table.iter().all(|b| *b != 0), "Incomplete pattern? Bogus code!"); |
| 226 | + |
| 227 | + // We build the expression with text since it's easier. |
| 228 | + let mut expr = "{\n".to_owned(); |
| 229 | + expr.push_str("enum Case {\n"); |
| 230 | + for (i, _) in cases.iter().enumerate() { |
| 231 | + write!(&mut expr, "Case{} = {},", i + 1, i + 1).unwrap(); |
| 232 | + } |
| 233 | + expr.push_str("}\n"); // enum Case |
| 234 | + |
| 235 | + expr.push_str("static __CASES: [Case; 256] = ["); |
| 236 | + for byte in table.iter() { |
| 237 | + write!(&mut expr, "Case::Case{}, ", *byte).unwrap(); |
| 238 | + } |
| 239 | + expr.push_str("];\n"); |
| 240 | + |
| 241 | + let mut tokens = Tokens::new(); |
| 242 | + let to_be_matched = syn::Delimited { |
| 243 | + delim: if binding.is_some() { syn::DelimToken::Brace } else { syn::DelimToken::Paren }, |
| 244 | + tts: to_be_matched |
| 245 | + }; |
| 246 | + to_be_matched.to_tokens(&mut tokens); |
| 247 | + |
| 248 | + if let Some(ref binding) = binding { |
| 249 | + write!(&mut expr, "let {} = {};\n", binding.to_string(), tokens.as_str()).unwrap(); |
| 250 | + } |
| 251 | + |
| 252 | + write!(&mut expr, "match __CASES[{} as usize] {{", match binding { |
| 253 | + Some(binding) => binding.to_string(), |
| 254 | + None => tokens.to_string(), |
| 255 | + }).unwrap(); |
| 256 | + |
| 257 | + for (i, case) in cases.into_iter().enumerate() { |
| 258 | + let mut case_tokens = Tokens::new(); |
| 259 | + let case = syn::Delimited { |
| 260 | + delim: syn::DelimToken::Brace, |
| 261 | + tts: case.0 |
| 262 | + }; |
| 263 | + case.to_tokens(&mut case_tokens); |
| 264 | + write!(&mut expr, "Case::Case{} => {},\n", i + 1, case_tokens.as_str()).unwrap(); |
| 265 | + } |
| 266 | + expr.push_str("}\n"); // match |
| 267 | + |
| 268 | + expr.push_str("}\n"); // top |
| 269 | + |
| 270 | + expr |
| 271 | +} |
0 commit comments