Skip to content

Shrink token::Lit's hash count from usize to u16. #38043

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
+39 −32
Closed

Shrink token::Lit's hash count from usize to u16. #38043

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/grammar/verify.rs
View file
Original file line number Diff line number Diff line change
@@ -75,7 +75,7 @@ fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
"ANDAND" => Token::AndAnd,
"AT" => Token::At,
"LBRACKET" => Token::OpenDelim(DelimToken::Bracket),
"LIT_STR_RAW" => Token::Literal(Lit::StrRaw(Name(0), 0), None),
"LIT_STR_RAW" => Token::Literal(0, Lit::StrRaw(Name(0)), None),
"RPAREN" => Token::CloseDelim(DelimToken::Paren),
"SLASH" => Token::BinOp(BinOpToken::Slash),
"COMMA" => Token::Comma,
@@ -112,7 +112,7 @@ fn parse_token_list(file: &str) -> HashMap<String, token::Token> {
"GT" => Token::Gt,
"LE" => Token::Le,
"LIT_BINARY" => Token::Literal(Lit::ByteStr(Name(0)), None),
"LIT_BINARY_RAW" => Token::Literal(Lit::ByteStrRaw(Name(0), 0), None),
"LIT_BINARY_RAW" => Token::Literal(0, Lit::ByteStrRaw(Name(0)), None),
"QUESTION" => Token::Question,
"SHEBANG" => Token::Shebang(Name(0)),
_ => continue,
@@ -204,16 +204,16 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, token::Token>, surrogate_
Token::BinOp(..) => Token::BinOp(str_to_binop(content)),
Token::BinOpEq(..) => Token::BinOpEq(str_to_binop(&content[..content.len() - 1])),
Token::Literal(Lit::Str_(..), n) => Token::Literal(Lit::Str_(fix(content)), n),
Token::Literal(Lit::StrRaw(..), n) => Token::Literal(Lit::StrRaw(fix(content),
count(content)), n),
Token::Literal(Lit::StrRaw(..), n) => Token::Literal(Lit::StrRaw(count(content),
fix(content)), n),
Token::Literal(Lit::Char(..), n) => Token::Literal(Lit::Char(fixchar(content)), n),
Token::Literal(Lit::Byte(..), n) => Token::Literal(Lit::Byte(fixchar(content)), n),
Token::DocComment(..) => Token::DocComment(nm),
Token::Literal(Lit::Integer(..), n) => Token::Literal(Lit::Integer(nm), n),
Token::Literal(Lit::Float(..), n) => Token::Literal(Lit::Float(nm), n),
Token::Literal(Lit::ByteStr(..), n) => Token::Literal(Lit::ByteStr(nm), n),
Token::Literal(Lit::ByteStrRaw(..), n) => Token::Literal(Lit::ByteStrRaw(fix(content),
count(content)), n),
Token::Literal(Lit::ByteStrRaw(..), n) => Token::Literal(Lit::ByteStrRaw(count(content),
fix(content)), n),
Token::Ident(..) => Token::Ident(ast::Ident::with_empty_ctxt(nm)),
Token::Lifetime(..) => Token::Lifetime(ast::Ident::with_empty_ctxt(nm)),
ref t => t.clone()
4 changes: 2 additions & 2 deletions src/librustc_incremental/calculate_svh/svh_visitor.rs
View file
Original file line number Diff line number Diff line change
@@ -1042,8 +1042,8 @@ impl<'a, 'hash, 'tcx> StrictVersionHashVisitor<'a, 'hash, 'tcx> {
token::Lit::Float(val) |
token::Lit::Str_(val) |
token::Lit::ByteStr(val) => val.as_str().hash(self.st),
token::Lit::StrRaw(val, n) |
token::Lit::ByteStrRaw(val, n) => {
token::Lit::StrRaw(n, val) |
token::Lit::ByteStrRaw(n, val) => {
val.as_str().hash(self.st);
n.hash(self.st);
}
2 changes: 1 addition & 1 deletion src/libsyntax/ast.rs
View file
Original file line number Diff line number Diff line change
@@ -1079,7 +1079,7 @@ pub enum StrStyle {
/// A raw string, like `r##"foo"##`
///
/// The uint is the number of `#` symbols used
Raw(usize)
Raw(u16)
}

/// A literal
2 changes: 1 addition & 1 deletion src/libsyntax/diagnostics/plugin.rs
View file
Original file line number Diff line number Diff line change
@@ -101,7 +101,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
},
(3, Some(&TokenTree::Token(_, token::Ident(ref code))),
Some(&TokenTree::Token(_, token::Comma)),
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(description, _), None)))) => {
Some(&TokenTree::Token(_, token::Literal(token::StrRaw(_, description), None)))) => {
(code, Some(description))
}
_ => unreachable!()
4 changes: 4 additions & 0 deletions src/libsyntax/ext/build.rs
View file
Original file line number Diff line number Diff line change
@@ -143,6 +143,7 @@ pub trait AstBuilder {
fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr>;
fn expr_isize(&self, sp: Span, i: isize) -> P<ast::Expr>;
fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr>;
fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr>;
fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr>;
fn expr_bool(&self, sp: Span, value: bool) -> P<ast::Expr>;

@@ -738,6 +739,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
fn expr_u32(&self, sp: Span, u: u32) -> P<ast::Expr> {
self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U32)))
}
fn expr_u16(&self, sp: Span, u: u16) -> P<ast::Expr> {
self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U16)))
}
fn expr_u8(&self, sp: Span, u: u8) -> P<ast::Expr> {
self.expr_lit(sp, ast::LitKind::Int(u as u64, ast::LitIntType::Unsigned(ast::UintTy::U8)))
}
6 changes: 3 additions & 3 deletions src/libsyntax/ext/quote.rs
View file
Original file line number Diff line number Diff line change
@@ -649,9 +649,9 @@ fn expr_mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
return mk_lit!("Str_", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)))
}

token::Literal(token::StrRaw(ident, n), suf) => {
return mk_lit!("StrRaw", suf, mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)),
cx.expr_usize(sp, n))
token::Literal(token::StrRaw(n, ident), suf) => {
return mk_lit!("StrRaw", suf, cx.expr_u16(sp, n),
mk_name(cx, sp, ast::Ident::with_empty_ctxt(ident)));
}

token::Ident(ident) => {
13 changes: 7 additions & 6 deletions src/libsyntax/parse/lexer/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1428,7 +1428,7 @@ impl<'a> StringReader<'a> {
Symbol::intern("??")
};
let suffix = self.scan_optional_raw_name();
return Ok(token::Literal(token::StrRaw(id, hash_count), suffix));
return Ok(token::Literal(token::StrRaw(hash_count, id), suffix));
}
'-' => {
if self.nextch_is('>') {
@@ -1597,6 +1597,7 @@ impl<'a> StringReader<'a> {
let mut hash_count = 0;
while self.ch_is('#') {
self.bump();
// njn: check for u16 overflow?
hash_count += 1;
}

@@ -1641,8 +1642,8 @@ impl<'a> StringReader<'a> {
self.bump();
}
self.bump();
return token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos),
hash_count);
return token::ByteStrRaw(hash_count,
self.name_from_to(content_start_bpos, content_end_bpos));
}
}

@@ -1849,7 +1850,7 @@ mod tests {
assert_eq!(setup(&cm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string())
.next_token()
.tok,
token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None));
token::Literal(token::StrRaw(3, Symbol::intern("\"#a\\b\x00c\"")), None));
}

#[test]
@@ -1882,10 +1883,10 @@ mod tests {
token::Literal(token::Integer(Symbol::intern("2")),
Some(Symbol::intern("us"))));
assert_eq!(setup(&cm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok,
token::Literal(token::StrRaw(Symbol::intern("raw"), 3),
token::Literal(token::StrRaw(3, Symbol::intern("raw")),
Some(Symbol::intern("suffix"))));
assert_eq!(setup(&cm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok,
token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3),
token::Literal(token::ByteStrRaw(3, Symbol::intern("raw")),
Some(Symbol::intern("suffix"))));
}

8 changes: 4 additions & 4 deletions src/libsyntax/parse/parser.rs
View file
Original file line number Diff line number Diff line change
@@ -1528,14 +1528,14 @@ impl<'a> Parser<'a> {
let s = Symbol::intern(&parse::str_lit(&s.as_str()));
(true, LitKind::Str(s, ast::StrStyle::Cooked))
}
token::StrRaw(s, n) => {
token::StrRaw(n, s) => {
let s = Symbol::intern(&parse::raw_str_lit(&s.as_str()));
(true, LitKind::Str(s, ast::StrStyle::Raw(n)))
}
token::ByteStr(i) => {
(true, LitKind::ByteStr(parse::byte_str_lit(&i.as_str())))
}
token::ByteStrRaw(i, _) => {
token::ByteStrRaw(_, i) => {
(true, LitKind::ByteStr(Rc::new(i.to_string().into_bytes())))
}
};
@@ -5654,7 +5654,7 @@ impl<'a> Parser<'a> {
/// the `extern` keyword, if one is found.
fn parse_opt_abi(&mut self) -> PResult<'a, Option<abi::Abi>> {
match self.token {
token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => {
token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(_, s), suf) => {
let sp = self.span;
self.expect_no_suffix(sp, "ABI spec", suf);
self.bump();
@@ -6151,7 +6151,7 @@ impl<'a> Parser<'a> {
pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
let ret = match self.token {
token::Literal(token::Str_(s), suf) => (s, ast::StrStyle::Cooked, suf),
token::Literal(token::StrRaw(s, n), suf) => (s, ast::StrStyle::Raw(n), suf),
token::Literal(token::StrRaw(n, s), suf) => (s, ast::StrStyle::Raw(n), suf),
_ => return None
};
self.bump();
4 changes: 2 additions & 2 deletions src/libsyntax/parse/token.rs
View file
Original file line number Diff line number Diff line change
@@ -56,9 +56,9 @@ pub enum Lit {
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, usize), /* raw str delimited by n hash symbols */
StrRaw(u16, ast::Name), /* raw str delimited by n hash symbols */
ByteStr(ast::Name),
ByteStrRaw(ast::Name, usize), /* raw byte str delimited by n hash symbols */
ByteStrRaw(u16, ast::Name), /* raw byte str delimited by n hash symbols */
}

impl Lit {
11 changes: 6 additions & 5 deletions src/libsyntax/print/pprust.rs
View file
Original file line number Diff line number Diff line change
@@ -255,12 +255,13 @@ pub fn token_to_string(tok: &Token) -> String {
token::Float(c) => c.to_string(),
token::Integer(c) => c.to_string(),
token::Str_(s) => format!("\"{}\"", s),
token::StrRaw(s, n) => format!("r{delim}\"{string}\"{delim}",
delim=repeat("#", n),
token::StrRaw(n, s) => format!("r{delim}\"{string}\"{delim}",
delim=repeat("#", n as usize),
string=s),
token::ByteStr(v) => format!("b\"{}\"", v),
token::ByteStrRaw(s, n) => format!("br{delim}\"{string}\"{delim}",
delim=repeat("#", n),
token::ByteStrRaw(n, s) => format!("br{delim}\"{string}\"{delim}",
delim=repeat("#", n as
usize),
string=s),
};

@@ -687,7 +688,7 @@ pub trait PrintState<'a> {
}
ast::StrStyle::Raw(n) => {
(format!("r{delim}\"{string}\"{delim}",
delim=repeat("#", n),
delim=repeat("#", n as usize),
string=st))
}
};
5 changes: 3 additions & 2 deletions src/libsyntax/tokenstream.rs
View file
Original file line number Diff line number Diff line change
@@ -157,6 +157,7 @@ impl TokenTree {

// Searches for the occurrences of `"#*` and returns the minimum number of `#`s
// required to wrap the text.
// njn: check for u16 overflow?
let num_of_hashes = stripped.chars()
.scan(0, |cnt, x| {
*cnt = if x == '"' {
@@ -177,7 +178,7 @@ impl TokenTree {
tts: vec![TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))),
TokenTree::Token(sp, token::Eq),
TokenTree::Token(sp, token::Literal(
token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))],
token::StrRaw(num_of_hashes, Symbol::intern(&stripped)), None))],
close_span: sp,
}))
}
@@ -303,7 +304,7 @@ impl TokenTree {
span: sp,
})
}
TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => {
TokenTree::Token(sp, Token::Literal(Lit::StrRaw(n, s), _)) => {
let l = LitKind::Str(Symbol::intern(&parse::raw_str_lit(&s.as_str())),
ast::StrStyle::Raw(n));
Some(Spanned {