Auto merge of #52552 - eddyb:proc-macro-prep, r=alexcrichton

Prepare proc_macro for decoupling it from the rest of the compiler.

This is #49219 up to the point where the bridge is introduced. Aside from moving some code around, the largest change is the rewrite of `proc_macro::quote` to be simpler and do less introspection.

I'd like to also extend `quote!` with `${stmt;...;expr}` instead of just `$variable` (and maybe even `$(... $iter ...)*`), which seems pretty straight-forward now, but I don't know if/when I should.

r? @alexcrichton or @dtolnay cc @jseyfried @petrochenkov

Author: bors

src/libproc_macro/diagnostic.rs

@@ -10,7 +10,8 @@
 
 use Span;
 
-use rustc_errors as rustc;
+use rustc_errors as errors;
+use syntax_pos::MultiSpan;
 
 /// An enum representing a diagnostic level.
 #[unstable(feature = "proc_macro_diagnostic", issue = "38356")]
@@ -97,38 +98,21 @@ impl Diagnostic {
     /// Emit the diagnostic.
     #[unstable(feature = "proc_macro_diagnostic", issue = "38356")]
     pub fn emit(self) {
-        ::__internal::with_sess(move |sess, _| {
-            let handler = &sess.span_diagnostic;
-            let level = __internal::level_to_internal_level(self.level);
-            let mut diag = rustc::DiagnosticBuilder::new(handler, level, &*self.message);
+        let level = self.level.to_internal();
+        let mut diag = errors::Diagnostic::new(level, &*self.message);
 
-            if let Some(span) = self.span {
-                diag.set_span(span.0);
-            }
+        if let Some(span) = self.span {
+            diag.set_span(span.0);
+        }
 
-            for child in self.children {
-                let span = child.span.map(|s| s.0);
-                let level = __internal::level_to_internal_level(child.level);
-                diag.sub(level, &*child.message, span);
-            }
+        for child in self.children {
+            let span = child.span.map_or(MultiSpan::new(), |s| s.0.into());
+            let level = child.level.to_internal();
+            diag.sub(level, &*child.message, span, None);
+        }
 
-            diag.emit();
+        ::__internal::with_sess(move |sess, _| {
+            errors::DiagnosticBuilder::new_diagnostic(&sess.span_diagnostic, diag).emit();
         });
     }
 }
-
-#[unstable(feature = "proc_macro_internals", issue = "27812")]
-#[doc(hidden)]
-pub mod __internal {
-    use super::{Level, rustc};
-
-    pub fn level_to_internal_level(level: Level) -> rustc::Level {
-        match level {
-            Level::Error => rustc::Level::Error,
-            Level::Warning => rustc::Level::Warning,
-            Level::Note => rustc::Level::Note,
-            Level::Help => rustc::Level::Help,
-            Level::__Nonexhaustive => unreachable!("Level::__Nonexhaustive")
-        }
-    }
-}

src/libproc_macro/lib.rs

@@ -14,35 +14,26 @@
 //! This quasiquoter uses macros 2.0 hygiene to reliably access
 //! items from `proc_macro`, to build a `proc_macro::TokenStream`.
 
-use {Delimiter, Literal, Spacing, Span, Ident, Punct, Group, TokenStream, TokenTree};
-
-use syntax::ext::base::{ExtCtxt, ProcMacro};
-use syntax::parse::token;
-use syntax::symbol::Symbol;
-use syntax::tokenstream;
-
-/// This is the actual quote!() proc macro
-///
-/// It is manually loaded in CStore::load_macro_untracked
-pub struct Quoter;
-
-pub fn unquote<T: Into<TokenStream> + Clone>(tokens: &T) -> TokenStream {
-    tokens.clone().into()
-}
-
-pub trait Quote {
-    fn quote(self) -> TokenStream;
-}
-
-macro_rules! tt2ts {
-    ($e:expr) => (TokenStream::from(TokenTree::from($e)))
-}
-
-macro_rules! quote_tok {
-    (,) => { tt2ts!(Punct::new(',', Spacing::Alone)) };
-    (.) => { tt2ts!(Punct::new('.', Spacing::Alone)) };
-    (:) => { tt2ts!(Punct::new(':', Spacing::Alone)) };
-    (|) => { tt2ts!(Punct::new('|', Spacing::Alone)) };
+use {Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
+
+macro_rules! quote_tt {
+    (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, quote!($($t)*)) };
+    ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, quote!($($t)*)) };
+    ({$($t:tt)*}) => { Group::new(Delimiter::Brace, quote!($($t)*)) };
+    (,) => { Punct::new(',', Spacing::Alone) };
+    (.) => { Punct::new('.', Spacing::Alone) };
+    (:) => { Punct::new(':', Spacing::Alone) };
+    (;) => { Punct::new(';', Spacing::Alone) };
+    (!) => { Punct::new('!', Spacing::Alone) };
+    (<) => { Punct::new('<', Spacing::Alone) };
+    (>) => { Punct::new('>', Spacing::Alone) };
+    (&) => { Punct::new('&', Spacing::Alone) };
+    (=) => { Punct::new('=', Spacing::Alone) };
+    ($i:ident) => { Ident::new(stringify!($i), Span::def_site()) };
+}
+
+macro_rules! quote_ts {
+    ((@ $($t:tt)*)) => { $($t)* };
     (::) => {
         [
             TokenTree::from(Punct::new(':', Spacing::Joint)),
@@ -55,65 +46,45 @@ macro_rules! quote_tok {
             })
             .collect::<TokenStream>()
     };
-    (!) => { tt2ts!(Punct::new('!', Spacing::Alone)) };
-    (<) => { tt2ts!(Punct::new('<', Spacing::Alone)) };
-    (>) => { tt2ts!(Punct::new('>', Spacing::Alone)) };
-    (_) => { tt2ts!(Punct::new('_', Spacing::Alone)) };
-    (0) => { tt2ts!(Literal::i8_unsuffixed(0)) };
-    (&) => { tt2ts!(Punct::new('&', Spacing::Alone)) };
-    ($i:ident) => { tt2ts!(Ident::new(stringify!($i), Span::def_site())) };
-}
-
-macro_rules! quote_tree {
-    ((unquote $($t:tt)*)) => { $($t)* };
-    ((quote $($t:tt)*)) => { ($($t)*).quote() };
-    (($($t:tt)*)) => { tt2ts!(Group::new(Delimiter::Parenthesis, quote!($($t)*))) };
-    ([$($t:tt)*]) => { tt2ts!(Group::new(Delimiter::Bracket, quote!($($t)*))) };
-    ({$($t:tt)*}) => { tt2ts!(Group::new(Delimiter::Brace, quote!($($t)*))) };
-    ($t:tt) => { quote_tok!($t) };
+    ($t:tt) => { TokenTree::from(quote_tt!($t)) };
 }
 
+/// Simpler version of the real `quote!` macro, implemented solely
+/// through `macro_rules`, for bootstrapping the real implementation
+/// (see the `quote` function), which does not have access to the
+/// real `quote!` macro due to the `proc_macro` crate not being
+/// able to depend on itself.
+///
+/// Note: supported tokens are a subset of the real `quote!`, but
+/// unquoting is different: instead of `$x`, this uses `(@ expr)`.
 macro_rules! quote {
     () => { TokenStream::new() };
     ($($t:tt)*) => {
-        [$(quote_tree!($t),)*].iter()
-            .cloned()
-            .flat_map(|x| x.into_iter())
-            .collect::<TokenStream>()
+        [
+            $(TokenStream::from(quote_ts!($t)),)*
+        ].iter().cloned().collect::<TokenStream>()
     };
 }
 
-impl ProcMacro for Quoter {
-    fn expand<'cx>(&self, cx: &'cx mut ExtCtxt,
-                   _: ::syntax_pos::Span,
-                   stream: tokenstream::TokenStream)
-                   -> tokenstream::TokenStream {
-        ::__internal::set_sess(cx, || TokenStream(stream).quote().0)
-    }
-}
-
-impl<T: Quote> Quote for Option<T> {
-    fn quote(self) -> TokenStream {
-        match self {
-            Some(t) => quote!(Some((quote t))),
-            None => quote!(None),
-        }
+/// Quote a `TokenStream` into a `TokenStream`.
+/// This is the actual `quote!()` proc macro.
+///
+/// It is manually loaded in `CStore::load_macro_untracked`.
+#[unstable(feature = "proc_macro_quote", issue = "38356")]
+pub fn quote(stream: TokenStream) -> TokenStream {
+    if stream.is_empty() {
+        return quote!(::TokenStream::new());
     }
-}
-
-impl Quote for TokenStream {
-    fn quote(self) -> TokenStream {
-        if self.is_empty() {
-            return quote!(::TokenStream::new());
-        }
-        let mut after_dollar = false;
-        let tokens = self.into_iter().filter_map(|tree| {
+    let mut after_dollar = false;
+    let tokens = stream
+        .into_iter()
+        .filter_map(|tree| {
             if after_dollar {
                 after_dollar = false;
                 match tree {
                     TokenTree::Ident(_) => {
-                        let tree = TokenStream::from(tree);
-                        return Some(quote!(::__internal::unquote(&(unquote tree)),));
+                        return Some(quote!(Into::<::TokenStream>::into(
+                        Clone::clone(&(@ tree))),));
                     }
                     TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
                     _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
@@ -125,186 +96,55 @@ impl Quote for TokenStream {
                 }
             }
 
-            Some(quote!(::TokenStream::from((quote tree)),))
-        }).flat_map(|t| t.into_iter()).collect::<TokenStream>();
-
-        if after_dollar {
-            panic!("unexpected trailing `$` in `quote!`");
-        }
-
-        quote!(
-            [(unquote tokens)].iter()
-                .cloned()
-                .flat_map(|x| x.into_iter())
-                .collect::<::TokenStream>()
-        )
-    }
-}
-
-impl Quote for TokenTree {
-    fn quote(self) -> TokenStream {
-        match self {
-            TokenTree::Punct(tt) => quote!(::TokenTree::Punct( (quote tt) )),
-            TokenTree::Group(tt) => quote!(::TokenTree::Group( (quote tt) )),
-            TokenTree::Ident(tt) => quote!(::TokenTree::Ident( (quote tt) )),
-            TokenTree::Literal(tt) => quote!(::TokenTree::Literal( (quote tt) )),
-        }
-    }
-}
-
-impl Quote for char {
-    fn quote(self) -> TokenStream {
-        TokenTree::from(Literal::character(self)).into()
-    }
-}
-
-impl<'a> Quote for &'a str {
-    fn quote(self) -> TokenStream {
-        TokenTree::from(Literal::string(self)).into()
-    }
-}
-
-impl Quote for u16 {
-    fn quote(self) -> TokenStream {
-        TokenTree::from(Literal::u16_unsuffixed(self)).into()
-    }
-}
-
-impl Quote for Group {
-    fn quote(self) -> TokenStream {
-        quote!(::Group::new((quote self.delimiter()), (quote self.stream())))
-    }
-}
-
-impl Quote for Punct {
-    fn quote(self) -> TokenStream {
-        quote!(::Punct::new((quote self.as_char()), (quote self.spacing())))
-    }
-}
-
-impl Quote for Ident {
-    fn quote(self) -> TokenStream {
-        quote!(::Ident::new((quote self.sym.as_str()), (quote self.span())))
-    }
-}
-
-impl Quote for Span {
-    fn quote(self) -> TokenStream {
-        quote!(::Span::def_site())
-    }
-}
-
-macro_rules! literals {
-    ($($i:ident),*; $($raw:ident),*) => {
-        pub struct SpannedSymbol {
-            sym: Symbol,
-            span: Span,
-        }
-
-        impl SpannedSymbol {
-            pub fn new(string: &str, span: Span) -> SpannedSymbol {
-                SpannedSymbol { sym: Symbol::intern(string), span }
-            }
-        }
-
-        impl Quote for SpannedSymbol {
-            fn quote(self) -> TokenStream {
-                quote!(::__internal::SpannedSymbol::new((quote self.sym.as_str()),
-                                                        (quote self.span)))
-            }
-        }
-
-        pub enum LiteralKind {
-            $($i,)*
-            $($raw(u16),)*
-        }
-
-        impl LiteralKind {
-            pub fn with_contents_and_suffix(self, contents: SpannedSymbol,
-                                            suffix: Option<SpannedSymbol>) -> Literal {
-                let sym = contents.sym;
-                let suffix = suffix.map(|t| t.sym);
-                match self {
-                    $(LiteralKind::$i => {
-                        Literal {
-                            lit: token::Lit::$i(sym),
-                            suffix,
-                            span: contents.span,
-                        }
-                    })*
-                    $(LiteralKind::$raw(n) => {
-                        Literal {
-                            lit: token::Lit::$raw(sym, n),
-                            suffix,
-                            span: contents.span,
-                        }
-                    })*
-                }
-            }
-        }
-
-        impl Literal {
-            fn kind_contents_and_suffix(self) -> (LiteralKind, SpannedSymbol, Option<SpannedSymbol>)
-            {
-                let (kind, contents) = match self.lit {
-                    $(token::Lit::$i(contents) => (LiteralKind::$i, contents),)*
-                    $(token::Lit::$raw(contents, n) => (LiteralKind::$raw(n), contents),)*
-                };
-                let suffix = self.suffix.map(|sym| SpannedSymbol::new(&sym.as_str(), self.span()));
-                (kind, SpannedSymbol::new(&contents.as_str(), self.span()), suffix)
-            }
-        }
-
-        impl Quote for LiteralKind {
-            fn quote(self) -> TokenStream {
-                match self {
-                    $(LiteralKind::$i => quote! {
-                        ::__internal::LiteralKind::$i
-                    },)*
-                    $(LiteralKind::$raw(n) => quote! {
-                        ::__internal::LiteralKind::$raw((quote n))
-                    },)*
-                }
-            }
-        }
+            Some(quote!(::TokenStream::from((@ match tree {
+                TokenTree::Punct(tt) => quote!(::TokenTree::Punct(::Punct::new(
+                    (@ TokenTree::from(Literal::character(tt.as_char()))),
+                    (@ match tt.spacing() {
+                        Spacing::Alone => quote!(::Spacing::Alone),
+                        Spacing::Joint => quote!(::Spacing::Joint),
+                    }),
+                ))),
+                TokenTree::Group(tt) => quote!(::TokenTree::Group(::Group::new(
+                    (@ match tt.delimiter() {
+                        Delimiter::Parenthesis => quote!(::Delimiter::Parenthesis),
+                        Delimiter::Brace => quote!(::Delimiter::Brace),
+                        Delimiter::Bracket => quote!(::Delimiter::Bracket),
+                        Delimiter::None => quote!(::Delimiter::None),
+                    }),
+                    (@ quote(tt.stream())),
+                ))),
+                TokenTree::Ident(tt) => quote!(::TokenTree::Ident(::Ident::new(
+                    (@ TokenTree::from(Literal::string(&tt.to_string()))),
+                    (@ quote_span(tt.span())),
+                ))),
+                TokenTree::Literal(tt) => quote!(::TokenTree::Literal({
+                    let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
+                        .parse::<::TokenStream>()
+                        .unwrap()
+                        .into_iter();
+                    if let (Some(::TokenTree::Literal(mut lit)), None) =
+                        (iter.next(), iter.next())
+                    {
+                        lit.set_span((@ quote_span(tt.span())));
+                        lit
+                    } else {
+                        unreachable!()
+                    }
+                }))
+            })),))
+        })
+        .collect::<TokenStream>();
 
-        impl Quote for Literal {
-            fn quote(self) -> TokenStream {
-                let (kind, contents, suffix) = self.kind_contents_and_suffix();
-                quote! {
-                    (quote kind).with_contents_and_suffix((quote contents), (quote suffix))
-                }
-            }
-        }
+    if after_dollar {
+        panic!("unexpected trailing `$` in `quote!`");
     }
-}
 
-literals!(Byte, Char, Float, Str_, Integer, ByteStr; StrRaw, ByteStrRaw);
-
-impl Quote for Delimiter {
-    fn quote(self) -> TokenStream {
-        macro_rules! gen_match {
-            ($($i:ident),*) => {
-                match self {
-                    $(Delimiter::$i => { quote!(::Delimiter::$i) })*
-                }
-            }
-        }
-
-        gen_match!(Parenthesis, Brace, Bracket, None)
-    }
+    quote!([(@ tokens)].iter().cloned().collect::<::TokenStream>())
 }
 
-impl Quote for Spacing {
-    fn quote(self) -> TokenStream {
-        macro_rules! gen_match {
-            ($($i:ident),*) => {
-                match self {
-                    $(Spacing::$i => { quote!(::Spacing::$i) })*
-                }
-            }
-        }
-
-        gen_match!(Alone, Joint)
-    }
+/// Quote a `Span` into a `TokenStream`.
+/// This is needed to implement a custom quoter.
+#[unstable(feature = "proc_macro_quote", issue = "38356")]
+pub fn quote_span(_: Span) -> TokenStream {
+    quote!(::Span::def_site())
 }

src/libproc_macro/quote.rs

@@ -0,0 +1,284 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use {Delimiter, Level, Spacing, Span, __internal};
+use {Group, Ident, Literal, Punct, TokenTree};
+
+use rustc_errors as errors;
+use syntax::ast;
+use syntax::parse::lexer::comments;
+use syntax::parse::token;
+use syntax::tokenstream;
+use syntax_pos::symbol::{keywords, Symbol};
+
+impl Ident {
+    pub(crate) fn new_maybe_raw(string: &str, span: Span, is_raw: bool) -> Ident {
+        let sym = Symbol::intern(string);
+        if is_raw
+            && (sym == keywords::Underscore.name()
+                || ast::Ident::with_empty_ctxt(sym).is_path_segment_keyword())
+        {
+            panic!("`{:?}` is not a valid raw identifier", string)
+        }
+        Ident { sym, span, is_raw }
+    }
+}
+
+impl Delimiter {
+    pub(crate) fn from_internal(delim: token::DelimToken) -> Delimiter {
+        match delim {
+            token::Paren => Delimiter::Parenthesis,
+            token::Brace => Delimiter::Brace,
+            token::Bracket => Delimiter::Bracket,
+            token::NoDelim => Delimiter::None,
+        }
+    }
+
+    pub(crate) fn to_internal(self) -> token::DelimToken {
+        match self {
+            Delimiter::Parenthesis => token::Paren,
+            Delimiter::Brace => token::Brace,
+            Delimiter::Bracket => token::Bracket,
+            Delimiter::None => token::NoDelim,
+        }
+    }
+}
+
+impl TokenTree {
+    pub(crate) fn from_internal(
+        stream: tokenstream::TokenStream,
+        stack: &mut Vec<TokenTree>,
+    ) -> TokenTree {
+        use syntax::parse::token::*;
+
+        let (tree, is_joint) = stream.as_tree();
+        let (span, token) = match tree {
+            tokenstream::TokenTree::Token(span, token) => (span, token),
+            tokenstream::TokenTree::Delimited(span, delimed) => {
+                let delimiter = Delimiter::from_internal(delimed.delim);
+                let mut g = Group::new(delimiter, ::TokenStream(delimed.tts.into()));
+                g.set_span(Span(span));
+                return g.into();
+            }
+        };
+
+        let op_kind = if is_joint {
+            Spacing::Joint
+        } else {
+            Spacing::Alone
+        };
+        macro_rules! tt {
+            ($e:expr) => {{
+                let mut x = TokenTree::from($e);
+                x.set_span(Span(span));
+                x
+            }};
+        }
+        macro_rules! op {
+            ($a:expr) => {
+                tt!(Punct::new($a, op_kind))
+            };
+            ($a:expr, $b:expr) => {{
+                stack.push(tt!(Punct::new($b, op_kind)));
+                tt!(Punct::new($a, Spacing::Joint))
+            }};
+            ($a:expr, $b:expr, $c:expr) => {{
+                stack.push(tt!(Punct::new($c, op_kind)));
+                stack.push(tt!(Punct::new($b, Spacing::Joint)));
+                tt!(Punct::new($a, Spacing::Joint))
+            }};
+        }
+
+        match token {
+            Eq => op!('='),
+            Lt => op!('<'),
+            Le => op!('<', '='),
+            EqEq => op!('=', '='),
+            Ne => op!('!', '='),
+            Ge => op!('>', '='),
+            Gt => op!('>'),
+            AndAnd => op!('&', '&'),
+            OrOr => op!('|', '|'),
+            Not => op!('!'),
+            Tilde => op!('~'),
+            BinOp(Plus) => op!('+'),
+            BinOp(Minus) => op!('-'),
+            BinOp(Star) => op!('*'),
+            BinOp(Slash) => op!('/'),
+            BinOp(Percent) => op!('%'),
+            BinOp(Caret) => op!('^'),
+            BinOp(And) => op!('&'),
+            BinOp(Or) => op!('|'),
+            BinOp(Shl) => op!('<', '<'),
+            BinOp(Shr) => op!('>', '>'),
+            BinOpEq(Plus) => op!('+', '='),
+            BinOpEq(Minus) => op!('-', '='),
+            BinOpEq(Star) => op!('*', '='),
+            BinOpEq(Slash) => op!('/', '='),
+            BinOpEq(Percent) => op!('%', '='),
+            BinOpEq(Caret) => op!('^', '='),
+            BinOpEq(And) => op!('&', '='),
+            BinOpEq(Or) => op!('|', '='),
+            BinOpEq(Shl) => op!('<', '<', '='),
+            BinOpEq(Shr) => op!('>', '>', '='),
+            At => op!('@'),
+            Dot => op!('.'),
+            DotDot => op!('.', '.'),
+            DotDotDot => op!('.', '.', '.'),
+            DotDotEq => op!('.', '.', '='),
+            Comma => op!(','),
+            Semi => op!(';'),
+            Colon => op!(':'),
+            ModSep => op!(':', ':'),
+            RArrow => op!('-', '>'),
+            LArrow => op!('<', '-'),
+            FatArrow => op!('=', '>'),
+            Pound => op!('#'),
+            Dollar => op!('$'),
+            Question => op!('?'),
+            SingleQuote => op!('\''),
+
+            Ident(ident, false) => tt!(self::Ident::new(&ident.as_str(), Span(span))),
+            Ident(ident, true) => tt!(self::Ident::new_raw(&ident.as_str(), Span(span))),
+            Lifetime(ident) => {
+                let ident = ident.without_first_quote();
+                stack.push(tt!(self::Ident::new(&ident.as_str(), Span(span))));
+                tt!(Punct::new('\'', Spacing::Joint))
+            }
+            Literal(lit, suffix) => tt!(self::Literal {
+                lit,
+                suffix,
+                span: Span(span)
+            }),
+            DocComment(c) => {
+                let style = comments::doc_comment_style(&c.as_str());
+                let stripped = comments::strip_doc_comment_decoration(&c.as_str());
+                let stream = vec![
+                    tt!(self::Ident::new("doc", Span(span))),
+                    tt!(Punct::new('=', Spacing::Alone)),
+                    tt!(self::Literal::string(&stripped)),
+                ].into_iter()
+                    .collect();
+                stack.push(tt!(Group::new(Delimiter::Bracket, stream)));
+                if style == ast::AttrStyle::Inner {
+                    stack.push(tt!(Punct::new('!', Spacing::Alone)));
+                }
+                tt!(Punct::new('#', Spacing::Alone))
+            }
+
+            Interpolated(_) => __internal::with_sess(|sess, _| {
+                let tts = token.interpolated_to_tokenstream(sess, span);
+                tt!(Group::new(Delimiter::None, ::TokenStream(tts)))
+            }),
+
+            DotEq => op!('.', '='),
+            OpenDelim(..) | CloseDelim(..) => unreachable!(),
+            Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
+        }
+    }
+
+    pub(crate) fn to_internal(self) -> tokenstream::TokenStream {
+        use syntax::parse::token::*;
+        use syntax::tokenstream::{Delimited, TokenTree};
+
+        let (ch, kind, span) = match self {
+            self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()),
+            self::TokenTree::Group(tt) => {
+                return TokenTree::Delimited(
+                    tt.span.0,
+                    Delimited {
+                        delim: tt.delimiter.to_internal(),
+                        tts: tt.stream.0.into(),
+                    },
+                ).into();
+            }
+            self::TokenTree::Ident(tt) => {
+                let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw);
+                return TokenTree::Token(tt.span.0, token).into();
+            }
+            self::TokenTree::Literal(self::Literal {
+                lit: Lit::Integer(ref a),
+                suffix,
+                span,
+            })
+                if a.as_str().starts_with("-") =>
+            {
+                let minus = BinOp(BinOpToken::Minus);
+                let integer = Symbol::intern(&a.as_str()[1..]);
+                let integer = Literal(Lit::Integer(integer), suffix);
+                let a = TokenTree::Token(span.0, minus);
+                let b = TokenTree::Token(span.0, integer);
+                return vec![a, b].into_iter().collect();
+            }
+            self::TokenTree::Literal(self::Literal {
+                lit: Lit::Float(ref a),
+                suffix,
+                span,
+            })
+                if a.as_str().starts_with("-") =>
+            {
+                let minus = BinOp(BinOpToken::Minus);
+                let float = Symbol::intern(&a.as_str()[1..]);
+                let float = Literal(Lit::Float(float), suffix);
+                let a = TokenTree::Token(span.0, minus);
+                let b = TokenTree::Token(span.0, float);
+                return vec![a, b].into_iter().collect();
+            }
+            self::TokenTree::Literal(tt) => {
+                let token = Literal(tt.lit, tt.suffix);
+                return TokenTree::Token(tt.span.0, token).into();
+            }
+        };
+
+        let token = match ch {
+            '=' => Eq,
+            '<' => Lt,
+            '>' => Gt,
+            '!' => Not,
+            '~' => Tilde,
+            '+' => BinOp(Plus),
+            '-' => BinOp(Minus),
+            '*' => BinOp(Star),
+            '/' => BinOp(Slash),
+            '%' => BinOp(Percent),
+            '^' => BinOp(Caret),
+            '&' => BinOp(And),
+            '|' => BinOp(Or),
+            '@' => At,
+            '.' => Dot,
+            ',' => Comma,
+            ';' => Semi,
+            ':' => Colon,
+            '#' => Pound,
+            '$' => Dollar,
+            '?' => Question,
+            '\'' => SingleQuote,
+            _ => unreachable!(),
+        };
+
+        let tree = TokenTree::Token(span.0, token);
+        match kind {
+            Spacing::Alone => tree.into(),
+            Spacing::Joint => tree.joint(),
+        }
+    }
+}
+
+impl Level {
+    pub(crate) fn to_internal(self) -> errors::Level {
+        match self {
+            Level::Error => errors::Level::Error,
+            Level::Warning => errors::Level::Warning,
+            Level::Note => errors::Level::Note,
+            Level::Help => errors::Level::Help,
+            Level::__Nonexhaustive => unreachable!("Level::__Nonexhaustive"),
+        }
+    }
+}

src/libproc_macro/rustc.rs

@@ -379,7 +379,7 @@ impl Diagnostic {
 
     /// Convenience function for internal use, clients should use one of the
     /// public methods above.
-    pub(crate) fn sub(&mut self,
+    pub fn sub(&mut self,
            level: Level,
            message: &str,
            span: MultiSpan,

src/librustc_errors/diagnostic.rs

@@ -39,7 +39,6 @@ use syntax::ast;
 use syntax::attr;
 use syntax::codemap;
 use syntax::edition::Edition;
-use syntax::ext::base::SyntaxExtension;
 use syntax::parse::filemap_to_stream;
 use syntax::symbol::Symbol;
 use syntax_pos::{Span, NO_EXPANSION, FileName};
@@ -517,8 +516,11 @@ impl CrateStore for cstore::CStore {
             return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
         } else if data.name == "proc_macro" &&
                   self.get_crate_data(id.krate).item_name(id.index) == "quote" {
+            use syntax::ext::base::SyntaxExtension;
+            use syntax_ext::proc_macro_impl::BangProcMacro;
+
             let ext = SyntaxExtension::ProcMacro {
-                expander: Box::new(::proc_macro::__internal::Quoter),
+                expander: Box::new(BangProcMacro { inner: ::proc_macro::quote }),
                 allow_internal_unstable: true,
                 edition: data.root.edition,
             };

src/librustc_metadata/cstore_impl.rs

@@ -19,6 +19,7 @@
 #![feature(libc)]
 #![feature(macro_at_most_once_rep)]
 #![feature(proc_macro_internals)]
+#![feature(proc_macro_quote)]
 #![feature(quote)]
 #![feature(rustc_diagnostic_macros)]
 #![feature(slice_sort_by_cached_key)]

src/librustc_metadata/lib.rs

@@ -1775,12 +1775,6 @@ fn ident_continue(c: Option<char>) -> bool {
     (c > '\x7f' && c.is_xid_continue())
 }
 
-// The string is a valid identifier or a lifetime identifier.
-pub fn is_valid_ident(s: &str) -> bool {
-    let mut chars = s.chars();
-    ident_start(chars.next()) && chars.all(|ch| ident_continue(Some(ch)))
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;