Add token::GenerateCompletion

Author: sanxiyn

src/librustdoc/html/highlight.rs

@@ -171,7 +171,8 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,
 
             token::Lifetime(..) => "lifetime",
             token::DocComment(..) => "doccomment",
-            token::Underscore | token::Eof | token::Interpolated(..) |
+            token::Underscore | token::Eof | token::GenerateCompletion |
+                token::Interpolated(..) |
                 token::MatchNt(..) | token::SubstNt(..) => "",
         };
 

src/libsyntax/parse/lexer/mod.rs

@@ -84,6 +84,15 @@ impl<'a> Reader for StringReader<'a> {
     fn is_eof(&self) -> bool { self.curr.is_none() }
     /// Return the next token. EFFECT: advances the string_reader.
     fn next_token(&mut self) -> TokenAndSpan {
+        if let Some(pos) = self.complete_at {
+            if self.peek_span.lo >= pos {
+                self.complete_at = None;
+                return TokenAndSpan {
+                    tok: token::GenerateCompletion,
+                    sp: codemap::mk_sp(pos, pos),
+                }
+            }
+        }
         let ret_val = TokenAndSpan {
             tok: replace(&mut self.peek_tok, token::Underscore),
             sp: self.peek_span,

src/libsyntax/parse/token.rs

@@ -161,6 +161,9 @@ pub enum Token {
     /// A macro variable with special meaning.
     SpecialVarNt(SpecialMacroVar),
 
+    /* For completion */
+    GenerateCompletion,
+
     // Junk. These carry no data because we don't really care about the data
     // they *would* carry, and don't really want to allocate a new ident for
     // them. Instead, users could extract that from the associated span.

src/libsyntax/print/pprust.rs

@@ -294,6 +294,16 @@ pub fn token_to_string(tok: &Token) -> String {
 
         token::SpecialVarNt(var)    => format!("${}", var.as_str()),
 
+        // A proofreading mark is chosen, because the compiler is proofreading
+        // your code.
+        //
+        // U+2038 CARET or U+2041 CARET INSERTION POINT would be better,
+        // but both have poor font support. Neither are supported by
+        // Courier New, the default monospaced font on Windows, for example.
+        token::GenerateCompletion => {
+            "\u{B6}".to_string() // PILCROW SIGN
+        }
+
         token::Interpolated(ref nt) => match *nt {
             token::NtExpr(ref e)  => expr_to_string(&**e),
             token::NtMeta(ref e)  => meta_item_to_string(&**e),