Lex doc comments as attributes

This means they no longer round trip but it should be more faithful to what
macro_rules! is doing

Closes #73

Author: alexcrichton

src/stable.rs

@@ -591,10 +591,32 @@ impl fmt::Display for Literal {
     }
 }
 
-named!(token_stream -> ::TokenStream, map!(
-    many0!(token_tree),
-    |trees| ::TokenStream::_new(TokenStream { inner: trees })
-));
+fn token_stream(mut input: Cursor) -> PResult<::TokenStream> {
+    let mut trees = Vec::new();
+    loop {
+        let input_no_ws = skip_whitespace(input);
+        if input_no_ws.rest.len() == 0 {
+            break
+        }
+        if let Ok((a, comment)) = doc_comment(input_no_ws) {
+            input = a;
+            trees.push(TokenTree::Op(Op::new('#', Spacing::Alone)));
+            let stream = vec![
+                TokenTree::Term(::Term::new("doc", ::Span::call_site())),
+                TokenTree::Op(Op::new('=', Spacing::Alone)),
+                TokenTree::Literal(::Literal::string(comment)),
+            ];
+            let stream = stream.into_iter().collect();
+            trees.push(TokenTree::Group(Group::new(Delimiter::Bracket, stream)));
+            continue
+        }
+
+        let (a, tt) = token_tree(input_no_ws)?;
+        trees.push(tt);
+        input = a;
+    }
+    Ok((input, ::TokenStream::_new(TokenStream { inner: trees })))
+}
 
 #[cfg(not(procmacro2_semver_exempt))]
 fn token_tree(input: Cursor) -> PResult<TokenTree> {
@@ -721,8 +743,6 @@ named!(literal_nocapture -> (), alt!(
     float
     |
     int
-    |
-    doc_comment
 ));
 
 named!(string -> (), alt!(
@@ -1146,31 +1166,31 @@ fn op_char(input: Cursor) -> PResult<char> {
     }
 }
 
-named!(doc_comment -> (), alt!(
+named!(doc_comment -> &str, alt!(
     do_parse!(
         punct!("//!") >>
-        take_until_newline_or_eof!() >>
-        (())
+        s: take_until_newline_or_eof!() >>
+        (s)
     )
     |
     do_parse!(
         option!(whitespace) >>
         peek!(tag!("/*!")) >>
-        block_comment >>
-        (())
+        s: block_comment >>
+        (s)
     )
     |
     do_parse!(
         punct!("///") >>
         not!(tag!("/")) >>
-        take_until_newline_or_eof!() >>
-        (())
+        s: take_until_newline_or_eof!() >>
+        (s)
     )
     |
     do_parse!(
         option!(whitespace) >>
         peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
-        block_comment >>
-        (())
+        s: block_comment >>
+        (s)
     )
 ));

src/strnom.rs

@@ -268,7 +268,7 @@ macro_rules! take_until_newline_or_eof {
         } else {
             match $i.find('\n') {
                 Some(i) => Ok(($i.advance(i), &$i.rest[..i])),
-                None => Ok(($i.advance($i.len()), "")),
+                None => Ok(($i.advance($i.len()), &$i.rest[..$i.len()])),
             }
         }
     }};
@@ -389,37 +389,3 @@ macro_rules! map {
         map!($i, call!($f), $g)
     };
 }
-
-macro_rules! many0 {
-    ($i:expr, $f:expr) => {{
-        let ret;
-        let mut res = ::std::vec::Vec::new();
-        let mut input = $i;
-
-        loop {
-            if input.is_empty() {
-                ret = Ok((input, res));
-                break;
-            }
-
-            match $f(input) {
-                Err(LexError) => {
-                    ret = Ok((input, res));
-                    break;
-                }
-                Ok((i, o)) => {
-                    // loop trip must always consume (otherwise infinite loops)
-                    if i.len() == input.len() {
-                        ret = Err(LexError);
-                        break;
-                    }
-
-                    res.push(o);
-                    input = i;
-                }
-            }
-        }
-
-        ret
-    }};
-}

tests/test.rs

@@ -29,12 +29,6 @@ fn roundtrip() {
     roundtrip("a");
     roundtrip("<<");
     roundtrip("<<=");
-    roundtrip(
-        "
-        /// a
-        wut
-    ",
-    );
     roundtrip(
         "
         1
@@ -192,11 +186,34 @@ fn tricky_doc_comment() {
 
     let stream = "/// doc".parse::<proc_macro2::TokenStream>().unwrap();
     let tokens = stream.into_iter().collect::<Vec<_>>();
-    assert!(tokens.len() == 1, "not length 1 -- {:?}", tokens);
+    assert!(tokens.len() == 2, "not length 1 -- {:?}", tokens);
     match tokens[0] {
-        proc_macro2::TokenTree::Literal(_) => {}
+        proc_macro2::TokenTree::Op(ref tt) => assert_eq!(tt.op(), '#'),
         _ => panic!("wrong token {:?}", tokens[0]),
     }
+    let mut tokens = match tokens[1] {
+        proc_macro2::TokenTree::Group(ref tt) => {
+            assert_eq!(tt.delimiter(), proc_macro2::Delimiter::Bracket);
+            tt.stream().into_iter()
+        }
+        _ => panic!("wrong token {:?}", tokens[0]),
+    };
+
+    match tokens.next().unwrap() {
+        proc_macro2::TokenTree::Term(ref tt) => assert_eq!(tt.as_str(), "doc"),
+        t => panic!("wrong token {:?}", t),
+    }
+    match tokens.next().unwrap() {
+        proc_macro2::TokenTree::Op(ref tt) => assert_eq!(tt.op(), '='),
+        t => panic!("wrong token {:?}", t),
+    }
+    match tokens.next().unwrap() {
+        proc_macro2::TokenTree::Literal(ref tt) => {
+            assert_eq!(tt.to_string(), "\" doc\"");
+        }
+        t => panic!("wrong token {:?}", t),
+    }
+    assert!(tokens.next().is_none());
 }
 
 #[test]