Add support for processing instructions

html5ever/src/serialize/mod.rs

@@ -284,6 +284,6 @@ impl<Wr: Write> Serializer for HtmlSerializer<Wr> {
         self.writer.write_all(target.as_bytes())?;
         self.writer.write_all(b" ")?;
         self.writer.write_all(data.as_bytes())?;
-        self.writer.write_all(b">")
+        self.writer.write_all(b"?>")
     }
 }

html5ever/src/tokenizer/interface.rs

@@ -16,7 +16,9 @@ use crate::LocalName;
 use std::borrow::Cow;
 
 pub use self::TagKind::{EndTag, StartTag};
-pub use self::Token::{CharacterTokens, CommentToken, DoctypeToken, TagToken};
+pub use self::Token::{
+    CharacterTokens, CommentToken, DoctypeToken, ProcessingInstructionToken, TagToken,
+};
 pub use self::Token::{EOFToken, NullCharacterToken, ParseError};
 
 /// A `DOCTYPE` token.
@@ -79,6 +81,13 @@ impl Tag {
     }
 }
 
+/// A processing instruction token.
+#[derive(PartialEq, Eq, Clone, Debug, Default)]
+pub struct ProcessingInstruction {
+    pub target: StrTendril,
+    pub data: StrTendril,
+}
+
 #[derive(PartialEq, Eq, Debug)]
 pub enum Token {
     /// A DOCTYPE declaration like `<!DOCTYPE html>`
@@ -87,6 +96,8 @@ pub enum Token {
     TagToken(Tag),
     /// A comment like `<!-- foo -->`.
     CommentToken(StrTendril),
+    /// A processing instruction like `<?target data ?>`.
+    ProcessingInstructionToken(ProcessingInstruction),
     /// A sequence of characters.
     CharacterTokens(StrTendril),
     /// A `U+0000 NULL` character in the input.

html5ever/src/tokenizer/mod.rs

@@ -10,8 +10,10 @@
 //! The HTML5 tokenizer.
 
 pub use self::interface::{CharacterTokens, EOFToken, NullCharacterToken, ParseError};
-pub use self::interface::{CommentToken, DoctypeToken, TagToken, Token};
-pub use self::interface::{Doctype, EndTag, StartTag, Tag, TagKind};
+pub use self::interface::{
+    CommentToken, DoctypeToken, ProcessingInstructionToken, TagToken, Token,
+};
+pub use self::interface::{Doctype, EndTag, ProcessingInstruction, StartTag, Tag, TagKind};
 pub use self::interface::{TokenSink, TokenSinkResult};
 
 use self::states::AttrValueKind::*;
@@ -167,6 +169,9 @@ pub struct Tokenizer<Sink> {
     /// Current doctype token.
     current_doctype: RefCell<Doctype>,
 
+    /// Current processing instruction token
+    current_processing_instruction: RefCell<ProcessingInstruction>,
+
     /// Last start tag name, for use in checking "appropriate end tag".
     last_start_tag_name: RefCell<Option<LocalName>>,
 
@@ -211,6 +216,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
             current_attr_value: RefCell::new(StrTendril::new()),
             current_comment: RefCell::new(StrTendril::new()),
             current_doctype: RefCell::new(Doctype::default()),
+            current_processing_instruction: RefCell::new(ProcessingInstruction::default()),
             last_start_tag_name: RefCell::new(start_tag_name),
             temp_buf: RefCell::new(StrTendril::new()),
             state_profile: RefCell::new(BTreeMap::new()),
@@ -625,6 +631,21 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
     fn emit_error(&self, error: Cow<'static, str>) {
         self.process_token_and_continue(ParseError(error));
     }
+
+    fn convert_to_comment(&self) {
+        let mut comment = self.current_comment.borrow_mut();
+        comment.clear();
+        comment.push_char('?');
+        // The spec doesn't specify what should happen to the temporary buffer after conversion, but
+        // nothing uses it afterwards so it should be fine to clear it
+        comment.push_tendril(&mem::take(&mut *self.temp_buf.borrow_mut()));
+    }
+
+    fn emit_current_processing_instruction(&self) {
+        self.process_token_and_continue(ProcessingInstructionToken(
+            self.current_processing_instruction.take(),
+        ));
+    }
 }
 //§ END
 
@@ -940,8 +961,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
                     '?' => {
                         // Set the temporary buffer to the empty string.
                         // Switch to the processing instruction open state.
-                        self.bad_char_error();
-                        go!(self: clear_comment; reconsume BogusComment)
+                        go!(self: clear_temp; to State::ProcessingInstructionOpen);
                     },
                     // ↪ Anything else
                     _ => {
@@ -2654,7 +2674,148 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
                     },
                 }
             },
-            // TODO: What about the processing-instruction related states?
+
+            // https://html.spec.whatwg.org/#processing-instruction-open-state
+            states::ProcessingInstructionOpen => loop {
+                match get_char!(self, input) {
+                    // ↪ ASCII alpha
+                    // ↪ U+005F LOW LINE (_)
+                    character if character.is_ascii_alphabetic() || character == '_' => {
+                        // Reconsume in the processing instruction target state.
+                        go!(self: reconsume State::ProcessingInstructionTarget);
+                    },
+                    // ↪ Anything else
+                    _ => {
+                        // This is an invalid-first-character-of-processing-instruction-target parse error.
+                        // Convert the temporary buffer to a comment. Reconsume in the bogus comment state.
+                        self.bad_char_error();
+                        self.convert_to_comment();
+                        go!(self: reconsume BogusComment);
+                    },
+                }
+            },
+
+            // https://html.spec.whatwg.org/#processing-instruction-target-state
+            states::ProcessingInstructionTarget => loop {
+                match get_char!(self, input) {
+                    // ↪ U+0009 CHARACTER TABULATION (tab)
+                    // ↪ U+000A LINE FEED (LF)
+                    // ↪ U+000C FORM FEED (FF)
+                    // ↪ U+0020 SPACE
+                    // ↪ U+003F QUESTION MARK (?)
+                    // ↪ U+003E GREATER-THAN SIGN (>)
+                    '\t' | '\n' | '\x0C' | ' ' | '?' | '>' => {
+                        // Let target be the concatenation of the code points in the temporary buffer, in the order they were added to the buffer.
+                        let is_xml_or_xml_stylesheet = {
+                            let target = self.temp_buf.borrow();
+                            target.eq_ignore_ascii_case("xml")
+                                || target.eq_ignore_ascii_case("xml-stylesheet")
+                        };
+                        // If target is an ASCII case-insensitive match for "xml" or "xml-stylesheet":
+                        if is_xml_or_xml_stylesheet {
+                            // This is a disallowed-processing-instruction-target parse error.
+                            // Convert the temporary buffer to a comment.
+                            // Reconsume in the bogus comment state.
+                            self.emit_error(Borrowed("Disallowed processing instruction target"));
+                            self.convert_to_comment();
+                            go!(self: reconsume BogusComment);
+                        } else {
+                            let target = self.temp_buf.take();
+                            // Create a processing instruction token whose target is target and data is the empty string.
+                            // Reconsume in the after processing instruction target state.
+                            *self.current_processing_instruction.borrow_mut() =
+                                ProcessingInstruction {
+                                    target,
+                                    data: Default::default(),
+                                };
+                            go!(self: reconsume AfterProcessingInstructionTarget);
+                        }
+                    },
+                    // ↪ ASCII alphanumeric
+                    // ↪ U+002D HYPHEN-MINUS (-)
+                    // ↪ U+005F LOW LINE (_)
+                    character
+                        if character.is_ascii_alphanumeric()
+                            || character == '-'
+                            || character == '_' =>
+                    {
+                        // Append the current input character to the temporary buffer.
+                        go!(self: push_temp character);
+                    },
+                    // ↪ Anything else
+                    _ => {
+                        // This is an invalid-processing-instruction-target parse error.
+                        // Convert the temporary buffer to a comment. Reconsume in the bogus comment state.
+                        self.bad_char_error();
+                        self.convert_to_comment();
+                        go!(self: reconsume BogusComment);
+                    },
+                }
+            },
+
+            // https://html.spec.whatwg.org/#after-processing-instruction-target-state
+            states::AfterProcessingInstructionTarget => loop {
+                match get_char!(self, input) {
+                    // ↪ U+0009 CHARACTER TABULATION (tab)
+                    // ↪ U+000A LINE FEED (LF)
+                    // ↪ U+000C FORM FEED (FF)
+                    // ↪ U+0020 SPACE
+                    '\t' | '\n' | '\x0C' | ' ' => {
+                        // Ignore the character.
+                    },
+                    // ↪ Anything else
+                    _ => {
+                        // Reconsume in the processing instruction data state.
+                        go!(self: reconsume ProcessingInstructionData);
+                    },
+                }
+            },
+
+            // https://html.spec.whatwg.org/#processing-instruction-data-state
+            states::ProcessingInstructionData => loop {
+                match get_char!(self, input) {
+                    // ↪ U+003F QUESTION MARK (?)
+                    '?' => {
+                        // Switch to the processing instruction questionable state.
+                        go!(self: to State::ProcessingInstructionQuestionable)
+                    },
+                    // ↪ U+003E GREATER-THAN SIGN (>)
+                    '>' => {
+                        // Switch to the data state. Emit the current processing instruction token.
+                        self.emit_current_processing_instruction();
+                        go!(self: to State::Data)
+                    },
+                    // ↪ Anything else
+                    character => {
+                        // Append the current input character to the current processing instruction token's data.
+                        self.current_processing_instruction
+                            .borrow_mut()
+                            .data
+                            .push_char(character);
+                    },
+                }
+            },
+
+            // https://html.spec.whatwg.org/#processing-instruction-questionable-state
+            states::ProcessingInstructionQuestionable => loop {
+                match get_char!(self, input) {
+                    // ↪ U+003E GREATER-THAN SIGN (>)
+                    '>' => {
+                        // Switch to the data state. Emit the current processing instruction token.
+                        self.emit_current_processing_instruction();
+                        go!(self: to State::Data)
+                    },
+                    // ↪ Anything else
+                    _ => {
+                        // Append U+003F (?) to the current processing instruction token's data. Reconsume in the processing instruction data state.
+                        self.current_processing_instruction
+                            .borrow_mut()
+                            .data
+                            .push_char('?');
+                        go!(self: reconsume ProcessingInstructionData);
+                    },
+                }
+            },
             //§ END
         }
     }
@@ -2775,7 +2936,12 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
             | states::AfterAttributeValueQuoted
             | states::SelfClosingStartTag
             | states::ScriptDataEscapedDash(_)
-            | states::ScriptDataEscapedDashDash(_) => {
+            | states::ScriptDataEscapedDashDash(_)
+            | states::ProcessingInstructionOpen
+            | states::ProcessingInstructionTarget
+            | states::AfterProcessingInstructionTarget
+            | states::ProcessingInstructionData
+            | states::ProcessingInstructionQuestionable => {
                 self.bad_eof_error();
                 go!(self: to State::Data)
             },

html5ever/src/tokenizer/states.rs

@@ -128,4 +128,14 @@ pub enum State {
     CdataSectionBracket,
     /// <https://html.spec.whatwg.org/#cdata-section-end-state>
     CdataSectionEnd,
+    /// <https://html.spec.whatwg.org/#processing-instruction-open-state>
+    ProcessingInstructionOpen,
+    /// <https://html.spec.whatwg.org/#processing-instruction-target-state>
+    ProcessingInstructionTarget,
+    /// <https://html.spec.whatwg.org/#after-processing-instruction-target-state>
+    AfterProcessingInstructionTarget,
+    /// <https://html.spec.whatwg.org/#processing-instruction-data-state>
+    ProcessingInstructionData,
+    /// <https://html.spec.whatwg.org/#processing-instruction-questionable-state>
+    ProcessingInstructionQuestionable,
 }

html5ever/src/tree_builder/mod.rs

@@ -526,6 +526,9 @@ where
 
             tokenizer::TagToken(x) => Token::Tag(x),
             tokenizer::CommentToken(x) => Token::Comment(x),
+            tokenizer::ProcessingInstructionToken(x) => {
+                Token::ProcessingInstruction(x.target, x.data)
+            },
             tokenizer::NullCharacterToken => Token::NullCharacter,
             tokenizer::EOFToken => Token::Eof,
 
@@ -1342,6 +1345,40 @@ where
         ProcessResult::Done
     }
 
+    fn append_processing_instruction(
+        &self,
+        target: StrTendril,
+        data: StrTendril,
+    ) -> ProcessResult<Handle> {
+        let processing_instruction = self.sink.create_pi(target, data);
+        self.insert_appropriately(AppendNode(processing_instruction), None);
+        ProcessResult::Done
+    }
+
+    fn append_processing_instruction_to_doc(
+        &self,
+        target: StrTendril,
+        data: StrTendril,
+    ) -> ProcessResult<Handle> {
+        let processing_instruction = self.sink.create_pi(target, data);
+        self.sink
+            .append(&self.doc_handle, AppendNode(processing_instruction));
+        ProcessResult::Done
+    }
+
+    fn append_processing_instruction_to_html(
+        &self,
+        target: StrTendril,
+        data: StrTendril,
+    ) -> ProcessResult<Handle> {
+        let open_elements = self.open_elems.borrow();
+        let element_target = html_elem(&open_elements);
+        let processing_instruction = self.sink.create_pi(target, data);
+        self.sink
+            .append(element_target, AppendNode(processing_instruction));
+        ProcessResult::Done
+    }
+
     //§ creating-and-inserting-nodes
     fn create_root(&self, attrs: Vec<Attribute>) {
         let elem = create_element(