Simple HTMLTokenizer done, innerHTML and outerHTML done. With tests.

Author: ntoll

polyplug.py

@@ -184,16 +184,22 @@ def get_attrs(self):
 
     def get_text(self, until="<"):
         """
-        Return textual content until the start of a new Node ("<") or
-        until matches.
+        Return textual content until the start of a new Node ("<") or until
+        "until" matches.
         """
         result = ""
         until_len = len(until)
         while result[-until_len:] != until and self.char:
             result += self.get_char()
-        self.pos = self.pos - until_len
-        self.next_char()
-        return result[:-until_len]
+        if result[-until_len:] == until:
+            self.pos = self.pos - until_len
+            if self.char:
+                self.pos = self.pos - 1
+            self.next_char()
+            return result[:-until_len]
+        else:
+            # EOF
+            return result
 
     def tokenize(self, parent=None):
         """
@@ -205,29 +211,25 @@ def tokenize(self, parent=None):
         """
         if not isinstance(parent, ElementNode):
             raise ValueError("Parent must be an ElementNode")
-        current_children = []
         current_node = None
         current_parent = parent
         while self.char:
             if self.match("<"):
                 # Tag opens.
                 if self.match("/"):
                     # End tag. Close and check depth of tree.
-                    if not current_node:
-                        # Can't close an un-opened current node.
-                        raise ValueError("Missing opening tag.")
                     # Get the name of the closing tag.
                     name = self.get_name()
-                    if name == current_node.tagName:
+                    if current_node and name == current_node.tagName:
                         # Close current node and continue at current depth.
-                        current_children.append(current_node)
+                        current_parent.add_child(current_node)
                         current_node = None
                     elif name == current_parent.tagName:
                         # Step back up the tree to the parent context.
-                        for child in current_children:
-                            current_parent.add_child(child)
                         current_node = current_parent
                         current_parent = current_node.parent
+                        current_parent.add_child(current_node)
+                        current_node = None
                     else:
                         # Unexpected close tag.
                         raise ValueError("Unexpected close tag.", name)
@@ -246,31 +248,41 @@ def tokenize(self, parent=None):
                         value += self.get_char()
                         if value[-3:] == "-->":
                             break
-                    comment = CommentNode(nodeValue=value)
-                    current_children.append(comment)
+                    comment = CommentNode(nodeValue=value[:-3])
+                    current_parent.add_child(comment)
                 else:
                     # ElementNode
                     tagName = self.get_name()
                     attrs = self.get_attrs()
+                    if current_node:
+                        current_parent = current_node
                     if tagName == "textarea":
+                        self.expect(">")
                         value = self.get_text(until="</textarea>")
-                        current_node = ElementNode(
-                            tagName=tagName, attributes=attrs, value=value
+                        textarea_node = ElementNode(
+                            tagName=tagName,
+                            attributes=attrs,
+                            value=value,
+                            parent=current_parent,
                         )
-                        expect("</textarea>")
+                        for c in "</textarea>":
+                            self.expect(c)
+                        current_parent.add_child(textarea_node)
                     else:
                         current_node = ElementNode(
-                            tagName=tagName, attributes=attrs
+                            tagName=tagName,
+                            attributes=attrs,
+                            parent=current_parent,
                         )
-                    current_children = []
+                        self.expect(">")
             else:
                 # TextNode
                 value = self.get_text()
                 text = TextNode(nodeValue=value)
-                current_children.append(text)
-        # Append "root" children to the parent node.
-        for child in current_children:
-            parent.add_child(child)
+                if current_node:
+                    current_node.add_child(text)
+                else:
+                    current_parent.add_child(text)
 
 
 class Node:
@@ -333,6 +345,46 @@ def add_child(self, child):
         children.append(node_dict)
         self._node["childNodes"] = children
 
+    @property
+    def outerHTML(self):
+        """
+        Get a string representation of the element's outer HTML.
+        """
+        result = "<" + self.tagName
+        for attr, val in self.attributes.items():
+            result += " " + attr + "=\"" + val + "\""
+        result += ">"
+        if self.tagName == "textarea":
+            result += self.value
+        else:
+            result += self.innerHTML
+        result += "</" + self.tagName + ">"
+        return result
+
+    @property
+    def innerHTML(self):
+        """
+        Get a string representation of the element's inner HTML.
+        """
+        result = ""
+        for child in self.childNodes:
+            if isinstance(child, ElementNode):
+                result += child.outerHTML
+            elif isinstance(child, TextNode):
+                result += child.nodeValue
+            elif isinstance(child, CommentNode):
+                result += "<!--" + child.nodeValue + "-->"
+        return result
+
+    @innerHTML.setter
+    def innerHTML(self, raw):
+        """
+        Use the raw innerHTML to create children.
+        """
+        self._node["childNodes"] = []
+        tok = HTMLTokenizer(raw)
+        tok.tokenize(self)
+
     @property
     def childNodes(self):
         if self.tagName == "textarea":

tests/test_polyplug.py

@@ -323,6 +323,49 @@ def test_element_node_add_child_as_dict():
     assert new_child.nodeValue == "Hello"
 
 
+def test_element_node_get_outer_html():
+    """
+    Get a string representation of the node's complete structure.
+    """
+    n = polyplug.ElementNode(tagName="div", attributes={"foo": "bar"})
+    n.add_child(polyplug.TextNode(nodeValue="Hello"))
+    assert n.outerHTML == "<div foo=\"bar\">Hello</div>"
+
+
+def test_element_node_get_inner_html_empty():
+    """
+    Get a string representation of the node's inner structure. Empty.
+    """
+    n = polyplug.ElementNode(tagName="div", attributes={"foo": "bar"})
+    assert n.innerHTML == ""
+
+
+def test_element_node_get_set_inner_html_complex():
+    """
+    Get a string representation of the node's inner structure.
+    """
+    n = polyplug.ElementNode(tagName="div", attributes={"foo": "bar"})
+    n.innerHTML = "<!-- comment --><p>Hello</p>"
+    assert n.innerHTML == "<!-- comment --><p>Hello</p>"
+
+def test_element_node_set_inner_html_empty():
+    """
+    Set the innerHTML of the node as empty.
+    """
+    n = polyplug.ElementNode(tagName="div", attributes={"foo": "bar"})
+    n.innerHTML = "<!-- comment --><p>Hello</p>"
+    n.innerHTML = ""
+    assert n.innerHTML == ""
+
+
+def test_element_node_set_inner_html_textarea():
+    """
+    Set the innerHTML of the node as a textarea.
+    """
+    n = polyplug.ElementNode(tagName="div", attributes={"foo": "bar"})
+    n.innerHTML = "<textarea>Test <fake html></textarea>"
+    assert n.innerHTML == "<textarea>Test <fake html></textarea>"
+
 def test_text_node():
     """
     The TextNode instantiates as expected.
@@ -511,6 +554,17 @@ def test_htmltokenizer_get_attrs_no_attributes():
 
 
 def test_htmltokenizer_get_text():
+    """
+    Get the textual content of a TextNode (i.e. everything until encountering
+    "<").
+    """
+    raw = "Hello, world!<div>"
+    tok = polyplug.HTMLTokenizer(raw)
+    assert tok.get_text() == "Hello, world!"
+    assert tok.char == "<"
+
+
+def test_htmltokenizer_get_text_bound_check():
     """
     Get the textual content of a TextNode (i.e. everything until encountering
     "<").
@@ -521,6 +575,17 @@ def test_htmltokenizer_get_text():
     assert tok.char == "<"
 
 
+def test_htmltokenizer_get_text_eof():
+    """
+    Get the textual content of a TextNode (i.e. everything until encountering
+    "<").
+    """
+    raw = "Hello, world!"
+    tok = polyplug.HTMLTokenizer(raw)
+    assert tok.get_text() == "Hello, world!"
+    assert tok.char == ""
+
+
 def test_htmltokenizer_get_text_until():
     """
     Get the textual content of a TextNode (i.e. everything until encountering
@@ -571,3 +636,126 @@ def test_htmltokenizer_tokenize_comment():
     tok = polyplug.HTMLTokenizer(raw)
     tok.tokenize(parent)
     assert len(parent.childNodes) == 1
+    assert isinstance(parent.childNodes[0], polyplug.CommentNode)
+    assert parent.childNodes[0].nodeValue == " Test comment. "
+
+
+def test_htmltokenizer_tokenize_prolog():
+    """
+    Given a valid parent, an XML prolog node is ignored. <? foo ?>
+    """
+    parent = polyplug.ElementNode(tagName="div")
+    raw = "<? xml ?>"
+    tok = polyplug.HTMLTokenizer(raw)
+    tok.tokenize(parent)
+    assert len(parent.childNodes) == 0
+
+
+def test_htmltokenizer_tokenize_text():
+    """
+    Textual content becomes a child TextNode.
+    """
+    parent = polyplug.ElementNode(tagName="div")
+    raw = "Test text."
+    tok = polyplug.HTMLTokenizer(raw)
+    tok.tokenize(parent)
+    assert len(parent.childNodes) == 1
+    assert isinstance(parent.childNodes[0], polyplug.TextNode)
+    assert parent.childNodes[0].nodeValue == "Test text."
+
+
+def test_htmltokenizer_tokenize_element():
+    """
+    An HTML element becomes a child ElementNode.
+    """
+    parent = polyplug.ElementNode(tagName="div")
+    raw = "<p>Hello</p>"
+    tok = polyplug.HTMLTokenizer(raw)
+    tok.tokenize(parent)
+    assert len(parent.childNodes) == 1
+    p_node = parent.childNodes[0]
+    assert isinstance(p_node, polyplug.ElementNode)
+    assert len(p_node.childNodes) == 1
+    assert isinstance(p_node.childNodes[0], polyplug.TextNode)
+    assert p_node.childNodes[0].nodeValue == "Hello"
+
+
+def test_htmltokenizer_tokenize_element_textarea():
+    """
+    An HTML element becomes a child ElementNode. But textarea nodes will only
+    have a value (no children) containing the text within the tags.
+    """
+    parent = polyplug.ElementNode(tagName="div")
+    raw = "<textarea>Hello <fake-html></textarea>"
+    tok = polyplug.HTMLTokenizer(raw)
+    tok.tokenize(parent)
+    assert len(parent.childNodes) == 1
+    textarea = parent.childNodes[0]
+    assert isinstance(textarea, polyplug.ElementNode)
+    assert len(textarea.childNodes) == 0
+    assert textarea.value == "Hello <fake-html>"
+
+
+def test_htmltokenizer_tokenize_element_unexpected():
+    """
+    An unexpected close tag results in a ValueError.
+    """
+    parent = polyplug.ElementNode(tagName="div")
+    raw = "</textarea>"
+    tok = polyplug.HTMLTokenizer(raw)
+    with pytest.raises(ValueError):
+        tok.tokenize(parent)
+
+
+def test_htmltokenizer_tokenize_complex_tree():
+    """
+    A more complex tree with several branches and node types.
+    """
+    parent = polyplug.ElementNode(tagName="div")
+    raw = "<!-- comment --><div id='myId'><p>Hello</p><p>world</p></div>"
+    expected = {
+        "childNodes": [
+            {
+                "childNodes": [],
+                "nodeName": "#comment",
+                "nodeType": 8,
+                "nodeValue": " comment ",
+            },
+            {
+                "attributes": {"id": "myId"},
+                "childNodes": [
+                    {
+                        "childNodes": [
+                            {
+                                "childNodes": [],
+                                "nodeName": "#text",
+                                "nodeType": 3,
+                                "nodeValue": "Hello",
+                            }
+                        ],
+                        "nodeType": 1,
+                        "tagName": "p",
+                    },
+                    {
+                        "childNodes": [
+                            {
+                                "childNodes": [],
+                                "nodeName": "#text",
+                                "nodeType": 3,
+                                "nodeValue": "world",
+                            }
+                        ],
+                        "nodeType": 1,
+                        "tagName": "p",
+                    },
+                ],
+                "nodeType": 1,
+                "tagName": "div",
+            },
+        ],
+        "nodeType": 1,
+        "tagName": "div",
+    }
+    tok = polyplug.HTMLTokenizer(raw)
+    tok.tokenize(parent)
+    assert parent.as_dict == expected