Fix case-sensitivity issues

* (Functional) pseudo-classes are always case-insensitive
* Add the 'xhtml' flag
* Element names and attribute names are case sensitive
  for HTML, but not XHTML or XML.

Author: SimonSapin

CHANGES

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+Version 0.5
+-----------
+
+Not released yet.
+
+* Fix case sensitivity issues.
+* Add the ``xhtml`` parameter for :class:`HTMLTranslator`.
+
+
 Version 0.4
 -----------
 

cssselect/__init__.py

@@ -17,5 +17,5 @@
 from cssselect.xpath import GenericTranslator, HTMLTranslator, ExpressionError
 
 
-VERSION = '0.4'
+VERSION = '0.5'
 __version__ = VERSION

cssselect/parser.py

@@ -376,7 +376,8 @@ def parse_simple_selector(stream, inside_negation=False):
             if stream.peek() == '(':
                 stream.next()
                 stream.skip_whitespace()
-                if ident == 'not':
+                is_negation = ident.lower() == 'not'
+                if is_negation:
                     if inside_negation:
                         raise SelectorSyntaxError('Got nested :not()')
                     argument, argument_pseudo_element = parse_simple_selector(
@@ -396,7 +397,7 @@ def parse_simple_selector(stream, inside_negation=False):
                 if not next == ')':
                     raise SelectorSyntaxError(
                         "Expected ')', got '%s'" % next)
-                if ident == 'not':
+                if is_negation:
                     result = Negation(result, argument)
                 else:
                     result = Function(result, ident, argument)

cssselect/tests.py

@@ -284,72 +284,72 @@ def xpath(css):
             return str(GenericTranslator().css_to_xpath(css, prefix=''))
 
         assert xpath('*') == "*"
-        assert xpath('E') == "e"
-        assert xpath('E[foo]') == "e[@foo]"
-        assert xpath('E[foo="bar"]') == "e[@foo = 'bar']"
-        assert xpath('E[foo~="bar"]') == (
+        assert xpath('e') == "e"
+        assert xpath('e[foo]') == "e[@foo]"
+        assert xpath('e[foo="bar"]') == "e[@foo = 'bar']"
+        assert xpath('e[foo~="bar"]') == (
             "e[@foo and contains("
                "concat(' ', normalize-space(@foo), ' '), ' bar ')]")
-        assert xpath('E[foo^="bar"]') == (
+        assert xpath('e[foo^="bar"]') == (
             "e[@foo and starts-with(@foo, 'bar')]")
-        assert xpath('E[foo$="bar"]') == (
+        assert xpath('e[foo$="bar"]') == (
             "e[@foo and substring(@foo, string-length(@foo)-2) = 'bar']")
-        assert xpath('E[foo*="bar"]') == (
+        assert xpath('e[foo*="bar"]') == (
             "e[@foo and contains(@foo, 'bar')]")
-        assert xpath('E[hreflang|="en"]') == (
+        assert xpath('e[hreflang|="en"]') == (
             "e[@hreflang and ("
                "@hreflang = 'en' or starts-with(@hreflang, 'en-'))]")
-        assert xpath('E:nth-child(1)') == (
+        assert xpath('e:nth-child(1)') == (
             "*/*[name() = 'e' and (position() = 1)]")
-        assert xpath('E:nth-last-child(1)') == (
+        assert xpath('e:nth-last-child(1)') == (
             "*/*[name() = 'e' and (position() = last() - 1)]")
-        assert xpath('E:nth-last-child(2n+2)') == (
+        assert xpath('e:nth-last-child(2n+2)') == (
             "*/*[name() = 'e' and ("
                "(position() +2) mod -2 = 0 and position() < (last() -2))]")
-        assert xpath('E:nth-of-type(1)') == (
+        assert xpath('e:nth-of-type(1)') == (
             "*/e[position() = 1]")
-        assert xpath('E:nth-last-of-type(1)') == (
+        assert xpath('e:nth-last-of-type(1)') == (
             "*/e[position() = last() - 1]")
-        assert xpath('E:nth-last-of-type(1)') == (
+        assert xpath('e:nth-last-of-type(1)') == (
             "*/e[position() = last() - 1]")
-        assert xpath('div E:nth-last-of-type(1) .aclass') == (
+        assert xpath('div e:nth-last-of-type(1) .aclass') == (
             "div/descendant-or-self::*/e[position() = last() - 1]"
                "/descendant-or-self::*/*[@class and contains("
                "concat(' ', normalize-space(@class), ' '), ' aclass ')]")
-        assert xpath('E:first-child') == (
+        assert xpath('e:first-child') == (
             "*/*[name() = 'e' and (position() = 1)]")
-        assert xpath('E:last-child') == (
+        assert xpath('e:last-child') == (
             "*/*[name() = 'e' and (position() = last())]")
-        assert xpath('E:first-of-type') == (
+        assert xpath('e:first-of-type') == (
             "*/e[position() = 1]")
-        assert xpath('E:last-of-type') == (
+        assert xpath('e:last-of-type') == (
             "*/e[position() = last()]")
-        assert xpath('E:only-child') == (
+        assert xpath('e:only-child') == (
             "*/*[name() = 'e' and (last() = 1)]")
-        assert xpath('E:only-of-type') == (
+        assert xpath('e:only-of-type') == (
             "e[last() = 1]")
-        assert xpath('E:empty') == (
+        assert xpath('e:empty') == (
             "e[not(*) and not(normalize-space())]")
-        assert xpath('E:root') == (
+        assert xpath('e:root') == (
             "e[not(parent::*)]")
-        assert xpath('E:contains("foo")') == (
+        assert xpath('e:contains("foo")') == (
             "e[contains(string(.), 'foo')]")
-        assert xpath('E:contains(foo)') == (
+        assert xpath('e:contains(foo)') == (
             "e[contains(string(.), 'foo')]")
-        assert xpath('E.warning') == (
+        assert xpath('e.warning') == (
             "e[@class and contains("
                "concat(' ', normalize-space(@class), ' '), ' warning ')]")
-        assert xpath('E#myid') == (
+        assert xpath('e#myid') == (
             "e[@id = 'myid']")
-        assert xpath('E:not(:nth-child(odd))') == (
+        assert xpath('e:not(:nth-child(odd))') == (
             "e[not((position() -1) mod 2 = 0 and position() >= 1)]")
-        assert xpath('E F') == (
+        assert xpath('e f') == (
             "e/descendant-or-self::*/f")
-        assert xpath('E > F') == (
+        assert xpath('e > f') == (
             "e/f")
-        assert xpath('E + F') == (
+        assert xpath('e + f') == (
             "e/following-sibling::*[name() = 'f' and (position() = 1)]")
-        assert xpath('E ~ F') == (
+        assert xpath('e ~ f') == (
             "e/following-sibling::f")
         assert xpath('div#container p') == (
             "div[@id = 'container']/descendant-or-self::*/p")
@@ -426,12 +426,17 @@ def pcss(main, *selectors, **kwargs):
             return result
 
         all_ids = pcss('*')
+        assert len(all_ids) == 27
         assert all_ids[:4] == ['html', 'nil', 'nil', 'outer-div']
         assert all_ids[-1:] == ['foobar-span']
         assert pcss('div') == ['outer-div', 'li-div', 'foobar-div']
+        assert pcss('DIV', html_only=True) == [
+            'outer-div', 'li-div', 'foobar-div']  # case-insensitive in HTML
         assert pcss('div div') == ['li-div']
         assert pcss('div, div div') == ['outer-div', 'li-div', 'foobar-div']
         assert pcss('a[name]') == ['name-anchor']
+        assert pcss('a[NAme]', html_only=True) == [
+            'name-anchor'] # case-insensitive in HTML:
         assert pcss('a[rel]') == ['tag-anchor', 'nofollow-anchor']
         assert pcss('a[rel="tag"]') == ['tag-anchor']
         assert pcss('a[href*="localhost"]') == ['tag-anchor']
@@ -441,7 +446,7 @@ def pcss(main, *selectors, **kwargs):
         assert pcss('div[foobar~="bc"]', 'div[foobar~="cde"]') == [
             'foobar-div']
         assert pcss('div[foobar~="cd"]') == []
-        assert pcss('*[lang|="en"]', '*[lang|="en-US"]') == ['second-li']
+        assert pcss('*[lang|="en"]', '[lang|="en-US"]') == ['second-li']
         assert pcss('*[lang|="e"]') == []
         assert pcss('li:nth-child(3)') == ['third-li']
         assert pcss('li:nth-child(10)') == []
@@ -471,12 +476,12 @@ def pcss(main, *selectors, **kwargs):
         self.assertRaises(ExpressionError, pcss, 'p *:only-of-type')
         self.assertRaises(ExpressionError, pcss, 'p:lang(fr)')
         assert pcss('p:only-of-type') == ['paragraph']
-        assert pcss('a:empty') == ['name-anchor']
+        assert pcss('a:empty', 'a:EMpty') == ['name-anchor']
         assert pcss('li:empty') == [
             'third-li', 'fourth-li', 'fifth-li', 'sixth-li', 'seventh-li']
         assert pcss(':root', 'html:root') == ['html']
         assert pcss('li:root', '* :root') == []
-        assert pcss('*:contains("link")') == [
+        assert pcss('*:contains("link")', ':CONtains("link")') == [
             'html', 'nil', 'outer-div', 'tag-anchor', 'nofollow-anchor']
         assert pcss('*:contains("LInk")') == []  # case sensitive
         assert pcss('*:contains("e")') == [
@@ -488,7 +493,6 @@ def pcss(main, *selectors, **kwargs):
         assert pcss('ol *.c', 'ol li.c', 'li ~ li.c', 'ol > li.c') == [
             'third-li', 'fourth-li']
         assert pcss('#first-li', 'li#first-li', '*#first-li') == ['first-li']
-        # Need some tests of :not()']
         assert pcss('li div', 'li > div', 'div div') == ['li-div']
         assert pcss('div > div') == []
         assert pcss('div>.c', 'div > .c') == ['first-ol']
@@ -507,6 +511,10 @@ def pcss(main, *selectors, **kwargs):
             'fieldset', 'checkbox-disabled']
         assert pcss(':enabled', html_only=True) == [
             'checkbox-unchecked', 'checkbox-checked']
+        assert pcss('a:not([href])') == ['name-anchor']
+        assert pcss('ol :Not(li[class])') == [
+            'first-li', 'second-li', 'li-div',
+            'fifth-li', 'sixth-li', 'seventh-li']
 
     def test_select_shakespeare(self):
         document = html.document_fromstring(HTML_SHAKESPEARE)

cssselect/xpath.py

@@ -93,6 +93,10 @@ def join(self, combiner, other):
 class GenericTranslator(object):
     """
     Translator for "generic" XML documents.
+
+    Everything is case-sensitive, no assumption is made on the meaning
+    of element names and attribute names.
+
     """
     combinator_mapping = {
         ' ': 'descendant',
@@ -116,6 +120,24 @@ class GenericTranslator(object):
     #: http://www.w3.org/TR/selectors/#id-selectors
     id_attribute = 'id'
 
+    #: The case sensitivity of document language element names,
+    #: attribute names, and attribute values in selectors depends
+    #: on the document language.
+    #: http://www.w3.org/TR/selectors/#casesens
+    #:
+    #: When a document language defines one of these as case-insensitive,
+    #: cssselect assumes that the document parser makes the parsed values
+    #: lower-case. Making the selector lower-case too makes the comparaison
+    #: case-insensitive.
+    #:
+    #: In HTML, element names and attributes names (but not attribute values)
+    #: are case-insensitive. All of lxml.html, html5lib, BeautifulSoup4
+    #: and HTMLParser make them lower-case in their parse result, so
+    #: the assumption holds.
+    lower_case_element_names = False
+    lower_case_attribute_names = False
+    lower_case_attribute_values = False
+
     def css_to_xpath(self, css, prefix='descendant-or-self::'):
         """Translate a *group of selectors* to XPath.
 
@@ -201,7 +223,7 @@ def xpath_negation(self, negation):
 
     def xpath_function(self, function):
         """Translate a functional pseudo-class."""
-        method = 'xpath_%s_function' % function.name.replace('-', '_')
+        method = 'xpath_%s_function' % function.name.replace('-', '_').lower()
         method = getattr(self, method, None)
         if not method:
             raise ExpressionError(
@@ -210,7 +232,7 @@ def xpath_function(self, function):
 
     def xpath_pseudo(self, pseudo):
         """Translate a pseudo-class."""
-        method = 'xpath_%s_pseudo' % pseudo.ident.replace('-', '_')
+        method = 'xpath_%s_pseudo' % pseudo.ident.replace('-', '_').lower()
         method = getattr(self, method, None)
         if not method:
             # TODO: better error message for pseudo-elements?
@@ -226,12 +248,19 @@ def xpath_attrib(self, selector):
             raise ExpressionError(
                 "Unknown attribute operator: %r" % selector.operator)
         method = getattr(self, 'xpath_attrib_%s' % operator)
-        # FIXME: what if attrib is *?
+        if self.lower_case_attribute_names:
+            name = selector.attrib.lower()
+        else:
+            name = selector.attrib
         if selector.namespace == '*':
-            name = '@' + selector.attrib
+            name = '@' + name
+        else:
+            name = '@%s:%s' % (selector.namespace, name)
+        if self.lower_case_attribute_values:
+            value = selector.value.lower()
         else:
-            name = '@%s:%s' % (selector.namespace, selector.attrib)
-        return method(self.xpath(selector.selector), name, selector.value)
+            value = selector.value
+        return method(self.xpath(selector.selector), name, value)
 
     def xpath_class(self, class_selector):
         """Translate a class selector."""
@@ -243,23 +272,18 @@ def xpath_class(self, class_selector):
     def xpath_hash(self, id_selector):
         """Translate an ID selector."""
         xpath = self.xpath(id_selector.selector)
-        return xpath.add_condition('@%s = %s' % (
-            self.id_attribute, self.xpath_literal(id_selector.id)))
+        return self.xpath_attrib_equals(xpath, '@id', id_selector.id)
 
     def xpath_element(self, selector):
         """Translate a type or universal selector."""
-        if selector.namespace == '*':
-            # Fixed case sensitive matching on lxml 2.3.4 patched for external cssselect with Python 2.7 64-bit on Windows.
-            # Case insensitive matching is not working unless source elements are lower case.
-            # For HTMLTranslator, I kept the existing behavior of setting the element to lower case.
-            # "...in HTML, element names are case-insensitive, but in XML they are case-sensitive."
-            # http://www.w3.org/TR/CSS2/selector.html#pattern-matching
-            element = selector.element
-            if isinstance(self, HTMLTranslator):
-                element = element.lower()
+        if self.lower_case_element_names:
+            element = selector.element.lower()
         else:
-            # FIXME: Should we lowercase here?
-            element = '%s:%s' % (selector.namespace, selector.element)
+            element = selector.element
+        if selector.namespace != '*':
+            # Namespace prefixes are case-sensitive.
+            # http://www.w3.org/TR/css3-namespace/#prefixes
+            element = '%s:%s' % (selector.namespace, element)
         return XPathExpr(element=element)
 
 
@@ -465,8 +489,24 @@ def xpath_attrib_substringmatch(self, xpath, name, value):
 
 class HTMLTranslator(GenericTranslator):
     """
-    Translator for HTML documents.
+    Translator for (X)HTML documents.
+
+    Has a more useful implementation of some pseudo-classes, based on
+    HTML-specific element names and attribute names.
+    The API is the same as :class:`GenericTranslator`.
+
+    :param xhtml:
+        If false (the default), element names and attribute names
+        are case-insensitive.
+
     """
+    def __init__(self, xhtml=False):
+        self.xhtml = xhtml  # Might be useful for sub-classes?
+        if not xhtml:
+            # See their definition in GenericTranslator.
+            self.lower_case_element_names = True
+            self.lower_case_attribute_names = True
+
     def xpath_checked_pseudo(self, xpath):
         # FIXME: is this really all the elements?
         return xpath.add_condition(

docs/index.rst

@@ -57,9 +57,6 @@ selectors. For example, ``div, h1.title + p`` is a group of 2 selectors.
     :members: css_to_xpath, selector_to_xpath
 
 .. autoclass:: HTMLTranslator
-
-    The API is the same as :class:`GenericTranslator`.
-
 .. autoexception:: SelectorError
 .. autoexception:: SelectorSyntaxError
 .. autoexception:: ExpressionError