more tokenizer updates: self closing tag flag

--HG--
extra : convert_revision : svn%3Aacbfec75-9323-0410-a652-858a13e371e0/trunk%401209

Author: ryanking

lib/html5/constants.rb

@@ -1041,7 +1041,8 @@ def self._(str); str end
            ". Expected end of file."),
       "unexpected-end-table-in-caption" =>
         _("Unexpected end table tag in caption. Generates implied end caption."),
-      "end-html-in-innerhtml" => _("Unexpected html end tag in inner html mode.")
+      "end-html-in-innerhtml" => _("Unexpected html end tag in inner html mode."),
+      "expected-self-closing-tag" => _("Expected a > after the /.")
   }
 
 end

lib/html5/tokenizer.rb

@@ -428,8 +428,7 @@ def tag_name_state
       elsif data == ">"
         emit_current_token
       elsif data == "/"
-        process_solidus_in_tag
-        @state = :before_attribute_name_state
+        @state = :self_closing_tag_state
       else
         @current_token[:name] += data
       end
@@ -627,6 +626,11 @@ def after_attribute_value_state
         if !process_solidus_in_tag
           @state = :before_attribute_name_state
         end
+      elsif data == :EOF
+        @token_queue << {:type =>  :ParseError, :data => "unexpected-EOF-after-attribute-value"}
+        emit_current_token
+        @stream.unget(data)
+        @state = :data_state
       else
         @token_queue.push({:type => :ParseError, :data => "unexpected-character-after-attribute-value"})
         @stream.unget(data)
@@ -635,6 +639,25 @@ def after_attribute_value_state
       true
     end
 
+    def self_closing_tag_state
+      c = @stream.char
+      case c
+      when ">"
+        emit_current_token
+        @current_token[:self_closing] = true
+        @state = :data_state
+      when :EOF
+        @token_queue << {:type => :ParseError, :data => "eof-in-tag-name"}
+        @stream.unget(c)
+        @state = :data_state
+      else
+        @token_queue << {:type => :ParseError, :data => "expected-self-closing-tag"}
+        @stream.unget(c)
+        @state = :before_attribute_name_state
+      end
+      true
+    end
+    
     def bogus_comment_state
       # Make a new comment token and give it as value all the characters
       # until the first > or :EOF (chars_until checks for :EOF automatically)

test/test_tokenizer.rb

@@ -63,7 +63,7 @@ def tokenizer_test(data)
         '' ] * "\n"
 
       assert_nothing_raised message do
-        tokenizer = HTML5::HTMLTokenizer.new(data['input'])
+        tokenizer = HTML5::HTMLTokenizer.new(data['input'], :encoding => 'utf-8')
 
         tokenizer.content_model_flag = content_model_flag.to_sym