handle invalid bytes in string literals, rework error

Author: WillLillis

lib/docs/wasm/html_render.zig

@@ -171,7 +171,6 @@ pub fn fileSourceHtml(
                 try out.appendSlice(gpa, "</span>");
             },
 
-            .comment,
             .doc_comment,
             .container_doc_comment,
             => {

lib/std/zig/Ast.zig

@@ -458,6 +458,12 @@ pub fn renderError(tree: Ast, parse_error: Error, stream: anytype) !void {
             return stream.writeAll("for input is not captured");
         },
 
+        .invalid_byte => {
+            return stream.print("string/comment contains invalid byte: '{'}'", .{
+                std.zig.fmtEscapes(tree.source[(tree.tokens.items(.start)[parse_error.token] + parse_error.extra.offset)..][0..1]),
+            });
+        },
+
         .expected_token => {
             const found_tag = token_tags[parse_error.token + @intFromBool(parse_error.token_is_prev)];
             const expected_symbol = parse_error.extra.expected_tag.symbol();
@@ -2926,6 +2932,7 @@ pub const Error = struct {
     extra: union {
         none: void,
         expected_tag: Token.Tag,
+        offset: usize,
     } = .{ .none = {} },
 
     pub const Tag = enum {
@@ -2996,6 +3003,9 @@ pub const Error = struct {
 
         /// `expected_tag` is populated.
         expected_token,
+
+        /// `offset` is populated
+        invalid_byte,
     };
 };
 

lib/std/zig/AstGen.zig

@@ -13986,8 +13986,9 @@ fn lowerAstErrors(astgen: *AstGen) !void {
     const parse_err = tree.errors[0];
     const tok = parse_err.token + @intFromBool(parse_err.token_is_prev);
     const tok_start = token_starts[tok];
+    const start_char = tree.source[tok_start];
 
-    if (token_tags[tok] == .invalid and (tree.source[tok_start] == '\\' or tree.source[tok_start] == '/')) {
+    if (token_tags[tok] == .invalid and (start_char == '\"' or start_char == '\\' or start_char == '/')) {
         const tok_len: u32 = @intCast(tree.tokenSlice(tok).len);
         const tok_end = tok_start + tok_len;
         const bad_off = blk: {
@@ -14001,28 +14002,14 @@ fn lowerAstErrors(astgen: *AstGen) !void {
             break :blk idx - tok_start;
         };
 
-        const byte_abs = tok_start + bad_off;
-        try notes.append(gpa, try astgen.errNoteTokOff(tok, bad_off, "invalid byte: '{'}'", .{
-            std.zig.fmtEscapes(tree.source[byte_abs..][0..1]),
-        }));
-        const expected_tag = switch (tree.source[tok_start]) {
-            '\\' => std.zig.Token.Tag.multiline_string_literal_line,
-            '/' => blk: {
-                if (std.mem.indexOf(u8, tree.source[tok_start..tok_end], "///")) |idx| {
-                    if (idx == 0) break :blk std.zig.Token.Tag.doc_comment;
-                }
-                break :blk std.zig.Token.Tag.comment;
-            },
-            else => unreachable,
-        };
         const err: Ast.Error = .{
-            .tag = Ast.Error.Tag.expected_token,
+            .tag = Ast.Error.Tag.invalid_byte,
             .token = tok,
-            .extra = .{ .expected_tag = expected_tag },
+            .extra = .{ .offset = bad_off },
         };
         msg.clearRetainingCapacity();
         try tree.renderError(err, msg.writer(gpa));
-        return try astgen.appendErrorTokNotes(tok, "{s}", .{msg.items}, notes.items);
+        return try astgen.appendErrorTokNotesOff(tok, bad_off, "{s}", .{msg.items}, notes.items);
     }
 
     var cur_err = tree.errors[0];

lib/std/zig/tokenizer.zig

@@ -137,7 +137,6 @@ pub const Token = struct {
         angle_bracket_angle_bracket_right_equal,
         tilde,
         number_literal,
-        comment,
         doc_comment,
         container_doc_comment,
         keyword_addrspace,
@@ -200,7 +199,6 @@ pub const Token = struct {
                 .eof,
                 .builtin,
                 .number_literal,
-                .comment,
                 .doc_comment,
                 .container_doc_comment,
                 => null,
@@ -329,7 +327,6 @@ pub const Token = struct {
                 .eof => "EOF",
                 .builtin => "a builtin function",
                 .number_literal => "a number literal",
-                .comment => "a comment",
                 .doc_comment, .container_doc_comment => "a document comment",
                 else => unreachable,
             };

test/cases/compile_errors/normal_string_with_newline.zig

@@ -5,4 +5,4 @@ b";
 // backend=stage2
 // target=native
 //
-// :1:13: error: expected expression, found 'invalid token'
+// :1:15: error: string/comment contains invalid byte: '\n'

test/cases/compile_errors/tab_inside_comment.zig

@@ -5,5 +5,4 @@ export fn entry() void {}
 // backend=stage2
 // target=native
 //
-// :1:1: error: expected 'a comment', found invalid bytes
-// :1:8: note: invalid byte: '\t'
+// :1:8: error: string/comment contains invalid byte: '\t'

test/cases/compile_errors/tab_inside_doc_comment.zig

@@ -5,5 +5,4 @@ export fn entry() void {}
 // backend=stage2
 // target=native
 //
-// :1:1: error: expected 'a document comment', found invalid bytes
-// :1:13: note: invalid byte: '\t'
+// :1:13: error: string/comment contains invalid byte: '\t'

test/cases/compile_errors/tab_inside_multiline_string.zig

@@ -10,5 +10,4 @@ export fn entry() void {
 // backend=stage2
 // target=native
 //
-// :4:9: error: expected 'a string literal', found invalid bytes
-// :4:11: note: invalid byte: '\t'
+// :4:11: error: string/comment contains invalid byte: '\t'

test/cases/compile_errors/tab_inside_string.zig

@@ -0,0 +1,10 @@
+export fn entry() void {
+    const foo = "	hello";
+    _ = foo;
+}
+
+// error
+// backend=stage2
+// target=native
+//
+// :2:18: error: string/comment contains invalid byte: '\t'

test/compile_errors.zig

@@ -217,16 +217,15 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
         const case = ctx.obj("invalid byte in string", b.graph.host);
 
         case.addError("_ = \"\x01Q\";", &[_][]const u8{
-            ":1:5: error: expected expression, found 'invalid token'",
+            ":1:6: error: string/comment contains invalid byte: '\\x01'",
         });
     }
 
     {
         const case = ctx.obj("invalid byte in comment", b.graph.host);
 
         case.addError("//\x01Q", &[_][]const u8{
-            ":1:1: error: expected 'a comment', found invalid bytes",
-            ":1:3: note: invalid byte: '\\x01'",
+            ":1:3: error: string/comment contains invalid byte: '\\x01'",
         });
     }
 

tools/docgen.zig

@@ -782,7 +782,6 @@ fn tokenizeAndPrintRaw(
                 try out.writeAll("</span>");
             },
 
-            .comment,
             .doc_comment,
             .container_doc_comment,
             => {

tools/doctest.zig

@@ -677,7 +677,6 @@ fn tokenizeAndPrint(arena: Allocator, out: anytype, raw_src: []const u8) !void {
                 try out.writeAll("</span>");
             },
 
-            .comment,
             .doc_comment,
             .container_doc_comment,
             => {