Commit 289e9c3507d622bd42a6dee4df8a69dd71a7dfe6 | zig

author	Ryan Liptak <squeek502@hotmail.com> 2025-01-17 19:07:46-08:00
committer	Ryan Liptak <squeek502@hotmail.com> 2025-01-17 23:10:41-08:00
log	289e9c3507d622bd42a6dee4df8a69dd71a7dfe6
tree	a7909fd15b7e4c2605f62b89bade74ada8d9b91b
parent	4baa448335a8158d424a1e648c907fa7566f7060

resinator: Sync with upstream

Note: This mostly matches resinator v0.1.0 rather than the latest master version, since the latest master version focuses on adding support for .res -> .obj conversion which is not necessary for the future planned relationship of zig and resinator (resinator will likely be moved out of the compiler and into the build system, a la translate-c).

So, ultimately the changes here consist mostly of bug fixes for obscure edge cases.

17 files changed, 2144 insertions(+), 948 deletions(-)

lib
- compiler
  - resinator
    ast.zig
    bmp.zig
    cli.zig
    code_pages.zig
    comments.zig
    compile.zig
    disjoint_code_page.zig
    errors.zig
    lang.zig
    lex.zig
    literals.zig
    main.zig
    parse.zig
    preprocess.zig
    rc.zig
    res.zig
    source_mapping.zig

lib/compiler/resinator/ast.zig+49-49

lib/compiler/resinator/bmp.zig+10-3

 @@ -60,9 +60,16 @@ pub const BitmapInfo = struct {
     }
     pub fn getBitmasksByteLen(self: *const BitmapInfo) u8 {
-        return switch (self.compression) {
+        // Only BITMAPINFOHEADER (3.1) has trailing bytes for the BITFIELDS
-            .BI_BITFIELDS => 12,
+        // The 2.0 format doesn't have a compression field and 4.0+ has dedicated
-            .BI_ALPHABITFIELDS => 16,
+        // fields for the masks in the header.
+        const dib_version = BitmapHeader.Version.get(self.dib_header_size);
+        return switch (dib_version) {
+            .@"nt3.1" => switch (self.compression) {
+                .BI_BITFIELDS => 12,
+                .BI_ALPHABITFIELDS => 16,
+                else => 0,
+            },
             else => 0,
         };
     }

lib/compiler/resinator/cli.zig+92-35

 @@ -1,5 +1,6 @@
 const std = @import("std");
-const CodePage = @import("code_pages.zig").CodePage;
+const code_pages = @import("code_pages.zig");
+const SupportedCodePage = code_pages.SupportedCodePage;
 const lang = @import("lang.zig");
 const res = @import("res.zig");
 const Allocator = std.mem.Allocator;
 @@ -14,6 +15,8 @@ pub const usage_string_after_command_name =
     \\The sequence -- can be used to signify when to stop parsing options.
     \\This is necessary when the input path begins with a forward slash.
     \\
+    \\Supported option prefixes are /, -, and --, so e.g. /h, -h, and --h all work.
+    \\
     \\Supported Win32 RC Options:
     \\  /?, /h                  Print this help and exit.
     \\  /v                      Verbose (print progress messages).
 @@ -56,8 +59,6 @@ pub const usage_string_after_command_name =
     \\                            the .rc includes or otherwise depends on.
     \\  /:depfile-fmt <value>     Output format of the depfile, if /:depfile is set.
     \\    json                    (default) A top-level JSON array of paths
-    \\  /:mingw-includes <path>   Path to a directory containing MinGW include files. If
-    \\                            not specified, bundled MinGW include files will be used.
     \\
     \\Note: For compatibility reasons, all custom options start with :
     \\
 @@ -136,7 +137,7 @@ pub const Options = struct {
     ignore_include_env_var: bool = false,
     preprocess: Preprocess = .yes,
     default_language_id: ?u16 = null,
-    default_code_page: ?CodePage = null,
+    default_code_page: ?SupportedCodePage = null,
     verbose: bool = false,
     symbols: std.StringArrayHashMapUnmanaged(SymbolValue) = .empty,
     null_terminate_string_table_strings: bool = false,
 @@ -148,7 +149,6 @@ pub const Options = struct {
     auto_includes: AutoIncludes = .any,
     depfile_path: ?[]const u8 = null,
     depfile_fmt: DepfileFormat = .json,
-    mingw_includes_dir: ?[]const u8 = null,
     pub const AutoIncludes = enum { any, msvc, gnu, none };
     pub const DepfileFormat = enum { json };
 @@ -243,9 +243,6 @@ pub const Options = struct {
         if (self.depfile_path) |depfile_path| {
             self.allocator.free(depfile_path);
         }
-        if (self.mingw_includes_dir) |mingw_includes_dir| {
-            self.allocator.free(mingw_includes_dir);
-        }
     }
     pub fn dumpVerbose(self: *const Options, writer: anytype) !void {
 @@ -358,6 +355,29 @@ pub const Arg = struct {
         };
     }
+    pub fn looksLikeFilepath(self: Arg) bool {
+        const meets_min_requirements = self.prefix == .slash and isSupportedInputExtension(std.fs.path.extension(self.full));
+        if (!meets_min_requirements) return false;
+        const could_be_fo_option = could_be_fo_option: {
+            var window_it = std.mem.window(u8, self.full[1..], 2, 1);
+            while (window_it.next()) |window| {
+                if (std.ascii.eqlIgnoreCase(window, "fo")) break :could_be_fo_option true;
+                // If we see '/' before "fo", then it's not possible for this to be a valid
+                // `/fo` option.
+                if (window[0] == '/') break;
+            }
+            break :could_be_fo_option false;
+        };
+        if (!could_be_fo_option) return true;
+        // It's still possible for a file path to look like a /fo option but not actually
+        // be one, e.g. `/foo/bar.rc`. As a last ditch effort to reduce false negatives,
+        // check if the file path exists and, if so, then we ignore the 'could be /fo option'-ness
+        std.fs.accessAbsolute(self.full, .{}) catch return false;
+        return true;
+    }
     pub const Value = struct {
         slice: []const u8,
         index_increment: u2 = 1,
 @@ -432,6 +452,16 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
             }
         }
+        const args_remaining = args.len - arg_i;
+        if (args_remaining <= 2 and arg.looksLikeFilepath()) {
+            var err_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i };
+            var msg_writer = err_details.msg.writer(allocator);
+            try msg_writer.writeAll("this argument was inferred to be a filepath, so argument parsing was terminated");
+            try diagnostics.append(err_details);
+            break;
+        }
         while (arg.name().len > 0) {
             const arg_name = arg.name();
             // Note: These cases should be in order from longest to shortest, since
 @@ -440,24 +470,6 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
             if (std.ascii.startsWithIgnoreCase(arg_name, ":no-preprocess")) {
                 options.preprocess = .no;
                 arg.name_offset += ":no-preprocess".len;
-            } else if (std.ascii.startsWithIgnoreCase(arg_name, ":mingw-includes")) {
-                const value = arg.value(":mingw-includes".len, arg_i, args) catch {
-                    var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() };
-                    var msg_writer = err_details.msg.writer(allocator);
-                    try msg_writer.print("missing value after {s}{s} option", .{ arg.prefixSlice(), arg.optionWithoutPrefix(":mingw-includes".len) });
-                    try diagnostics.append(err_details);
-                    arg_i += 1;
-                    break :next_arg;
-                };
-                if (options.mingw_includes_dir) |overwritten_path| {
-                    allocator.free(overwritten_path);
-                    options.mingw_includes_dir = null;
-                }
-                const path = try allocator.dupe(u8, value.slice);
-                errdefer allocator.free(path);
-                options.mingw_includes_dir = path;
-                arg_i += value.index_increment;
-                continue :next_arg;
             } else if (std.ascii.startsWithIgnoreCase(arg_name, ":auto-includes")) {
                 const value = arg.value(":auto-includes".len, arg_i, args) catch {
                     var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() };
 @@ -769,7 +781,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
                     arg_i += value.index_increment;
                     continue :next_arg;
                 };
-                options.default_code_page = CodePage.getByIdentifierEnsureSupported(code_page_id) catch |err| switch (err) {
+                options.default_code_page = code_pages.getByIdentifierEnsureSupported(code_page_id) catch |err| switch (err) {
                     error.InvalidCodePage => {
                         var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) };
                         var msg_writer = err_details.msg.writer(allocator);
 @@ -782,7 +794,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
                         var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) };
                         var msg_writer = err_details.msg.writer(allocator);
                         try msg_writer.print("unsupported code page: {s} (id={})", .{
-                            @tagName(CodePage.getByIdentifier(code_page_id) catch unreachable),
+                            @tagName(code_pages.getByIdentifier(code_page_id) catch unreachable),
                             code_page_id,
                         });
                         try diagnostics.append(err_details);
 @@ -900,18 +912,20 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
     const positionals = args[arg_i..];
-    if (positionals.len < 1) {
+    if (positionals.len == 0) {
         var err_details = Diagnostics.ErrorDetails{ .print_args = false, .arg_index = arg_i };
         var msg_writer = err_details.msg.writer(allocator);
         try msg_writer.writeAll("missing input filename");
         try diagnostics.append(err_details);
-        const last_arg = args[args.len - 1];
+        if (args.len > 0) {
-        if (arg_i > 0 and last_arg.len > 0 and last_arg[0] == '/' and std.ascii.endsWithIgnoreCase(last_arg, ".rc")) {
+            const last_arg = args[args.len - 1];
-            var note_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i - 1 };
+            if (arg_i > 0 and last_arg.len > 0 and last_arg[0] == '/' and std.ascii.endsWithIgnoreCase(last_arg, ".rc")) {
-            var note_writer = note_details.msg.writer(allocator);
+                var note_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i - 1 };
-            try note_writer.writeAll("if this argument was intended to be the input filename, then -- should be specified in front of it to exclude it from option parsing");
+                var note_writer = note_details.msg.writer(allocator);
-            try diagnostics.append(note_details);
+                try note_writer.writeAll("if this argument was intended to be the input filename, then -- should be specified in front of it to exclude it from option parsing");
+                try diagnostics.append(note_details);
+            }
         }
         // This is a fatal enough problem to justify an early return, since
 @@ -969,6 +983,12 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
     return options;
 }
+pub fn isSupportedInputExtension(ext: []const u8) bool {
+    if (std.ascii.eqlIgnoreCase(ext, ".rc")) return true;
+    if (std.ascii.eqlIgnoreCase(ext, ".rcpp")) return true;
+    return false;
+}
 /// Returns true if the str is a valid C identifier for use in a #define/#undef macro
 pub fn isValidIdentifier(str: []const u8) bool {
     for (str, 0..) |c, i| switch (c) {
 @@ -1271,6 +1291,43 @@ test "parse errors: basic" {
     );
 }
+test "inferred absolute filepaths" {
+    {
+        var options = try testParseWarning(&.{ "/fo", "foo.res", "/home/absolute/path.rc" },
+            \\<cli>: note: this argument was inferred to be a filepath, so argument parsing was terminated
+            \\ ... /home/absolute/path.rc
+            \\     ^~~~~~~~~~~~~~~~~~~~~~
+            \\
+        );
+        defer options.deinit();
+    }
+    {
+        var options = try testParseWarning(&.{ "/home/absolute/path.rc", "foo.res" },
+            \\<cli>: note: this argument was inferred to be a filepath, so argument parsing was terminated
+            \\ ... /home/absolute/path.rc ...
+            \\     ^~~~~~~~~~~~~~~~~~~~~~
+            \\
+        );
+        defer options.deinit();
+    }
+    {
+        // Only the last two arguments are checked, so the /h is parsed as an option
+        var options = try testParse(&.{ "/home/absolute/path.rc", "foo.rc", "foo.res" });
+        defer options.deinit();
+        try std.testing.expect(options.print_help_and_exit);
+    }
+    {
+        var options = try testParse(&.{ "/xvFO/some/absolute/path.res", "foo.rc" });
+        defer options.deinit();
+        try std.testing.expectEqual(true, options.verbose);
+        try std.testing.expectEqual(true, options.ignore_include_env_var);
+        try std.testing.expectEqualStrings("foo.rc", options.input_source.filename);
+        try std.testing.expectEqualStrings("/some/absolute/path.res", options.output_source.filename);
+    }
+}
 test "parse errors: /ln" {
     try testParseError(&.{ "/ln", "invalid", "foo.rc" },
         \\<cli>: error: invalid language tag: invalid

lib/compiler/resinator/code_pages.zig+78-139

 @@ -1,86 +1,30 @@
 const std = @import("std");
 const windows1252 = @import("windows1252.zig");
-// TODO: Parts of this comment block may be more relevant to string/NameOrOrdinal parsing
-//       than it is to the stuff in this file.
-//
-// ‰ representations for context:
-// Win-1252   89
-// UTF-8      E2 80 B0
-// UTF-16     20 30
-//
-// With code page 65001:
-//  ‰ RCDATA { "‰" L"‰" }
-// File encoded as Windows-1252:
-//  ‰ => <U+FFFD REPLACEMENT CHARACTER> as u16
-//  "‰" => 0x3F ('?')
-//  L"‰" => <U+FFFD REPLACEMENT CHARACTER> as u16
-// File encoded as UTF-8:
-//  ‰ => <U+2030 ‰> as u16
-//  "‰" => 0x89 ('‰' encoded as Windows-1252)
-//  L"‰" => <U+2030 ‰> as u16
-//
-// With code page 1252:
-//  ‰ RCDATA { "‰" L"‰" }
-// File encoded as Windows-1252:
-//  ‰ => <U+2030 ‰> as u16
-//  "‰" => 0x89 ('‰' encoded as Windows-1252)
-//  L"‰" => <U+2030 ‰> as u16
-// File encoded as UTF-8:
-//  ‰ => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16
-//       ^ first byte of utf8 representation
-//                    ^ second byte of UTF-8 representation (0x80), but interpretted as
-//                      Windows-1252 ('€') and then converted to UTF-16 (<U+20AC>)
-//                                   ^ third byte of utf8 representation
-//  "‰" => 0xE2, 0x80, 0xB0 (the bytes of the UTF-8 representation)
-//  L"‰" => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16 (see '‰ =>' explanation)
-//
-// With code page 1252:
-//  <0x90> RCDATA { "<0x90>" L"<0x90>" }
-// File encoded as Windows-1252:
-//  <0x90> => 0x90 as u16
-//  "<0x90>" => 0x90
-//  L"<0x90>" => 0x90 as u16
-// File encoded as UTF-8:
-//  <0x90> => 0xC2 as u16, 0x90 as u16
-//  "<0x90>" => 0xC2, 0x90 (the bytes of the UTF-8 representation of <U+0090>)
-//  L"<0x90>" => 0xC2 as u16, 0x90 as u16
-//
-// Within a raw data block, file encoded as Windows-1252 (Â is <0xC2>):
-//  "Âa" L"Âa" "\xC2ad" L"\xC2AD"
-// With code page 1252:
-//  C2 61 C2 00 61 00 C2 61 64 AD C2
-//  Â^ a^ Â~~~^ a~~~^ .^ a^ d^ ^~~~~\xC2AD
-//              \xC2~`
-// With code page 65001:
-//  3F 61 FD FF 61 00 C2 61 64 AD C2
-//  ^. a^ ^~~~. a~~~^ ^. a^ d^ ^~~~~\xC2AD
-//    `.       `.       `~\xC2
-//      `.       `.~<0xC2>a is not well-formed UTF-8 (0xC2 expects a continutation byte after it).
-//        `.        Because 'a' is a valid first byte of a UTF-8 sequence, it is not included in the
-//          `.      invalid sequence so only the <0xC2> gets converted to <U+FFFD>.
-//            `~Same as ^ but converted to '?' instead.
-//
-// Within a raw data block, file encoded as Windows-1252 (ð is <0xF0>, € is <0x80>):
-//  "ð€a" L"ð€a"
-// With code page 1252:
-//  F0 80 61 F0 00 AC 20 61 00
-//  ð^ €^ a^ ð~~~^ €~~~^ a~~~^
-// With code page 65001:
-//  3F 61 FD FF 61 00
-//  ^. a^ ^~~~. a~~~^
-//    `.       `.
-//      `.       `.~<0xF0><0x80> is not well-formed UTF-8, and <0x80> is not a valid first byte, so
-//        `.        both bytes are considered an invalid sequence and get converted to '<U+FFFD>'
-//          `~Same as ^ but converted to '?' instead.
 /// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
-pub const CodePage = enum(u16) {
+pub const SupportedCodePage = enum(u16) {
-    // supported
     windows1252 = 1252, // windows-1252    ANSI Latin 1; Western European (Windows)
     utf8 = 65001, // utf-8    Unicode (UTF-8)
-    // unsupported but valid
+    pub fn codepointAt(code_page: SupportedCodePage, index: usize, bytes: []const u8) ?Codepoint {
+        if (index >= bytes.len) return null;
+        switch (code_page) {
+            .windows1252 => {
+                // All byte values have a representation, so just convert the byte
+                return Codepoint{
+                    .value = windows1252.toCodepoint(bytes[index]),
+                    .byte_len = 1,
+                };
+            },
+            .utf8 => {
+                return Utf8.WellFormedDecoder.decode(bytes[index..]);
+            },
+        }
+    }
+};
+/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
+pub const UnsupportedCodePage = enum(u16) {
     ibm037 = 37, // IBM037    IBM EBCDIC US-Canada
     ibm437 = 437, // IBM437    OEM United States
     ibm500 = 500, // IBM500    IBM EBCDIC International
 @@ -231,50 +175,45 @@ pub const CodePage = enum(u16) {
     x_iscii_gu = 57010, // x-iscii-gu    ISCII Gujarati
     x_iscii_pa = 57011, // x-iscii-pa    ISCII Punjabi
     utf7 = 65000, // utf-7    Unicode (UTF-7)
+};
-    pub fn codepointAt(code_page: CodePage, index: usize, bytes: []const u8) ?Codepoint {
+pub const CodePage = blk: {
-        if (index >= bytes.len) return null;
+    const fields = @typeInfo(SupportedCodePage).@"enum".fields ++ @typeInfo(UnsupportedCodePage).@"enum".fields;
-        switch (code_page) {
+    break :blk @Type(.{ .@"enum" = .{
-            .windows1252 => {
+        .tag_type = u16,
-                // All byte values have a representation, so just convert the byte
+        .decls = &.{},
-                return Codepoint{
+        .fields = fields,
-                    .value = windows1252.toCodepoint(bytes[index]),
+        .is_exhaustive = true,
-                    .byte_len = 1,
+    } });
-                };
+};
-            },
-            .utf8 => {
-                return Utf8.WellFormedDecoder.decode(bytes[index..]);
-            },
-            else => unreachable,
-        }
-    }
-    pub fn isSupported(code_page: CodePage) bool {
-        return switch (code_page) {
-            .windows1252, .utf8 => true,
-            else => false,
-        };
-    }
-    pub fn getByIdentifier(identifier: u16) !CodePage {
+pub fn isSupported(code_page: CodePage) bool {
-        // There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but
+    inline for (@typeInfo(SupportedCodePage).@"enum".fields) |enumField| {
-        // this should be fine, especially since this function likely won't be called much.
+        if (@intFromEnum(code_page) == @intFromEnum(@field(SupportedCodePage, enumField.name))) {
-        inline for (@typeInfo(CodePage).@"enum".fields) |enumField| {
+            return true;
-            if (identifier == enumField.value) {
-                return @field(CodePage, enumField.name);
-            }
         }
-        return error.InvalidCodePage;
     }
+    return false;
+}
-    pub fn getByIdentifierEnsureSupported(identifier: u16) !CodePage {
+pub fn getByIdentifier(identifier: u16) !CodePage {
-        const code_page = try getByIdentifier(identifier);
+    // There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but
-        switch (isSupported(code_page)) {
+    // this should be fine, especially since this function likely won't be called much.
-            true => return code_page,
+    inline for (@typeInfo(CodePage).@"enum".fields) |enumField| {
-            false => return error.UnsupportedCodePage,
+        if (identifier == enumField.value) {
+            return @field(CodePage, enumField.name);
         }
     }
-};
+    return error.InvalidCodePage;
+}
+pub fn getByIdentifierEnsureSupported(identifier: u16) !SupportedCodePage {
+    const code_page = try getByIdentifier(identifier);
+    return if (isSupported(code_page))
+        @enumFromInt(@intFromEnum(code_page))
+    else
+        error.UnsupportedCodePage;
+}
 pub const Utf8 = struct {
     /// Implements decoding with rejection of ill-formed UTF-8 sequences based on section
 @@ -378,20 +317,20 @@ test "codepointAt invalid utf8" {
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(0, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 2,
-        }, CodePage.utf8.codepointAt(1, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(3, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(3, invalid_utf8).?);
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(4, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(4, invalid_utf8).?);
-        try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(5, invalid_utf8));
+        try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(5, invalid_utf8));
     }
     {
 @@ -399,12 +338,12 @@ test "codepointAt invalid utf8" {
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 2,
-        }, CodePage.utf8.codepointAt(0, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(2, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
-        try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(3, invalid_utf8));
+        try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(3, invalid_utf8));
     }
     {
 @@ -412,8 +351,8 @@ test "codepointAt invalid utf8" {
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(0, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
-        try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, invalid_utf8));
+        try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(1, invalid_utf8));
     }
     {
 @@ -421,8 +360,8 @@ test "codepointAt invalid utf8" {
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 2,
-        }, CodePage.utf8.codepointAt(0, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
-        try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8));
+        try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
     }
     {
 @@ -430,12 +369,12 @@ test "codepointAt invalid utf8" {
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(0, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(1, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
-        try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8));
+        try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
     }
     {
 @@ -444,11 +383,11 @@ test "codepointAt invalid utf8" {
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 2,
-        }, CodePage.utf8.codepointAt(0, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
         try std.testing.expectEqual(Codepoint{
             .value = Codepoint.invalid,
             .byte_len = 1,
-        }, CodePage.utf8.codepointAt(2, invalid_utf8).?);
+        }, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
     }
 }
 @@ -459,19 +398,19 @@ test "codepointAt utf8 encoded" {
     try std.testing.expectEqual(Codepoint{
         .value = '²',
         .byte_len = 2,
-    }, CodePage.utf8.codepointAt(0, utf8_encoded).?);
+    }, SupportedCodePage.utf8.codepointAt(0, utf8_encoded).?);
-    try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, utf8_encoded));
+    try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, utf8_encoded));
     // with code page windows1252
     try std.testing.expectEqual(Codepoint{
         .value = '\xC2',
         .byte_len = 1,
-    }, CodePage.windows1252.codepointAt(0, utf8_encoded).?);
+    }, SupportedCodePage.windows1252.codepointAt(0, utf8_encoded).?);
     try std.testing.expectEqual(Codepoint{
         .value = '\xB2',
         .byte_len = 1,
-    }, CodePage.windows1252.codepointAt(1, utf8_encoded).?);
+    }, SupportedCodePage.windows1252.codepointAt(1, utf8_encoded).?);
-    try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, utf8_encoded));
+    try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(2, utf8_encoded));
 }
 test "codepointAt windows1252 encoded" {
 @@ -481,15 +420,15 @@ test "codepointAt windows1252 encoded" {
     try std.testing.expectEqual(Codepoint{
         .value = Codepoint.invalid,
         .byte_len = 1,
-    }, CodePage.utf8.codepointAt(0, windows1252_encoded).?);
+    }, SupportedCodePage.utf8.codepointAt(0, windows1252_encoded).?);
-    try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, windows1252_encoded));
+    try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, windows1252_encoded));
     // with code page windows1252
     try std.testing.expectEqual(Codepoint{
         .value = '\xB2',
         .byte_len = 1,
-    }, CodePage.windows1252.codepointAt(0, windows1252_encoded).?);
+    }, SupportedCodePage.windows1252.codepointAt(0, windows1252_encoded).?);
-    try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, windows1252_encoded));
+    try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(1, windows1252_encoded));
 }
 pub const Codepoint = struct {

lib/compiler/resinator/comments.zig+25

...	@@ -174,6 +174,21 @@ pub fn removeComments(source: []const u8, buf: []u8, source_mappings: ?*SourceMa	...	@@ -174,6 +174,21 @@ pub fn removeComments(source: []const u8, buf: []u8, source_mappings: ?*SourceMa
174	},	174	},
175	},	175	},
176	}	176	}
		177	} else {
		178	switch (state) {
		179	.start,
		180	.line_comment,
		181	.multiline_comment,
		182	.multiline_comment_end,
		183	.single_quoted,
		184	.single_quoted_escape,
		185	.double_quoted,
		186	.double_quoted_escape,
		187	=> {},
		188	.forward_slash => {
		189	result.writeSlice(source[pending_start.?..index]);
		190	},
		191	}
177	}	192	}
178	return result.getWritten();	193	return result.getWritten();
179	}	194	}
...	@@ -334,6 +349,16 @@ test "comments appended to a line" {	...	@@ -334,6 +349,16 @@ test "comments appended to a line" {
334	);	349	);
335	}	350	}
336		351
		352	test "forward slash only" {
		353	try testRemoveComments(
		354	\\ /
		355	\\/
		356	,
		357	\\ /
		358	\\/
		359	);
		360	}
		361
337	test "remove comments with mappings" {	362	test "remove comments with mappings" {
338	const allocator = std.testing.allocator;	363	const allocator = std.testing.allocator;
339	var mut_source = "blah/\rcommented line\r/blah".*;	364	var mut_source = "blah/\rcommented line\r/blah".*;

lib/compiler/resinator/compile.zig+165-147

 @@ -4,7 +4,7 @@ const Allocator = std.mem.Allocator;
 const Node = @import("ast.zig").Node;
 const lex = @import("lex.zig");
 const Parser = @import("parse.zig").Parser;
-const Resource = @import("rc.zig").Resource;
+const ResourceType = @import("rc.zig").ResourceType;
 const Token = @import("lex.zig").Token;
 const literals = @import("literals.zig");
 const Number = literals.Number;
 @@ -21,7 +21,7 @@ const WORD = std.os.windows.WORD;
 const DWORD = std.os.windows.DWORD;
 const utils = @import("utils.zig");
 const NameOrOrdinal = res.NameOrOrdinal;
-const CodePage = @import("code_pages.zig").CodePage;
+const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
 const CodePageLookup = @import("ast.zig").CodePageLookup;
 const SourceMappings = @import("source_mapping.zig").SourceMappings;
 const windows1252 = @import("windows1252.zig");
 @@ -39,7 +39,10 @@ pub const CompileOptions = struct {
     /// freed by the caller.
     /// TODO: Maybe a dedicated struct for this purpose so that it's a bit nicer to work with.
     dependencies_list: ?*std.ArrayList([]const u8) = null,
-    default_code_page: CodePage = .windows1252,
+    default_code_page: SupportedCodePage = .windows1252,
+    /// If true, the first #pragma code_page directive only sets the input code page, but not the output code page.
+    /// This check must be done before comments are removed from the file.
+    disjoint_code_page: bool = false,
     ignore_include_env_var: bool = false,
     extra_include_paths: []const []const u8 = &.{},
     /// This is just an API convenience to allow separately passing 'system' (i.e. those
 @@ -66,6 +69,7 @@ pub fn compile(allocator: Allocator, source: []const u8, writer: anytype, option
     });
     var parser = Parser.init(&lexer, .{
         .warn_instead_of_error_on_invalid_code_page = options.warn_instead_of_error_on_invalid_code_page,
+        .disjoint_code_page = options.disjoint_code_page,
     });
     var tree = try parser.parse(allocator, options.diagnostics);
     defer tree.deinit();
 @@ -98,6 +102,7 @@ pub fn compile(allocator: Allocator, source: []const u8, writer: anytype, option
                 .end = 0,
                 .line_number = 1,
             },
+            .code_page = .utf8,
             .print_source_line = false,
             .extra = .{ .file_open_error = .{
                 .err = ErrorDetails.FileOpenError.enumFromError(err),
 @@ -213,7 +218,12 @@ pub const Compiler = struct {
             try self.addErrorDetails(.{
                 .err = .result_contains_fontdir,
                 .type = .hint,
-                .token = undefined,
+                .token = .{
+                    .id = .invalid,
+                    .start = 0,
+                    .end = 0,
+                    .line_number = 1,
+                },
             });
         }
         // once we've written every else out, we can write out the finalized STRINGTABLE resources
 @@ -301,7 +311,10 @@ pub const Compiler = struct {
                         // UTF-8, we can parse either string type directly to UTF-8.
                         var parser = literals.IterativeStringParser.init(bytes, .{
                             .start_column = column,
-                            .diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token },
+                            .diagnostics = self.errContext(literal_node.token),
+                            // TODO: Re-evaluate this. It's not been tested whether or not using the actual
+                            //       output code page would make more sense.
+                            .output_code_page = .windows1252,
                         });
                         while (try parser.nextUnchecked()) |parsed| {
 @@ -401,56 +414,55 @@ pub const Compiler = struct {
         return first_error orelse error.FileNotFound;
     }
+    /// Returns a Windows-1252 encoded string regardless of the current output code page.
+    /// All codepoints are encoded as a maximum of 2 bytes, where unescaped codepoints
+    /// >= 0x10000 are encoded as `??` and everything else is encoded as 1 byte.
     pub fn parseDlgIncludeString(self: *Compiler, token: Token) ![]u8 {
-        // For the purposes of parsing, we want to strip the L prefix
+        const bytes = self.sourceBytesForToken(token);
-        // if it exists since we want escaped integers to be limited to
+        const output_code_page = self.output_code_pages.getForToken(token);
-        // their ascii string range.
-        //
-        // We keep track of whether or not there was an L prefix, though,
-        // since there's more weirdness to come.
-        var bytes = self.sourceBytesForToken(token);
-        var was_wide_string = false;
-        if (bytes.slice[0] == 'L' or bytes.slice[0] == 'l') {
-            was_wide_string = true;
-            bytes.slice = bytes.slice[1..];
-        }
         var buf = try std.ArrayList(u8).initCapacity(self.allocator, bytes.slice.len);
         errdefer buf.deinit();
         var iterative_parser = literals.IterativeStringParser.init(bytes, .{
             .start_column = token.calculateColumn(self.source, 8, null),
-            .diagnostics = .{ .diagnostics = self.diagnostics, .token = token },
+            .diagnostics = self.errContext(token),
+            // TODO: Potentially re-evaluate this, it's not been tested whether or not
+            //       using the actual output code page would make more sense.
+            .output_code_page = .windows1252,
         });
-        // No real idea what's going on here, but this matches the rc.exe behavior
+        // This is similar to the logic in parseQuotedString, but ends up with everything
+        // encoded as Windows-1252. This effectively consolidates the two-step process
+        // of rc.exe into one step, since rc.exe's preprocessor converts to UTF-16 (this
+        // is when invalid sequences are replaced by the replacement character (U+FFFD)),
+        // and then that's run through the parser. Our preprocessor keeps things in their
+        // original encoding, meaning we emulate the <encoding> -> UTF-16 -> Windows-1252
+        // results all at once.
         while (try iterative_parser.next()) |parsed| {
             const c = parsed.codepoint;
-            switch (was_wide_string) {
+            switch (iterative_parser.declared_string_type) {
-                true => {
+                .wide => {
-                    switch (c) {
+                    if (windows1252.bestFitFromCodepoint(c)) |best_fit| {
-                        0...0x7F, 0xA0...0xFF => try buf.append(@intCast(c)),
+                        try buf.append(best_fit);
-                        0x80...0x9F => {
+                    } else if (c < 0x10000 or c == code_pages.Codepoint.invalid or parsed.escaped_surrogate_pair) {
-                            if (windows1252.bestFitFromCodepoint(c)) |_| {
+                        try buf.append('?');
-                                try buf.append(@intCast(c));
+                    } else {
-                            } else {
+                        try buf.appendSlice("??");
-                                try buf.append('?');
-                            }
-                        },
-                        else => {
-                            if (windows1252.bestFitFromCodepoint(c)) |best_fit| {
-                                try buf.append(best_fit);
-                            } else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
-                                try buf.append('?');
-                            } else {
-                                try buf.appendSlice("??");
-                            }
-                        },
                     }
                 },
-                false => {
+                .ascii => {
                     if (parsed.from_escaped_integer) {
-                        try buf.append(@truncate(c));
+                        const truncated: u8 = @truncate(c);
+                        switch (output_code_page) {
+                            .utf8 => switch (truncated) {
+                                0...0x7F => try buf.append(truncated),
+                                else => try buf.append('?'),
+                            },
+                            .windows1252 => {
+                                try buf.append(truncated);
+                            },
+                        }
                     } else {
                         if (windows1252.bestFitFromCodepoint(c)) |best_fit| {
                             try buf.append(best_fit);
 @@ -484,8 +496,12 @@ pub const Compiler = struct {
             const parsed_filename_terminated = std.mem.sliceTo(parsed_filename, 0);
             header.applyMemoryFlags(node.common_resource_attributes, self.source);
+            // This is effectively limited by `max_string_literal_codepoints` which is a u15.
+            // Each codepoint within a DLGINCLUDE string is encoded as a maximum of
+            // 2 bytes, which means that the maximum byte length of a DLGINCLUDE string is
+            // (including the NUL terminator): 32,767 * 2 + 1 = 65,535 or exactly the u16 max.
             header.data_size = @intCast(parsed_filename_terminated.len + 1);
-            try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+            try header.write(writer, self.errContext(node.id));
             try writer.writeAll(parsed_filename_terminated);
             try writer.writeByte(0);
             try writeDataPadding(writer, header.data_size);
 @@ -568,7 +584,7 @@ pub const Compiler = struct {
                         header.applyMemoryFlags(node.common_resource_attributes, self.source);
                         header.data_size = @intCast(try file.getEndPos());
-                        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+                        try header.write(writer, self.errContext(node.id));
                         try file.seekTo(0);
                         try writeResourceData(writer, file.reader(), header.data_size);
                         return;
 @@ -644,7 +660,7 @@ pub const Compiler = struct {
                             .version = self.state.version,
                             .characteristics = self.state.characteristics,
                         };
-                        try image_header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+                        try image_header.write(writer, self.errContext(node.id));
                         // From https://learn.microsoft.com/en-us/windows/win32/menurc/localheader:
                         // > The LOCALHEADER structure is the first data written to the RT_CURSOR
 @@ -817,12 +833,26 @@ pub const Compiler = struct {
                     header.data_size = icon_dir.getResDataSize();
-                    try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+                    try header.write(writer, self.errContext(node.id));
                     try icon_dir.writeResData(writer, first_icon_id);
                     try writeDataPadding(writer, header.data_size);
                     return;
                 },
-                .RCDATA, .HTML, .MANIFEST, .MESSAGETABLE, .DLGINIT, .PLUGPLAY => {
+                .RCDATA,
+                .HTML,
+                .MESSAGETABLE,
+                .DLGINIT,
+                .PLUGPLAY,
+                .VXD,
+                // Note: All of the below can only be specified by using a number
+                //       as the resource type.
+                .MANIFEST,
+                .CURSOR,
+                .ICON,
+                .ANICURSOR,
+                .ANIICON,
+                .FONTDIR,
+                => {
                     header.applyMemoryFlags(node.common_resource_attributes, self.source);
                 },
                 .BITMAP => {
 @@ -855,47 +885,32 @@ pub const Compiler = struct {
                     } else if (bitmap_info.getActualPaletteByteLen() < bitmap_info.getExpectedPaletteByteLen()) {
                         const num_padding_bytes = bitmap_info.getExpectedPaletteByteLen() - bitmap_info.getActualPaletteByteLen();
-                        // TODO: Make this configurable (command line option)
-                        const max_missing_bytes = 4096;
-                        if (num_padding_bytes > max_missing_bytes) {
-                            var numbers_as_bytes: [16]u8 = undefined;
-                            std.mem.writeInt(u64, numbers_as_bytes[0..8], num_padding_bytes, native_endian);
-                            std.mem.writeInt(u64, numbers_as_bytes[8..16], max_missing_bytes, native_endian);
-                            const values_string_index = try self.diagnostics.putString(&numbers_as_bytes);
-                            try self.addErrorDetails(.{
-                                .err = .bmp_too_many_missing_palette_bytes,
-                                .token = filename_token,
-                                .extra = .{ .number = values_string_index },
-                            });
-                            return self.addErrorDetailsAndFail(.{
-                                .err = .bmp_too_many_missing_palette_bytes,
-                                .type = .note,
-                                .print_source_line = false,
-                                .token = filename_token,
-                            });
-                        }
                         var number_as_bytes: [8]u8 = undefined;
                         std.mem.writeInt(u64, &number_as_bytes, num_padding_bytes, native_endian);
                         const value_string_index = try self.diagnostics.putString(&number_as_bytes);
                         try self.addErrorDetails(.{
                             .err = .bmp_missing_palette_bytes,
-                            .type = .warning,
+                            .type = .err,
                             .token = filename_token,
                             .extra = .{ .number = value_string_index },
                         });
                         const pixel_data_len = bitmap_info.getPixelDataLen(file_size);
+                        // TODO: This is a hack, but we know we have already added
+                        //       at least one entry to the diagnostics strings, so we can
+                        //       get away with using 0 to mean 'no string' here.
+                        var miscompiled_bytes_string_index: u32 = 0;
                         if (pixel_data_len > 0) {
                             const miscompiled_bytes = @min(pixel_data_len, num_padding_bytes);
                             std.mem.writeInt(u64, &number_as_bytes, miscompiled_bytes, native_endian);
-                            const miscompiled_bytes_string_index = try self.diagnostics.putString(&number_as_bytes);
+                            miscompiled_bytes_string_index = try self.diagnostics.putString(&number_as_bytes);
-                            try self.addErrorDetails(.{
-                                .err = .rc_would_miscompile_bmp_palette_padding,
-                                .type = .warning,
-                                .token = filename_token,
-                                .extra = .{ .number = miscompiled_bytes_string_index },
-                            });
                         }
+                        return self.addErrorDetailsAndFail(.{
+                            .err = .rc_would_miscompile_bmp_palette_padding,
+                            .type = .note,
+                            .print_source_line = false,
+                            .token = filename_token,
+                            .extra = .{ .number = miscompiled_bytes_string_index },
+                        });
                     }
                     // TODO: It might be possible that the calculation done in this function
 @@ -905,7 +920,7 @@ pub const Compiler = struct {
                     const bmp_bytes_to_write: u32 = @intCast(bitmap_info.getExpectedByteLen(file_size));
                     header.data_size = bmp_bytes_to_write;
-                    try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+                    try header.write(writer, self.errContext(node.id));
                     try file.seekTo(bmp.file_header_len);
                     const file_reader = file.reader();
                     try writeResourceDataNoPadding(writer, file_reader, bitmap_info.dib_header_size);
 @@ -914,12 +929,6 @@ pub const Compiler = struct {
                     }
                     if (bitmap_info.getExpectedPaletteByteLen() > 0) {
                         try writeResourceDataNoPadding(writer, file_reader, @intCast(bitmap_info.getActualPaletteByteLen()));
-                        // We know that the number of missing palette bytes is <= 4096
-                        // (see `bmp_too_many_missing_palette_bytes` error case above)
-                        const padding_bytes: usize = @intCast(bitmap_info.getMissingPaletteByteLen());
-                        if (padding_bytes > 0) {
-                            try writer.writeByteNTimes(0, padding_bytes);
-                        }
                     }
                     try file.seekTo(bitmap_info.pixel_data_offset);
                     const pixel_bytes: u32 = @intCast(file_size - bitmap_info.pixel_data_offset);
 @@ -932,13 +941,13 @@ pub const Compiler = struct {
                         // Add warning and skip this resource
                         // Note: The Win32 compiler prints this as an error but it doesn't fail the compilation
                         // and the duplicate resource is skipped.
-                        try self.addErrorDetails(ErrorDetails{
+                        try self.addErrorDetails(.{
                             .err = .font_id_already_defined,
                             .token = node.id,
                             .type = .warning,
                             .extra = .{ .number = header.name_value.ordinal },
                         });
-                        try self.addErrorDetails(ErrorDetails{
+                        try self.addErrorDetails(.{
                             .err = .font_id_already_defined,
                             .token = self.state.font_dir.ids.get(header.name_value.ordinal).?,
                             .type = .note,
 @@ -957,7 +966,7 @@ pub const Compiler = struct {
                     // We now know that the data size will fit in a u32
                     header.data_size = @intCast(file_size);
-                    try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+                    try header.write(writer, self.errContext(node.id));
                     var header_slurping_reader = headerSlurpingReader(148, file.reader());
                     try writeResourceData(writer, header_slurping_reader.reader(), header.data_size);
 @@ -968,19 +977,13 @@ pub const Compiler = struct {
                     }, node.id);
                     return;
                 },
-                .ACCELERATOR,
+                .ACCELERATOR, // Cannot use an external file, enforced by the parser
-                .ANICURSOR,
+                .DIALOG, // Cannot use an external file, enforced by the parser
-                .ANIICON,
+                .DLGINCLUDE, // Handled specially above
-                .CURSOR,
+                .MENU, // Cannot use an external file, enforced by the parser
-                .DIALOG,
+                .STRING, // Parser error if this resource is specified as a number
-                .DLGINCLUDE,
+                .TOOLBAR, // Cannot use an external file, enforced by the parser
-                .FONTDIR,
+                .VERSION, // Cannot use an external file, enforced by the parser
-                .ICON,
-                .MENU,
-                .STRING,
-                .TOOLBAR,
-                .VERSION,
-                .VXD,
                 => unreachable,
                 _ => unreachable,
             }
 @@ -998,7 +1001,7 @@ pub const Compiler = struct {
         }
         // We now know that the data size will fit in a u32
         header.data_size = @intCast(data_size);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+        try header.write(writer, self.errContext(node.id));
         try writeResourceData(writer, file.reader(), header.data_size);
     }
 @@ -1188,7 +1191,7 @@ pub const Compiler = struct {
                         };
                         const parsed = try literals.parseQuotedAsciiString(self.allocator, bytes, .{
                             .start_column = column,
-                            .diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token },
+                            .diagnostics = self.errContext(literal_node.token),
                             .output_code_page = self.output_code_pages.getForToken(literal_node.token),
                         });
                         errdefer self.allocator.free(parsed);
 @@ -1202,7 +1205,8 @@ pub const Compiler = struct {
                         };
                         const parsed_string = try literals.parseQuotedWideString(self.allocator, bytes, .{
                             .start_column = column,
-                            .diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token },
+                            .diagnostics = self.errContext(literal_node.token),
+                            .output_code_page = self.output_code_pages.getForToken(literal_node.token),
                         });
                         errdefer self.allocator.free(parsed_string);
                         return .{ .wide_string = parsed_string };
 @@ -1259,7 +1263,7 @@ pub const Compiler = struct {
         header.applyMemoryFlags(common_resource_attributes, self.source);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = id_token });
+        try header.write(writer, self.errContext(id_token));
     }
     pub fn writeResourceDataNoPadding(writer: anytype, data_reader: anytype, data_size: u32) !void {
 @@ -1297,7 +1301,8 @@ pub const Compiler = struct {
             const column = literal.token.calculateColumn(self.source, 8, null);
             return res.parseAcceleratorKeyString(bytes, is_virt, .{
                 .start_column = column,
-                .diagnostics = .{ .diagnostics = self.diagnostics, .token = literal.token },
+                .diagnostics = self.errContext(literal.token),
+                .output_code_page = self.output_code_pages.getForToken(literal.token),
             });
         }
     }
 @@ -1332,7 +1337,7 @@ pub const Compiler = struct {
         header.applyMemoryFlags(node.common_resource_attributes, self.source);
         header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+        try header.write(writer, self.errContext(node.id));
         var data_fbs = std.io.fixedBufferStream(data_buffer.items);
         try writeResourceData(writer, data_fbs.reader(), data_size);
 @@ -1348,6 +1353,16 @@ pub const Compiler = struct {
                 const modifier = rc.AcceleratorTypeAndOptions.map.get(type_or_option.slice(self.source)).?;
                 modifiers.apply(modifier);
             }
+            if ((modifiers.isSet(.control) or modifiers.isSet(.shift)) and !modifiers.isSet(.virtkey)) {
+                try self.addErrorDetails(.{
+                    .err = .accelerator_shift_or_control_without_virtkey,
+                    .type = .warning,
+                    // We know that one of SHIFT or CONTROL was specified, so there's at least one item
+                    // in this list.
+                    .token = accelerator.type_and_options[0],
+                    .token_span_end = accelerator.type_and_options[accelerator.type_and_options.len - 1],
+                });
+            }
             if (accelerator.event.isNumberExpression() and !modifiers.explicit_ascii_or_virtkey) {
                 return self.addErrorDetailsAndFail(.{
                     .err = .accelerator_type_required,
 @@ -1399,7 +1414,7 @@ pub const Compiler = struct {
         var limited_writer = limitedWriter(data_buffer.writer(), std.math.maxInt(u32));
         const data_writer = limited_writer.writer();
-        const resource = Resource.fromString(.{
+        const resource = ResourceType.fromString(.{
             .slice = node.type.slice(self.source),
             .code_page = self.input_code_pages.getForToken(node.type),
         });
 @@ -1414,8 +1429,6 @@ pub const Compiler = struct {
                 menu.deinit(self.allocator);
             }
         }
-        var skipped_menu_or_classes = std.ArrayList(*Node.SimpleStatement).init(self.allocator);
-        defer skipped_menu_or_classes.deinit();
         var last_menu: *Node.SimpleStatement = undefined;
         var last_class: *Node.SimpleStatement = undefined;
         var last_menu_would_be_forced_ordinal = false;
 @@ -1445,9 +1458,6 @@ pub const Compiler = struct {
                         },
                         .class => {
                             const is_duplicate = optional_statement_values.class != null;
-                            if (is_duplicate) {
-                                try skipped_menu_or_classes.append(last_class);
-                            }
                             const forced_ordinal = is_duplicate and optional_statement_values.class.? == .ordinal;
                             // In the Win32 RC compiler, if any CLASS values that are interpreted as
                             // an ordinal exist, it affects all future CLASS statements and forces
 @@ -1475,9 +1485,6 @@ pub const Compiler = struct {
                         },
                         .menu => {
                             const is_duplicate = optional_statement_values.menu != null;
-                            if (is_duplicate) {
-                                try skipped_menu_or_classes.append(last_menu);
-                            }
                             const forced_ordinal = is_duplicate and optional_statement_values.menu.? == .ordinal;
                             // In the Win32 RC compiler, if any MENU values that are interpreted as
                             // an ordinal exist, it affects all future MENU statements and forces
 @@ -1561,22 +1568,6 @@ pub const Compiler = struct {
             }
         }
-        for (skipped_menu_or_classes.items) |simple_statement| {
-            const statement_identifier = simple_statement.identifier;
-            const statement_type = rc.OptionalStatements.dialog_map.get(statement_identifier.slice(self.source)) orelse continue;
-            try self.addErrorDetails(.{
-                .err = .duplicate_menu_or_class_skipped,
-                .type = .warning,
-                .token = simple_statement.identifier,
-                .token_span_start = simple_statement.base.getFirstToken(),
-                .token_span_end = simple_statement.base.getLastToken(),
-                .extra = .{ .menu_or_class = switch (statement_type) {
-                    .menu => .menu,
-                    .class => .class,
-                    else => unreachable,
-                } },
-            });
-        }
         // The Win32 RC compiler miscompiles the value in the following scenario:
         // Multiple CLASS parameters are specified and any of them are treated as a number, then
         // the last CLASS is always treated as a number no matter what
 @@ -1739,7 +1730,7 @@ pub const Compiler = struct {
         header.applyMemoryFlags(node.common_resource_attributes, self.source);
         header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+        try header.write(writer, self.errContext(node.id));
         var data_fbs = std.io.fixedBufferStream(data_buffer.items);
         try writeResourceData(writer, data_fbs.reader(), data_size);
 @@ -1749,7 +1740,7 @@ pub const Compiler = struct {
         self: *Compiler,
         node: *Node.Dialog,
         data_writer: anytype,
-        resource: Resource,
+        resource: ResourceType,
         optional_statement_values: *const DialogOptionalStatementValues,
         x: Number,
         y: Number,
 @@ -1809,7 +1800,7 @@ pub const Compiler = struct {
         self: *Compiler,
         control: *Node.ControlStatement,
         data_writer: anytype,
-        resource: Resource,
+        resource: ResourceType,
         bytes_written_so_far: u32,
         controls_by_id: *std.AutoHashMap(u32, *const Node.ControlStatement),
     ) !void {
 @@ -2053,7 +2044,7 @@ pub const Compiler = struct {
         header.applyMemoryFlags(node.common_resource_attributes, self.source);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+        try header.write(writer, self.errContext(node.id));
         var data_fbs = std.io.fixedBufferStream(data_buffer.items);
         try writeResourceData(writer, data_fbs.reader(), data_size);
 @@ -2067,7 +2058,7 @@ pub const Compiler = struct {
         node: *Node.FontStatement,
     };
-    pub fn writeDialogFont(self: *Compiler, resource: Resource, values: FontStatementValues, writer: anytype) !void {
+    pub fn writeDialogFont(self: *Compiler, resource: ResourceType, values: FontStatementValues, writer: anytype) !void {
         const node = values.node;
         const point_size = evaluateNumberExpression(node.point_size, self.source, self.input_code_pages);
         try writer.writeInt(u16, point_size.asWord(), .little);
 @@ -2104,7 +2095,7 @@ pub const Compiler = struct {
             .slice = node.type.slice(self.source),
             .code_page = self.input_code_pages.getForToken(node.type),
         };
-        const resource = Resource.fromString(type_bytes);
+        const resource = ResourceType.fromString(type_bytes);
         std.debug.assert(resource == .menu or resource == .menuex);
         self.writeMenuData(node, data_writer, resource) catch |err| switch (err) {
 @@ -2128,7 +2119,7 @@ pub const Compiler = struct {
         header.applyMemoryFlags(node.common_resource_attributes, self.source);
         header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+        try header.write(writer, self.errContext(node.id));
         var data_fbs = std.io.fixedBufferStream(data_buffer.items);
         try writeResourceData(writer, data_fbs.reader(), data_size);
 @@ -2136,7 +2127,7 @@ pub const Compiler = struct {
     /// Expects `data_writer` to be a LimitedWriter limited to u32, meaning all writes to
     /// the writer within this function could return error.NoSpaceLeft
-    pub fn writeMenuData(self: *Compiler, node: *Node.Menu, data_writer: anytype, resource: Resource) !void {
+    pub fn writeMenuData(self: *Compiler, node: *Node.Menu, data_writer: anytype, resource: ResourceType) !void {
         // menu header
         const version: u16 = if (resource == .menu) 0 else 1;
         try data_writer.writeInt(u16, version, .little);
 @@ -2393,7 +2384,7 @@ pub const Compiler = struct {
         header.applyMemoryFlags(node.common_resource_attributes, self.source);
-        try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
+        try header.write(writer, self.errContext(node.id));
         var data_fbs = std.io.fixedBufferStream(data_buffer.items);
         try writeResourceData(writer, data_fbs.reader(), data_size);
 @@ -2525,14 +2516,14 @@ pub const Compiler = struct {
                     // It might be nice to have these errors point to the ids rather than the
                     // string tokens, but that would mean storing the id token of each string
                     // which doesn't seem worth it just for slightly better error messages.
-                    try self.addErrorDetails(ErrorDetails{
+                    try self.addErrorDetails(.{
                         .err = .string_already_defined,
                         .token = string.string,
                         .extra = .{ .string_and_language = .{ .id = string_id, .language = language } },
                     });
                     const existing_def_table = self.state.string_tables.tables.getPtr(language).?;
                     const existing_definition = existing_def_table.get(string_id).?;
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .string_already_defined,
                         .type = .note,
                         .token = existing_definition,
 @@ -2628,7 +2619,7 @@ pub const Compiler = struct {
         pub fn init(allocator: Allocator, id_bytes: SourceBytes, type_bytes: SourceBytes, data_size: DWORD, language: res.Language, version: DWORD, characteristics: DWORD) InitError!ResourceHeader {
             const type_value = type: {
-                const resource_type = Resource.fromString(type_bytes);
+                const resource_type = ResourceType.fromString(type_bytes);
                 if (res.RT.fromResource(resource_type)) |rt_constant| {
                     break :type NameOrOrdinal{ .ordinal = @intFromEnum(rt_constant) };
                 } else {
 @@ -2673,7 +2664,7 @@ pub const Compiler = struct {
             padding_after_name: u2,
         };
-        fn calcSize(self: ResourceHeader) error{Overflow}!SizeInfo {
+        pub fn calcSize(self: ResourceHeader) error{Overflow}!SizeInfo {
             var header_size: u32 = 8;
             header_size = try std.math.add(
                 u32,
 @@ -2699,6 +2690,7 @@ pub const Compiler = struct {
             const size_info = self.calcSize() catch {
                 try err_ctx.diagnostics.append(.{
                     .err = .resource_data_size_exceeds_max,
+                    .code_page = err_ctx.code_page,
                     .token = err_ctx.token,
                 });
                 return error.CompileError;
 @@ -2706,7 +2698,7 @@ pub const Compiler = struct {
             return self.writeSizeInfo(writer, size_info);
         }
-        fn writeSizeInfo(self: ResourceHeader, writer: anytype, size_info: SizeInfo) !void {
+        pub fn writeSizeInfo(self: ResourceHeader, writer: anytype, size_info: SizeInfo) !void {
             try writer.writeInt(DWORD, self.data_size, .little); // DataSize
             try writer.writeInt(DWORD, size_info.bytes, .little); // HeaderSize
             try self.type_value.write(writer); // TYPE
 @@ -2863,19 +2855,44 @@ pub const Compiler = struct {
             self.sourceBytesForToken(token),
             .{
                 .start_column = token.calculateColumn(self.source, 8, null),
-                .diagnostics = .{ .diagnostics = self.diagnostics, .token = token },
+                .diagnostics = self.errContext(token),
+                .output_code_page = self.output_code_pages.getForToken(token),
             },
         );
     }
-    fn addErrorDetails(self: *Compiler, details: ErrorDetails) Allocator.Error!void {
+    fn addErrorDetailsWithCodePage(self: *Compiler, details: ErrorDetails) Allocator.Error!void {
         try self.diagnostics.append(details);
     }
-    fn addErrorDetailsAndFail(self: *Compiler, details: ErrorDetails) error{ CompileError, OutOfMemory } {
+    /// Code page is looked up in input_code_pages using the token
-        try self.addErrorDetails(details);
+    fn addErrorDetails(self: *Compiler, details_without_code_page: errors.ErrorDetailsWithoutCodePage) Allocator.Error!void {
+        const details = ErrorDetails{
+            .err = details_without_code_page.err,
+            .code_page = self.input_code_pages.getForToken(details_without_code_page.token),
+            .token = details_without_code_page.token,
+            .token_span_start = details_without_code_page.token_span_start,
+            .token_span_end = details_without_code_page.token_span_end,
+            .type = details_without_code_page.type,
+            .print_source_line = details_without_code_page.print_source_line,
+            .extra = details_without_code_page.extra,
+        };
+        try self.addErrorDetailsWithCodePage(details);
+    }
+    /// Code page is looked up in input_code_pages using the token
+    fn addErrorDetailsAndFail(self: *Compiler, details_without_code_page: errors.ErrorDetailsWithoutCodePage) error{ CompileError, OutOfMemory } {
+        try self.addErrorDetails(details_without_code_page);
         return error.CompileError;
     }
+    fn errContext(self: *Compiler, token: Token) errors.DiagnosticsContext {
+        return .{
+            .diagnostics = self.diagnostics,
+            .token = token,
+            .code_page = self.input_code_pages.getForToken(token),
+        };
+    }
 };
 pub const OpenSearchPathError = std.fs.Dir.OpenError;
 @@ -3247,7 +3264,8 @@ pub const StringTable = struct {
                 const bytes = SourceBytes{ .slice = slice, .code_page = code_page };
                 const utf16_string = try literals.parseQuotedStringAsWideString(compiler.allocator, bytes, .{
                     .start_column = column,
-                    .diagnostics = .{ .diagnostics = compiler.diagnostics, .token = string_token },
+                    .diagnostics = compiler.errContext(string_token),
+                    .output_code_page = compiler.output_code_pages.getForToken(string_token),
                 });
                 defer compiler.allocator.free(utf16_string);

lib/compiler/resinator/disjoint_code_page.zig created+99

...	@@ -0,0 +1,99 @@
1	const std = @import("std");
2	const lex = @import("lex.zig");
3	const SourceMappings = @import("source_mapping.zig").SourceMappings;
4	const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
5
6	pub fn hasDisjointCodePage(source: []const u8, source_mappings: ?*const SourceMappings, default_code_page: SupportedCodePage) bool {
7	var line_handler = lex.LineHandler{ .buffer = source };
8	var i: usize = 0;
9	while (i < source.len) {
10	const codepoint = default_code_page.codepointAt(i, source) orelse break;
11	const c = codepoint.value;
12	switch (c) {
13	'\r', '\n' => {
14	_ = line_handler.incrementLineNumber(i);
15	// Any lines that are not from the root file interrupt the disjoint code page
16	if (source_mappings != null and !source_mappings.?.isRootFile(line_handler.line_number)) return false;
17	},
18	// whitespace is ignored
19	' ',
20	'\t',
21	// NBSP, this should technically be in the TODO below, but it is treated as whitespace
22	// due to a (misguided) special casing in the lexer, see the TODO in lex.zig
23	'\u{A0}',
24	=> {},
25
26	// TODO: All of the below are treated as whitespace by the Win32 RC preprocessor, which also
27	// means they are trimmed from the file during preprocessing. This means that these characters
28	// should be treated like ' ', '\t' above, but since the resinator preprocessor does not treat
29	// them as whitespace or trim whitespace, files with these characters are likely going to
30	// error. So, in the future some sort of emulation of/rejection of the Win32 behavior might
31	// make handling these codepoints specially make sense, but for now it doesn't really matter
32	// so they are not handled specially for simplicity's sake.
33	//'\u{1680}',
34	//'\u{180E}',
35	//'\u{2001}',
36	//'\u{2002}',
37	//'\u{2003}',
38	//'\u{2004}',
39	//'\u{2005}',
40	//'\u{2006}',
41	//'\u{2007}',
42	//'\u{2008}',
43	//'\u{2009}',
44	//'\u{200A}',
45	//'\u{2028}',
46	//'\u{2029}',
47	//'\u{202F}',
48	//'\u{205F}',
49	//'\u{3000}',
50
51	'#' => {
52	if (source_mappings != null and !source_mappings.?.isRootFile(line_handler.line_number)) {
53	return false;
54	}
55	const start_i = i;
56	while (i < source.len and source[i] != '\r' and source[i] != '\n') : (i += 1) {}
57	const line = source[start_i..i];
58	_ = (lex.parsePragmaCodePage(line) catch \|err\| switch (err) {
59	error.NotPragma => return false,
60	error.NotCodePagePragma => continue,
61	error.CodePagePragmaUnsupportedCodePage => continue,
62	else => continue,
63	}) orelse return false; // DEFAULT interrupts disjoint code page
64
65	// If we got a code page, then it is a disjoint code page pragma
66	return true;
67	},
68	else => {
69	// Any other character interrupts the disjoint code page
70	return false;
71	},
72	}
73
74	i += codepoint.byte_len;
75	}
76	return false;
77	}
78
79	test hasDisjointCodePage {
80	try std.testing.expect(hasDisjointCodePage("#pragma code_page(65001)\n", null, .windows1252));
81	// NBSP is a special case
82	try std.testing.expect(hasDisjointCodePage("\xA0\n#pragma code_page(65001)\n", null, .windows1252));
83	try std.testing.expect(hasDisjointCodePage("\u{A0}\n#pragma code_page(1252)\n", null, .utf8));
84	// other preprocessor commands don't interrupt
85	try std.testing.expect(hasDisjointCodePage("#pragma foo\n#pragma code_page(65001)\n", null, .windows1252));
86	// invalid code page doesn't interrupt
87	try std.testing.expect(hasDisjointCodePage("#pragma code_page(1234567)\n#pragma code_page(65001)\n", null, .windows1252));
88
89	try std.testing.expect(!hasDisjointCodePage("#if 1\n#endif\n#pragma code_page(65001)", null, .windows1252));
90	try std.testing.expect(!hasDisjointCodePage("// comment\n#pragma code_page(65001)", null, .windows1252));
91	try std.testing.expect(!hasDisjointCodePage("/* comment */\n#pragma code_page(65001)", null, .windows1252));
92	}
93
94	test "multiline comment edge case" {
95	// TODO
96	if (true) return error.SkipZigTest;
97
98	try std.testing.expect(hasDisjointCodePage("/* comment */#pragma code_page(65001)", null, .windows1252));
99	}

lib/compiler/resinator/errors.zig+391-204

...	@@ -8,7 +8,8 @@ const ico = @import("ico.zig");	...	@@ -8,7 +8,8 @@ const ico = @import("ico.zig");
8	const bmp = @import("bmp.zig");	8	const bmp = @import("bmp.zig");
9	const parse = @import("parse.zig");	9	const parse = @import("parse.zig");
10	const lang = @import("lang.zig");	10	const lang = @import("lang.zig");
11	const CodePage = @import("code_pages.zig").CodePage;	11	const code_pages = @import("code_pages.zig");
		12	const SupportedCodePage = code_pages.SupportedCodePage;
12	const builtin = @import("builtin");	13	const builtin = @import("builtin");
13	const native_endian = builtin.cpu.arch.endian();	14	const native_endian = builtin.cpu.arch.endian();
14		15
...	@@ -64,7 +65,7 @@ pub const Diagnostics = struct {	...	@@ -64,7 +65,7 @@ pub const Diagnostics = struct {
64	defer std.debug.unlockStdErr();	65	defer std.debug.unlockStdErr();
65	const stderr = std.io.getStdErr().writer();	66	const stderr = std.io.getStdErr().writer();
66	for (self.errors.items) \|err_details\| {	67	for (self.errors.items) \|err_details\| {
67	renderErrorMessage(self.allocator, stderr, tty_config, cwd, err_details, source, self.strings.items, source_mappings) catch return;	68	renderErrorMessage(stderr, tty_config, cwd, err_details, source, self.strings.items, source_mappings) catch return;
68	}	69	}
69	}	70	}
70		71
...	@@ -94,32 +95,22 @@ pub const Diagnostics = struct {	...	@@ -94,32 +95,22 @@ pub const Diagnostics = struct {
94	pub const DiagnosticsContext = struct {	95	pub const DiagnosticsContext = struct {
95	diagnostics: *Diagnostics,	96	diagnostics: *Diagnostics,
96	token: Token,	97	token: Token,
		98	/// Code page of the source file at the token location
		99	code_page: SupportedCodePage,
97	};	100	};
98		101
99	pub const ErrorDetails = struct {	102	pub const ErrorDetails = struct {
100	err: Error,	103	err: Error,
101	token: Token,	104	token: Token,
		105	/// Code page of the source file at the token location
		106	code_page: SupportedCodePage,
102	/// If non-null, should be before `token`. If null, `token` is assumed to be the start.	107	/// If non-null, should be before `token`. If null, `token` is assumed to be the start.
103	token_span_start: ?Token = null,	108	token_span_start: ?Token = null,
104	/// If non-null, should be after `token`. If null, `token` is assumed to be the end.	109	/// If non-null, should be after `token`. If null, `token` is assumed to be the end.
105	token_span_end: ?Token = null,	110	token_span_end: ?Token = null,
106	type: Type = .err,	111	type: Type = .err,
107	print_source_line: bool = true,	112	print_source_line: bool = true,
108	extra: union {	113	extra: Extra = .{ .none = {} },
109	none: void,
110	expected: Token.Id,
111	number: u32,
112	expected_types: ExpectedTypes,
113	resource: rc.Resource,
114	string_and_language: StringAndLanguage,
115	file_open_error: FileOpenError,
116	icon_read_error: IconReadError,
117	icon_dir: IconDirContext,
118	bmp_read_error: BitmapReadError,
119	accelerator_error: AcceleratorError,
120	statement_with_u16_param: StatementWithU16Param,
121	menu_or_class: enum { class, menu },
122	} = .{ .none = {} },
123		114
124	pub const Type = enum {	115	pub const Type = enum {
125	/// Fatal error, stops compilation	116	/// Fatal error, stops compilation
...	@@ -137,9 +128,25 @@ pub const ErrorDetails = struct {	...	@@ -137,9 +128,25 @@ pub const ErrorDetails = struct {
137	hint,	128	hint,
138	};	129	};
139		130
		131	pub const Extra = union {
		132	none: void,
		133	expected: Token.Id,
		134	number: u32,
		135	expected_types: ExpectedTypes,
		136	resource: rc.ResourceType,
		137	string_and_language: StringAndLanguage,
		138	file_open_error: FileOpenError,
		139	icon_read_error: IconReadError,
		140	icon_dir: IconDirContext,
		141	bmp_read_error: BitmapReadError,
		142	accelerator_error: AcceleratorError,
		143	statement_with_u16_param: StatementWithU16Param,
		144	menu_or_class: enum { class, menu },
		145	};
		146
140	comptime {	147	comptime {
141	// all fields in the extra union should be 32 bits or less	148	// all fields in the extra union should be 32 bits or less
142	for (std.meta.fields(std.meta.fieldInfo(ErrorDetails, .extra).type)) \|field\| {	149	for (std.meta.fields(Extra)) \|field\| {
143	std.debug.assert(@bitSizeOf(field.type) <= 32);	150	std.debug.assert(@bitSizeOf(field.type) <= 32);
144	}	151	}
145	}	152	}
...	@@ -321,6 +328,8 @@ pub const ErrorDetails = struct {	...	@@ -321,6 +328,8 @@ pub const ErrorDetails = struct {
321	close_paren_expression,	328	close_paren_expression,
322	unary_plus_expression,	329	unary_plus_expression,
323	rc_could_miscompile_control_params,	330	rc_could_miscompile_control_params,
		331	dangling_literal_at_eof,
		332	disjoint_code_page,
324		333
325	// Compiler	334	// Compiler
326	/// `string_and_language` is populated	335	/// `string_and_language` is populated
...	@@ -331,6 +340,7 @@ pub const ErrorDetails = struct {	...	@@ -331,6 +340,7 @@ pub const ErrorDetails = struct {
331	/// `accelerator_error` is populated	340	/// `accelerator_error` is populated
332	invalid_accelerator_key,	341	invalid_accelerator_key,
333	accelerator_type_required,	342	accelerator_type_required,
		343	accelerator_shift_or_control_without_virtkey,
334	rc_would_miscompile_control_padding,	344	rc_would_miscompile_control_padding,
335	rc_would_miscompile_control_class_ordinal,	345	rc_would_miscompile_control_class_ordinal,
336	/// `icon_dir` is populated	346	/// `icon_dir` is populated
...	@@ -356,11 +366,6 @@ pub const ErrorDetails = struct {	...	@@ -356,11 +366,6 @@ pub const ErrorDetails = struct {
356	/// `number` is populated and contains a string index for which the string contains	366	/// `number` is populated and contains a string index for which the string contains
357	/// the bytes of a `u64` (native endian). The `u64` contains the number of miscompiled bytes.	367	/// the bytes of a `u64` (native endian). The `u64` contains the number of miscompiled bytes.
358	rc_would_miscompile_bmp_palette_padding,	368	rc_would_miscompile_bmp_palette_padding,
359	/// `number` is populated and contains a string index for which the string contains
360	/// the bytes of two `u64`s (native endian). The first contains the number of missing
361	/// palette bytes and the second contains the max number of missing palette bytes.
362	/// If type is `.note`, then `extra` is `none`.
363	bmp_too_many_missing_palette_bytes,
364	resource_header_size_exceeds_max,	369	resource_header_size_exceeds_max,
365	resource_data_size_exceeds_max,	370	resource_data_size_exceeds_max,
366	control_extra_data_size_exceeds_max,	371	control_extra_data_size_exceeds_max,
...	@@ -383,15 +388,16 @@ pub const ErrorDetails = struct {	...	@@ -383,15 +388,16 @@ pub const ErrorDetails = struct {
383	rc_would_miscompile_dialog_menu_or_class_id_forced_ordinal,	388	rc_would_miscompile_dialog_menu_or_class_id_forced_ordinal,
384	rc_would_miscompile_dialog_menu_id_starts_with_digit,	389	rc_would_miscompile_dialog_menu_id_starts_with_digit,
385	dialog_menu_id_was_uppercased,	390	dialog_menu_id_was_uppercased,
386	/// `menu_or_class` is populated and contains the type of the parameter statement	391	duplicate_optional_statement_skipped,
387	duplicate_menu_or_class_skipped,
388	invalid_digit_character_in_ordinal,	392	invalid_digit_character_in_ordinal,
389		393
390	// Literals	394	// Literals
391	/// `number` is populated	395	/// `number` is populated
392	rc_would_miscompile_codepoint_byte_swap,	396	rc_would_miscompile_codepoint_whitespace,
393	/// `number` is populated	397	/// `number` is populated
394	rc_would_miscompile_codepoint_skip,	398	rc_would_miscompile_codepoint_skip,
		399	/// `number` is populated
		400	rc_would_miscompile_codepoint_bom,
395	tab_converted_to_spaces,	401	tab_converted_to_spaces,
396		402
397	// General (used in various places)	403	// General (used in various places)
...	@@ -403,10 +409,50 @@ pub const ErrorDetails = struct {	...	@@ -403,10 +409,50 @@ pub const ErrorDetails = struct {
403	failed_to_open_cwd,	409	failed_to_open_cwd,
404	};	410	};
405		411
		412	fn formatToken(
		413	ctx: TokenFormatContext,
		414	comptime fmt: []const u8,
		415	options: std.fmt.FormatOptions,
		416	writer: anytype,
		417	) !void {
		418	_ = fmt;
		419	_ = options;
		420
		421	switch (ctx.token.id) {
		422	.eof => return writer.writeAll(ctx.token.id.nameForErrorDisplay()),
		423	else => {},
		424	}
		425
		426	const slice = ctx.token.slice(ctx.source);
		427	var src_i: usize = 0;
		428	while (src_i < slice.len) {
		429	const codepoint = ctx.code_page.codepointAt(src_i, slice) orelse break;
		430	defer src_i += codepoint.byte_len;
		431	const display_codepoint = codepointForDisplay(codepoint) orelse continue;
		432	var buf: [4]u8 = undefined;
		433	const utf8_len = std.unicode.utf8Encode(display_codepoint, &buf) catch unreachable;
		434	try writer.writeAll(buf[0..utf8_len]);
		435	}
		436	}
		437
		438	const TokenFormatContext = struct {
		439	token: Token,
		440	source: []const u8,
		441	code_page: SupportedCodePage,
		442	};
		443
		444	fn fmtToken(self: ErrorDetails, source: []const u8) std.fmt.Formatter(formatToken) {
		445	return .{ .data = .{
		446	.token = self.token,
		447	.code_page = self.code_page,
		448	.source = source,
		449	} };
		450	}
		451
406	pub fn render(self: ErrorDetails, writer: anytype, source: []const u8, strings: []const []const u8) !void {	452	pub fn render(self: ErrorDetails, writer: anytype, source: []const u8, strings: []const []const u8) !void {
407	switch (self.err) {	453	switch (self.err) {
408	.unfinished_string_literal => {	454	.unfinished_string_literal => {
409	return writer.print("unfinished string literal at '{s}', expected closing '\"'", .{self.token.nameForErrorDisplay(source)});	455	return writer.print("unfinished string literal at '{s}', expected closing '\"'", .{self.fmtToken(source)});
410	},	456	},
411	.string_literal_too_long => {	457	.string_literal_too_long => {
412	return writer.print("string literal too long (max is currently {} characters)", .{self.extra.number});	458	return writer.print("string literal too long (max is currently {} characters)", .{self.extra.number});
...	@@ -474,33 +520,33 @@ pub const ErrorDetails = struct {	...	@@ -474,33 +520,33 @@ pub const ErrorDetails = struct {
474	number_slice.len += 1;	520	number_slice.len += 1;
475	}	521	}
476	const number = std.fmt.parseUnsigned(u16, number_slice, 10) catch unreachable;	522	const number = std.fmt.parseUnsigned(u16, number_slice, 10) catch unreachable;
477	const code_page = CodePage.getByIdentifier(number) catch unreachable;	523	const code_page = code_pages.getByIdentifier(number) catch unreachable;
478	// TODO: Improve or maybe add a note making it more clear that the code page	524	// TODO: Improve or maybe add a note making it more clear that the code page
479	// is valid and that the code page is unsupported purely due to a limitation	525	// is valid and that the code page is unsupported purely due to a limitation
480	// in this compiler.	526	// in this compiler.
481	return writer.print("unsupported code page '{s} (id={})' in #pragma code_page", .{ @tagName(code_page), number });	527	return writer.print("unsupported code page '{s} (id={})' in #pragma code_page", .{ @tagName(code_page), number });
482	},	528	},
483	.unfinished_raw_data_block => {	529	.unfinished_raw_data_block => {
484	return writer.print("unfinished raw data block at '{s}', expected closing '}}' or 'END'", .{self.token.nameForErrorDisplay(source)});	530	return writer.print("unfinished raw data block at '{s}', expected closing '}}' or 'END'", .{self.fmtToken(source)});
485	},	531	},
486	.unfinished_string_table_block => {	532	.unfinished_string_table_block => {
487	return writer.print("unfinished STRINGTABLE block at '{s}', expected closing '}}' or 'END'", .{self.token.nameForErrorDisplay(source)});	533	return writer.print("unfinished STRINGTABLE block at '{s}', expected closing '}}' or 'END'", .{self.fmtToken(source)});
488	},	534	},
489	.expected_token => {	535	.expected_token => {
490	return writer.print("expected '{s}', got '{s}'", .{ self.extra.expected.nameForErrorDisplay(), self.token.nameForErrorDisplay(source) });	536	return writer.print("expected '{s}', got '{s}'", .{ self.extra.expected.nameForErrorDisplay(), self.fmtToken(source) });
491	},	537	},
492	.expected_something_else => {	538	.expected_something_else => {
493	try writer.writeAll("expected ");	539	try writer.writeAll("expected ");
494	try self.extra.expected_types.writeCommaSeparated(writer);	540	try self.extra.expected_types.writeCommaSeparated(writer);
495	return writer.print("; got '{s}'", .{self.token.nameForErrorDisplay(source)});	541	return writer.print("; got '{s}'", .{self.fmtToken(source)});
496	},	542	},
497	.resource_type_cant_use_raw_data => switch (self.type) {	543	.resource_type_cant_use_raw_data => switch (self.type) {
498	.err, .warning => try writer.print("expected '<filename>', found '{s}' (resource type '{s}' can't use raw data)", .{ self.token.nameForErrorDisplay(source), self.extra.resource.nameForErrorDisplay() }),	544	.err, .warning => try writer.print("expected '<filename>', found '{s}' (resource type '{s}' can't use raw data)", .{ self.fmtToken(source), self.extra.resource.nameForErrorDisplay() }),
499	.note => try writer.print("if '{s}' is intended to be a filename, it must be specified as a quoted string literal", .{self.token.nameForErrorDisplay(source)}),	545	.note => try writer.print("if '{s}' is intended to be a filename, it must be specified as a quoted string literal", .{self.fmtToken(source)}),
500	.hint => return,	546	.hint => return,
501	},	547	},
502	.id_must_be_ordinal => {	548	.id_must_be_ordinal => {
503	try writer.print("id of resource type '{s}' must be an ordinal (u16), got '{s}'", .{ self.extra.resource.nameForErrorDisplay(), self.token.nameForErrorDisplay(source) });	549	try writer.print("id of resource type '{s}' must be an ordinal (u16), got '{s}'", .{ self.extra.resource.nameForErrorDisplay(), self.fmtToken(source) });
504	},	550	},
505	.name_or_id_not_allowed => {	551	.name_or_id_not_allowed => {
506	try writer.print("name or id is not allowed for resource type '{s}'", .{self.extra.resource.nameForErrorDisplay()});	552	try writer.print("name or id is not allowed for resource type '{s}'", .{self.extra.resource.nameForErrorDisplay()});
...	@@ -516,7 +562,7 @@ pub const ErrorDetails = struct {	...	@@ -516,7 +562,7 @@ pub const ErrorDetails = struct {
516	try writer.writeAll("ASCII character not equivalent to virtual key code");	562	try writer.writeAll("ASCII character not equivalent to virtual key code");
517	},	563	},
518	.empty_menu_not_allowed => {	564	.empty_menu_not_allowed => {
519	try writer.print("empty menu of type '{s}' not allowed", .{self.token.nameForErrorDisplay(source)});	565	try writer.print("empty menu of type '{s}' not allowed", .{self.fmtToken(source)});
520	},	566	},
521	.rc_would_miscompile_version_value_padding => switch (self.type) {	567	.rc_would_miscompile_version_value_padding => switch (self.type) {
522	.err, .warning => return writer.print("the padding before this quoted string value would be miscompiled by the Win32 RC compiler", .{}),	568	.err, .warning => return writer.print("the padding before this quoted string value would be miscompiled by the Win32 RC compiler", .{}),
...	@@ -570,19 +616,18 @@ pub const ErrorDetails = struct {	...	@@ -570,19 +616,18 @@ pub const ErrorDetails = struct {
570	.note => return writer.print("to avoid the potential miscompilation, consider adding a comma after the style parameter", .{}),	616	.note => return writer.print("to avoid the potential miscompilation, consider adding a comma after the style parameter", .{}),
571	.hint => return,	617	.hint => return,
572	},	618	},
		619	.dangling_literal_at_eof => {
		620	try writer.writeAll("dangling literal at end-of-file; this is not a problem, but it is likely a mistake");
		621	},
		622	.disjoint_code_page => switch (self.type) {
		623	.err, .warning => return writer.print("#pragma code_page as the first thing in the .rc script can cause the input and output code pages to become out-of-sync", .{}),
		624	.note => return writer.print("to avoid unexpected behavior, add a comment (or anything else) above the #pragma code_page line", .{}),
		625	.hint => return,
		626	},
573	.string_already_defined => switch (self.type) {	627	.string_already_defined => switch (self.type) {
574	.err, .warning => {	628	.err, .warning => {
575	const language_id = self.extra.string_and_language.language.asInt();	629	const language = self.extra.string_and_language.language;
576	const language_name = language_name: {	630	return writer.print("string with id {d} (0x{X}) already defined for language {}", .{ self.extra.string_and_language.id, self.extra.string_and_language.id, language });
577	if (std.meta.intToEnum(lang.LanguageId, language_id)) \|lang_enum_val\| {
578	break :language_name @tagName(lang_enum_val);
579	} else \|_\| {}
580	if (language_id == lang.LOCALE_CUSTOM_UNSPECIFIED) {
581	break :language_name "LOCALE_CUSTOM_UNSPECIFIED";
582	}
583	break :language_name "<UNKNOWN>";
584	};
585	return writer.print("string with id {d} (0x{X}) already defined for language {s} (0x{X})", .{ self.extra.string_and_language.id, self.extra.string_and_language.id, language_name, language_id });
586	},	631	},
587	.note => return writer.print("previous definition of string with id {d} (0x{X}) here", .{ self.extra.string_and_language.id, self.extra.string_and_language.id }),	632	.note => return writer.print("previous definition of string with id {d} (0x{X}) here", .{ self.extra.string_and_language.id, self.extra.string_and_language.id }),
588	.hint => return,	633	.hint => return,
...	@@ -597,14 +642,17 @@ pub const ErrorDetails = struct {	...	@@ -597,14 +642,17 @@ pub const ErrorDetails = struct {
597	try writer.print("unable to open file '{s}': {s}", .{ strings[self.extra.file_open_error.filename_string_index], @tagName(self.extra.file_open_error.err) });	642	try writer.print("unable to open file '{s}': {s}", .{ strings[self.extra.file_open_error.filename_string_index], @tagName(self.extra.file_open_error.err) });
598	},	643	},
599	.invalid_accelerator_key => {	644	.invalid_accelerator_key => {
600	try writer.print("invalid accelerator key '{s}': {s}", .{ self.token.nameForErrorDisplay(source), @tagName(self.extra.accelerator_error.err) });	645	try writer.print("invalid accelerator key '{s}': {s}", .{ self.fmtToken(source), @tagName(self.extra.accelerator_error.err) });
601	},	646	},
602	.accelerator_type_required => {	647	.accelerator_type_required => {
603	try writer.print("accelerator type [ASCII or VIRTKEY] required when key is an integer", .{});	648	try writer.writeAll("accelerator type [ASCII or VIRTKEY] required when key is an integer");
		649	},
		650	.accelerator_shift_or_control_without_virtkey => {
		651	try writer.writeAll("SHIFT or CONTROL used without VIRTKEY");
604	},	652	},
605	.rc_would_miscompile_control_padding => switch (self.type) {	653	.rc_would_miscompile_control_padding => switch (self.type) {
606	.err, .warning => return writer.print("the padding before this control would be miscompiled by the Win32 RC compiler (it would insert 2 extra bytes of padding)", .{}),	654	.err, .warning => return writer.print("the padding before this control would be miscompiled by the Win32 RC compiler (it would insert 2 extra bytes of padding)", .{}),
607	.note => return writer.print("to avoid the potential miscompilation, consider removing any 'control data' blocks from the controls in this dialog", .{}),	655	.note => return writer.print("to avoid the potential miscompilation, consider adding one more byte to the control data of the control preceding this one", .{}),
608	.hint => return,	656	.hint => return,
609	},	657	},
610	.rc_would_miscompile_control_class_ordinal => switch (self.type) {	658	.rc_would_miscompile_control_class_ordinal => switch (self.type) {
...	@@ -625,7 +673,7 @@ pub const ErrorDetails = struct {	...	@@ -625,7 +673,7 @@ pub const ErrorDetails = struct {
625	try writer.print("resource with format '{s}' (at index {}) is not allowed in {s} resource groups", .{ @tagName(self.extra.icon_dir.icon_format), self.extra.icon_dir.index, @tagName(self.extra.icon_dir.icon_type) });	673	try writer.print("resource with format '{s}' (at index {}) is not allowed in {s} resource groups", .{ @tagName(self.extra.icon_dir.icon_format), self.extra.icon_dir.index, @tagName(self.extra.icon_dir.icon_type) });
626	},	674	},
627	.icon_dir_and_resource_type_mismatch => {	675	.icon_dir_and_resource_type_mismatch => {
628	const unexpected_type: rc.Resource = if (self.extra.resource == .icon) .cursor else .icon;	676	const unexpected_type: rc.ResourceType = if (self.extra.resource == .icon) .cursor else .icon;
629	// TODO: Better wording	677	// TODO: Better wording
630	try writer.print("resource type '{s}' does not match type '{s}' specified in the file", .{ self.extra.resource.nameForErrorDisplay(), unexpected_type.nameForErrorDisplay() });	678	try writer.print("resource type '{s}' does not match type '{s}' specified in the file", .{ self.extra.resource.nameForErrorDisplay(), unexpected_type.nameForErrorDisplay() });
631	},	679	},
...	@@ -663,23 +711,15 @@ pub const ErrorDetails = struct {	...	@@ -663,23 +711,15 @@ pub const ErrorDetails = struct {
663	.bmp_missing_palette_bytes => {	711	.bmp_missing_palette_bytes => {
664	const bytes = strings[self.extra.number];	712	const bytes = strings[self.extra.number];
665	const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);	713	const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
666	try writer.print("bitmap has {d} missing color palette bytes which will be padded with zeroes", .{missing_bytes});	714	try writer.print("bitmap has {d} missing color palette bytes", .{missing_bytes});
667	},	715	},
668	.rc_would_miscompile_bmp_palette_padding => {	716	.rc_would_miscompile_bmp_palette_padding => {
669	const bytes = strings[self.extra.number];	717	try writer.writeAll("the Win32 RC compiler would erroneously pad out the missing bytes");
670	const miscompiled_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);	718	if (self.extra.number != 0) {
671	try writer.print("the missing color palette bytes would be miscompiled by the Win32 RC compiler (the added padding bytes would include {d} bytes of the pixel data)", .{miscompiled_bytes});
672	},
673	.bmp_too_many_missing_palette_bytes => switch (self.type) {
674	.err, .warning => {
675	const bytes = strings[self.extra.number];	719	const bytes = strings[self.extra.number];
676	const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);	720	const miscompiled_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
677	const max_missing_bytes = std.mem.readInt(u64, bytes[8..16], native_endian);	721	try writer.print(" (and the added padding bytes would include {d} bytes of the pixel data)", .{miscompiled_bytes});
678	try writer.print("bitmap has {} missing color palette bytes which exceeds the maximum of {}", .{ missing_bytes, max_missing_bytes });	722	}
679	},
680	// TODO: command line option
681	.note => try writer.writeAll("the maximum number of missing color palette bytes is configurable via <<TODO command line option>>"),
682	.hint => return,
683	},	723	},
684	.resource_header_size_exceeds_max => {	724	.resource_header_size_exceeds_max => {
685	try writer.print("resource's header length exceeds maximum of {} bytes", .{std.math.maxInt(u32)});	725	try writer.print("resource's header length exceeds maximum of {} bytes", .{std.math.maxInt(u32)});
...	@@ -749,23 +789,22 @@ pub const ErrorDetails = struct {	...	@@ -749,23 +789,22 @@ pub const ErrorDetails = struct {
749	.hint => return,	789	.hint => return,
750	},	790	},
751	.dialog_menu_id_was_uppercased => return,	791	.dialog_menu_id_was_uppercased => return,
752	.duplicate_menu_or_class_skipped => {	792	.duplicate_optional_statement_skipped => {
753	return writer.print("this {s} was ignored; when multiple {s} statements are specified, only the last takes precedence", .{	793	return writer.writeAll("this statement was ignored; when multiple statements of the same type are specified, only the last takes precedence");
754	@tagName(self.extra.menu_or_class),
755	@tagName(self.extra.menu_or_class),
756	});
757	},	794	},
758	.invalid_digit_character_in_ordinal => {	795	.invalid_digit_character_in_ordinal => {
759	return writer.writeAll("non-ASCII digit characters are not allowed in ordinal (number) values");	796	return writer.writeAll("non-ASCII digit characters are not allowed in ordinal (number) values");
760	},	797	},
761	.rc_would_miscompile_codepoint_byte_swap => switch (self.type) {	798	.rc_would_miscompile_codepoint_whitespace => {
762	.err, .warning => return writer.print("codepoint U+{X} within a string literal would be miscompiled by the Win32 RC compiler (the bytes of the UTF-16 code unit would be swapped)", .{self.extra.number}),	799	const treated_as = self.extra.number >> 8;
763	.note => return writer.print("to avoid the potential miscompilation, an integer escape sequence in a wide string literal could be used instead: L\"\\x{X}\"", .{self.extra.number}),	800	return writer.print("codepoint U+{X:0>4} within a string literal would be miscompiled by the Win32 RC compiler (it would get treated as U+{X:0>4})", .{ self.extra.number, treated_as });
764	.hint => return,
765	},	801	},
766	.rc_would_miscompile_codepoint_skip => switch (self.type) {	802	.rc_would_miscompile_codepoint_skip => {
767	.err, .warning => return writer.print("codepoint U+{X} within a string literal would be miscompiled by the Win32 RC compiler (the codepoint would be missing from the compiled resource)", .{self.extra.number}),	803	return writer.print("codepoint U+{X:0>4} within a string literal would be miscompiled by the Win32 RC compiler (the codepoint would be missing from the compiled resource)", .{self.extra.number});
768	.note => return writer.print("to avoid the potential miscompilation, an integer escape sequence in a wide string literal could be used instead: L\"\\x{X}\"", .{self.extra.number}),	804	},
		805	.rc_would_miscompile_codepoint_bom => switch (self.type) {
		806	.err, .warning => return writer.print("codepoint U+{X:0>4} within a string literal would cause the entire file to be miscompiled by the Win32 RC compiler", .{self.extra.number}),
		807	.note => return writer.writeAll("the presence of this codepoint causes all non-ASCII codepoints to be byteswapped by the Win32 RC preprocessor"),
769	.hint => return,	808	.hint => return,
770	},	809	},
771	.tab_converted_to_spaces => switch (self.type) {	810	.tab_converted_to_spaces => switch (self.type) {
...	@@ -790,14 +829,7 @@ pub const ErrorDetails = struct {	...	@@ -790,14 +829,7 @@ pub const ErrorDetails = struct {
790	after_len: usize,	829	after_len: usize,
791	};	830	};
792		831
793	pub fn visualTokenInfo(self: ErrorDetails, source_line_start: usize, source_line_end: usize) VisualTokenInfo {	832	pub fn visualTokenInfo(self: ErrorDetails, source_line_start: usize, source_line_end: usize, source: []const u8) VisualTokenInfo {
794	// Note: A perfect solution here would involve full grapheme cluster
795	// awareness, but oh well. This will give incorrect offsets
796	// if there are any multibyte codepoints within the relevant span,
797	// and even more inflated for grapheme clusters.
798	//
799	// We mitigate this slightly when we know we'll be pointing at
800	// something that displays as 1 character.
801	return switch (self.err) {	833	return switch (self.err) {
802	// These can technically be more than 1 byte depending on encoding,	834	// These can technically be more than 1 byte depending on encoding,
803	// but they always refer to one visual character/grapheme.	835	// but they always refer to one visual character/grapheme.
...	@@ -808,27 +840,65 @@ pub const ErrorDetails = struct {	...	@@ -808,27 +840,65 @@ pub const ErrorDetails = struct {
808	.illegal_private_use_character,	840	.illegal_private_use_character,
809	=> .{	841	=> .{
810	.before_len = 0,	842	.before_len = 0,
811	.point_offset = self.token.start - source_line_start,	843	.point_offset = cellCount(self.code_page, source, source_line_start, self.token.start),
812	.after_len = 0,	844	.after_len = 0,
813	},	845	},
814	else => .{	846	else => .{
815	.before_len = before: {	847	.before_len = before: {
816	const start = @max(source_line_start, if (self.token_span_start) \|span_start\| span_start.start else self.token.start);	848	const start = @max(source_line_start, if (self.token_span_start) \|span_start\| span_start.start else self.token.start);
817	break :before self.token.start - start;	849	break :before cellCount(self.code_page, source, start, self.token.start);
818	},	850	},
819	.point_offset = self.token.start - source_line_start,	851	.point_offset = cellCount(self.code_page, source, source_line_start, self.token.start),
820	.after_len = after: {	852	.after_len = after: {
821	const end = @min(source_line_end, if (self.token_span_end) \|span_end\| span_end.end else self.token.end);	853	const end = @min(source_line_end, if (self.token_span_end) \|span_end\| span_end.end else self.token.end);
822	// end may be less than start when pointing to EOF	854	// end may be less than start when pointing to EOF
823	if (end <= self.token.start) break :after 0;	855	if (end <= self.token.start) break :after 0;
824	break :after end - self.token.start - 1;	856	break :after cellCount(self.code_page, source, self.token.start, end) - 1;
825	},	857	},
826	},	858	},
827	};	859	};
828	}	860	}
829	};	861	};
830		862
831	pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_config: std.io.tty.Config, cwd: std.fs.Dir, err_details: ErrorDetails, source: []const u8, strings: []const []const u8, source_mappings: ?SourceMappings) !void {	863	/// Convenience struct only useful when the code page can be inferred from the token
		864	pub const ErrorDetailsWithoutCodePage = blk: {
		865	const details_info = @typeInfo(ErrorDetails);
		866	const fields = details_info.@"struct".fields;
		867	var fields_without_codepage: [fields.len - 1]std.builtin.Type.StructField = undefined;
		868	var i: usize = 0;
		869	for (fields) \|field\| {
		870	if (std.mem.eql(u8, field.name, "code_page")) continue;
		871	fields_without_codepage[i] = field;
		872	i += 1;
		873	}
		874	std.debug.assert(i == fields_without_codepage.len);
		875	break :blk @Type(.{ .@"struct" = .{
		876	.layout = .auto,
		877	.fields = &fields_without_codepage,
		878	.decls = &.{},
		879	.is_tuple = false,
		880	} });
		881	};
		882
		883	fn cellCount(code_page: SupportedCodePage, source: []const u8, start_index: usize, end_index: usize) usize {
		884	// Note: This is an imperfect solution. A proper implementation here would
		885	// involve full grapheme cluster awareness + grapheme width data, but oh well.
		886	var codepoint_count: usize = 0;
		887	var index: usize = start_index;
		888	while (index < end_index) {
		889	const codepoint = code_page.codepointAt(index, source) orelse break;
		890	defer index += codepoint.byte_len;
		891	_ = codepointForDisplay(codepoint) orelse continue;
		892	codepoint_count += 1;
		893	// no need to count more than we will display
		894	if (codepoint_count >= max_source_line_codepoints + truncated_str.len) break;
		895	}
		896	return codepoint_count;
		897	}
		898
		899	const truncated_str = "<...truncated...>";
		900
		901	pub fn renderErrorMessage(writer: anytype, tty_config: std.io.tty.Config, cwd: std.fs.Dir, err_details: ErrorDetails, source: []const u8, strings: []const []const u8, source_mappings: ?SourceMappings) !void {
832	if (err_details.type == .hint) return;	902	if (err_details.type == .hint) return;
833		903
834	const source_line_start = err_details.token.getLineStartForErrorDisplay(source);	904	const source_line_start = err_details.token.getLineStartForErrorDisplay(source);
...	@@ -884,45 +954,61 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con	...	@@ -884,45 +954,61 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con
884	}	954	}
885		955
886	const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);	956	const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);
887	const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len);	957	const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len, source);
		958	const truncated_visual_info = ErrorDetails.VisualTokenInfo{
		959	.before_len = if (visual_info.point_offset > max_source_line_codepoints and visual_info.before_len > 0)
		960	(visual_info.before_len + 1) -\| (visual_info.point_offset - max_source_line_codepoints)
		961	else
		962	visual_info.before_len,
		963	.point_offset = @min(max_source_line_codepoints + 1, visual_info.point_offset),
		964	.after_len = if (visual_info.point_offset > max_source_line_codepoints)
		965	@min(truncated_str.len - 3, visual_info.after_len)
		966	else
		967	@min(max_source_line_codepoints - visual_info.point_offset + (truncated_str.len - 2), visual_info.after_len),
		968	};
888		969
889	// Need this to determine if the 'line originated from' note is worth printing	970	// Need this to determine if the 'line originated from' note is worth printing
890	var source_line_for_display_buf = try std.ArrayList(u8).initCapacity(allocator, source_line.len);	971	var source_line_for_display_buf: [max_source_line_bytes]u8 = undefined;
891	defer source_line_for_display_buf.deinit();	972	const source_line_for_display = writeSourceSlice(&source_line_for_display_buf, source_line, err_details.code_page);
892	try writeSourceSlice(source_line_for_display_buf.writer(), source_line);	973
893		974	try writer.writeAll(source_line_for_display.line);
894	// TODO: General handling of long lines, not tied to this specific error	975	if (source_line_for_display.truncated) {
895	if (err_details.err == .string_literal_too_long) {
896	const before_slice = source_line[0..@min(source_line.len, visual_info.point_offset + 16)];
897	try writeSourceSlice(writer, before_slice);
898	try tty_config.setColor(writer, .dim);	976	try tty_config.setColor(writer, .dim);
899	try writer.writeAll("<...truncated...>");	977	try writer.writeAll(truncated_str);
900	try tty_config.setColor(writer, .reset);	978	try tty_config.setColor(writer, .reset);
901	} else {
902	try writer.writeAll(source_line_for_display_buf.items);
903	}	979	}
904	try writer.writeByte('\n');	980	try writer.writeByte('\n');
905		981
906	try tty_config.setColor(writer, .green);	982	try tty_config.setColor(writer, .green);
907	const num_spaces = visual_info.point_offset - visual_info.before_len;	983	const num_spaces = truncated_visual_info.point_offset - truncated_visual_info.before_len;
908	try writer.writeByteNTimes(' ', num_spaces);	984	try writer.writeByteNTimes(' ', num_spaces);
909	try writer.writeByteNTimes('~', visual_info.before_len);	985	try writer.writeByteNTimes('~', truncated_visual_info.before_len);
910	try writer.writeByte('^');	986	try writer.writeByte('^');
911	if (visual_info.after_len > 0) {	987	try writer.writeByteNTimes('~', truncated_visual_info.after_len);
912	var num_squiggles = visual_info.after_len;
913	if (err_details.err == .string_literal_too_long) {
914	num_squiggles = @min(num_squiggles, 15);
915	}
916	try writer.writeByteNTimes('~', num_squiggles);
917	}
918	try writer.writeByte('\n');	988	try writer.writeByte('\n');
919	try tty_config.setColor(writer, .reset);	989	try tty_config.setColor(writer, .reset);
920		990
921	if (corresponding_span != null and corresponding_file != null) {	991	if (corresponding_span != null and corresponding_file != null) {
922	var corresponding_lines = try CorrespondingLines.init(allocator, cwd, err_details, source_line_for_display_buf.items, corresponding_span.?, corresponding_file.?);	992	var worth_printing_lines: bool = true;
923	defer corresponding_lines.deinit(allocator);	993	var initial_lines_err: ?anyerror = null;
924		994	var corresponding_lines: ?CorrespondingLines = CorrespondingLines.init(
925	if (!corresponding_lines.worth_printing_note) return;	995	cwd,
		996	err_details,
		997	source_line_for_display.line,
		998	corresponding_span.?,
		999	corresponding_file.?,
		1000	) catch \|err\| switch (err) {
		1001	error.NotWorthPrintingLines => blk: {
		1002	worth_printing_lines = false;
		1003	break :blk null;
		1004	},
		1005	error.NotWorthPrintingNote => return,
		1006	else => \|e\| blk: {
		1007	initial_lines_err = e;
		1008	break :blk null;
		1009	},
		1010	};
		1011	defer if (corresponding_lines) \|*cl\| cl.deinit();
926		1012
927	try tty_config.setColor(writer, .bold);	1013	try tty_config.setColor(writer, .bold);
928	if (corresponding_file) \|file\| {	1014	if (corresponding_file) \|file\| {
...	@@ -947,85 +1033,222 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con	...	@@ -947,85 +1033,222 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con
947	try writer.print(" of file '{s}'\n", .{corresponding_file.?});	1033	try writer.print(" of file '{s}'\n", .{corresponding_file.?});
948	try tty_config.setColor(writer, .reset);	1034	try tty_config.setColor(writer, .reset);
949		1035
950	if (!corresponding_lines.worth_printing_lines) return;	1036	if (!worth_printing_lines) return;
951		1037
952	if (corresponding_lines.lines_is_error_message) {	1038	const write_lines_err: ?anyerror = write_lines: {
		1039	if (initial_lines_err) \|err\| break :write_lines err;
		1040	while (corresponding_lines.?.next() catch \|err\| {
		1041	break :write_lines err;
		1042	}) \|display_line\| {
		1043	try writer.writeAll(display_line.line);
		1044	if (display_line.truncated) {
		1045	try tty_config.setColor(writer, .dim);
		1046	try writer.writeAll(truncated_str);
		1047	try tty_config.setColor(writer, .reset);
		1048	}
		1049	try writer.writeByte('\n');
		1050	}
		1051	break :write_lines null;
		1052	};
		1053	if (write_lines_err) \|err\| {
953	try tty_config.setColor(writer, .red);	1054	try tty_config.setColor(writer, .red);
954	try writer.writeAll(" \| ");	1055	try writer.writeAll(" \| ");
955	try tty_config.setColor(writer, .reset);	1056	try tty_config.setColor(writer, .reset);
956	try tty_config.setColor(writer, .dim);	1057	try tty_config.setColor(writer, .dim);
957	try writer.writeAll(corresponding_lines.lines.items);	1058	try writer.print("unable to print line(s) from file: {s}\n", .{@errorName(err)});
958	try tty_config.setColor(writer, .reset);	1059	try tty_config.setColor(writer, .reset);
959	try writer.writeAll("\n\n");
960	return;
961	}	1060	}
962		1061	try writer.writeByte('\n');
963	try writer.writeAll(corresponding_lines.lines.items);
964	try writer.writeAll("\n\n");
965	}	1062	}
966	}	1063	}
967		1064
968	const CorrespondingLines = struct {	1065	const VisualLine = struct {
969	worth_printing_note: bool = true,	1066	line: []u8,
970	worth_printing_lines: bool = true,	1067	truncated: bool,
971	lines: std.ArrayListUnmanaged(u8) = .empty,	1068	};
972	lines_is_error_message: bool = false,
973
974	pub fn init(allocator: std.mem.Allocator, cwd: std.fs.Dir, err_details: ErrorDetails, lines_for_comparison: []const u8, corresponding_span: SourceMappings.CorrespondingSpan, corresponding_file: []const u8) !CorrespondingLines {
975	var corresponding_lines = CorrespondingLines{};
976		1069
		1070	const CorrespondingLines = struct {
		1071	// enough room for one more codepoint, just so that we don't have to keep
		1072	// track of this being truncated, since the extra codepoint will ensure
		1073	// the visual line will need to truncate in that case.
		1074	line_buf: [max_source_line_bytes + 4]u8 = undefined,
		1075	line_len: usize = 0,
		1076	visual_line_buf: [max_source_line_bytes]u8 = undefined,
		1077	visual_line_len: usize = 0,
		1078	truncated: bool = false,
		1079	line_num: usize = 1,
		1080	initial_line: bool = true,
		1081	last_byte: u8 = 0,
		1082	at_eof: bool = false,
		1083	span: SourceMappings.CorrespondingSpan,
		1084	file: std.fs.File,
		1085	buffered_reader: BufferedReaderType,
		1086	code_page: SupportedCodePage,
		1087
		1088	const BufferedReaderType = std.io.BufferedReader(512, std.fs.File.Reader);
		1089
		1090	pub fn init(cwd: std.fs.Dir, err_details: ErrorDetails, line_for_comparison: []const u8, corresponding_span: SourceMappings.CorrespondingSpan, corresponding_file: []const u8) !CorrespondingLines {
977	// We don't do line comparison for this error, so don't print the note if the line	1091	// We don't do line comparison for this error, so don't print the note if the line
978	// number is different	1092	// number is different
979	if (err_details.err == .string_literal_too_long and err_details.token.line_number == corresponding_span.start_line) {	1093	if (err_details.err == .string_literal_too_long and err_details.token.line_number != corresponding_span.start_line) {
980	corresponding_lines.worth_printing_note = false;	1094	return error.NotWorthPrintingNote;
981	return corresponding_lines;
982	}	1095	}
983		1096
984	// Don't print the originating line for this error, we know it's really long	1097	// Don't print the originating line for this error, we know it's really long
985	if (err_details.err == .string_literal_too_long) {	1098	if (err_details.err == .string_literal_too_long) {
986	corresponding_lines.worth_printing_lines = false;	1099	return error.NotWorthPrintingLines;
987	return corresponding_lines;
988	}	1100	}
989		1101
990	var writer = corresponding_lines.lines.writer(allocator);	1102	var corresponding_lines = CorrespondingLines{
991	if (utils.openFileNotDir(cwd, corresponding_file, .{})) \|file\| {	1103	.span = corresponding_span,
992	defer file.close();	1104	.file = try utils.openFileNotDir(cwd, corresponding_file, .{}),
993	var buffered_reader = std.io.bufferedReader(file.reader());	1105	.buffered_reader = undefined,
994	writeLinesFromStream(writer, buffered_reader.reader(), corresponding_span.start_line, corresponding_span.end_line) catch \|err\| switch (err) {	1106	.code_page = err_details.code_page,
995	error.LinesNotFound => {	1107	};
996	corresponding_lines.lines.clearRetainingCapacity();	1108	corresponding_lines.buffered_reader = BufferedReaderType{
997	try writer.print("unable to print line(s) from file: {s}", .{@errorName(err)});	1109	.unbuffered_reader = corresponding_lines.file.reader(),
998	corresponding_lines.lines_is_error_message = true;	1110	};
999	return corresponding_lines;	1111	errdefer corresponding_lines.deinit();
1000	},	1112
1001	else => \|e\| return e,	1113	var fbs = std.io.fixedBufferStream(&corresponding_lines.line_buf);
1002	};	1114	const writer = fbs.writer();
1003	} else \|err\| {	1115
1004	corresponding_lines.lines.clearRetainingCapacity();	1116	try corresponding_lines.writeLineFromStreamVerbatim(
1005	try writer.print("unable to print line(s) from file: {s}", .{@errorName(err)});	1117	writer,
1006	corresponding_lines.lines_is_error_message = true;	1118	corresponding_lines.buffered_reader.reader(),
1007	return corresponding_lines;	1119	corresponding_span.start_line,
1008	}	1120	);
		1121
		1122	const visual_line = writeSourceSlice(
		1123	&corresponding_lines.visual_line_buf,
		1124	corresponding_lines.line_buf[0..corresponding_lines.line_len],
		1125	err_details.code_page,
		1126	);
		1127	corresponding_lines.visual_line_len = visual_line.line.len;
		1128	corresponding_lines.truncated = visual_line.truncated;
1009		1129
1010	// If the lines are the same as they were before preprocessing, skip printing the note entirely	1130	// If the lines are the same as they were before preprocessing, skip printing the note entirely
1011	if (std.mem.eql(u8, lines_for_comparison, corresponding_lines.lines.items)) {	1131	if (corresponding_span.start_line == corresponding_span.end_line and std.mem.eql(
1012	corresponding_lines.worth_printing_note = false;	1132	u8,
		1133	line_for_comparison,
		1134	corresponding_lines.visual_line_buf[0..corresponding_lines.visual_line_len],
		1135	)) {
		1136	return error.NotWorthPrintingNote;
1013	}	1137	}
		1138
1014	return corresponding_lines;	1139	return corresponding_lines;
1015	}	1140	}
1016		1141
1017	pub fn deinit(self: *CorrespondingLines, allocator: std.mem.Allocator) void {	1142	pub fn next(self: *CorrespondingLines) !?VisualLine {
1018	self.lines.deinit(allocator);	1143	if (self.initial_line) {
		1144	self.initial_line = false;
		1145	return .{
		1146	.line = self.visual_line_buf[0..self.visual_line_len],
		1147	.truncated = self.truncated,
		1148	};
		1149	}
		1150	if (self.line_num > self.span.end_line) return null;
		1151	if (self.at_eof) return error.LinesNotFound;
		1152
		1153	self.line_len = 0;
		1154	self.visual_line_len = 0;
		1155
		1156	var fbs = std.io.fixedBufferStream(&self.line_buf);
		1157	const writer = fbs.writer();
		1158
		1159	try self.writeLineFromStreamVerbatim(
		1160	writer,
		1161	self.buffered_reader.reader(),
		1162	self.line_num,
		1163	);
		1164
		1165	const visual_line = writeSourceSlice(
		1166	&self.visual_line_buf,
		1167	self.line_buf[0..self.line_len],
		1168	self.code_page,
		1169	);
		1170	self.visual_line_len = visual_line.line.len;
		1171
		1172	return visual_line;
		1173	}
		1174
		1175	fn writeLineFromStreamVerbatim(self: *CorrespondingLines, writer: anytype, input: anytype, line_num: usize) !void {
		1176	while (try readByteOrEof(input)) \|byte\| {
		1177	switch (byte) {
		1178	'\n', '\r' => {
		1179	if (!utils.isLineEndingPair(self.last_byte, byte)) {
		1180	const line_complete = self.line_num == line_num;
		1181	self.line_num += 1;
		1182	if (line_complete) {
		1183	self.last_byte = byte;
		1184	return;
		1185	}
		1186	} else {
		1187	// reset last_byte to a non-line ending so that
		1188	// consecutive CRLF pairs don't get treated as one
		1189	// long line ending 'pair'
		1190	self.last_byte = 0;
		1191	continue;
		1192	}
		1193	},
		1194	else => {
		1195	if (self.line_num == line_num) {
		1196	if (writer.writeByte(byte)) {
		1197	self.line_len += 1;
		1198	} else \|err\| switch (err) {
		1199	error.NoSpaceLeft => {},
		1200	else => \|e\| return e,
		1201	}
		1202	}
		1203	},
		1204	}
		1205	self.last_byte = byte;
		1206	}
		1207	self.at_eof = true;
		1208	// hacky way to get next to return null
		1209	self.line_num += 1;
		1210	}
		1211
		1212	fn readByteOrEof(reader: anytype) !?u8 {
		1213	return reader.readByte() catch \|err\| switch (err) {
		1214	error.EndOfStream => return null,
		1215	else => \|e\| return e,
		1216	};
		1217	}
		1218
		1219	pub fn deinit(self: *CorrespondingLines) void {
		1220	self.file.close();
1019	}	1221	}
1020	};	1222	};
1021		1223
1022	fn writeSourceSlice(writer: anytype, slice: []const u8) !void {	1224	const max_source_line_codepoints = 120;
1023	for (slice) \|c\| try writeSourceByte(writer, c);	1225	const max_source_line_bytes = max_source_line_codepoints * 4;
		1226
		1227	fn writeSourceSlice(buf: []u8, slice: []const u8, code_page: SupportedCodePage) VisualLine {
		1228	var src_i: usize = 0;
		1229	var dest_i: usize = 0;
		1230	var codepoint_count: usize = 0;
		1231	while (src_i < slice.len) {
		1232	const codepoint = code_page.codepointAt(src_i, slice) orelse break;
		1233	defer src_i += codepoint.byte_len;
		1234	const display_codepoint = codepointForDisplay(codepoint) orelse continue;
		1235	codepoint_count += 1;
		1236	if (codepoint_count > max_source_line_codepoints) {
		1237	return .{ .line = buf[0..dest_i], .truncated = true };
		1238	}
		1239	const utf8_len = std.unicode.utf8Encode(display_codepoint, buf[dest_i..]) catch unreachable;
		1240	dest_i += utf8_len;
		1241	}
		1242	return .{ .line = buf[0..dest_i], .truncated = false };
1024	}	1243	}
1025		1244
1026	inline fn writeSourceByte(writer: anytype, byte: u8) !void {	1245	fn codepointForDisplay(codepoint: code_pages.Codepoint) ?u21 {
1027	switch (byte) {	1246	return switch (codepoint.value) {
1028	'\x00'...'\x08', '\x0E'...'\x1F', '\x7F' => try writer.writeAll("�"),	1247	'\x00'...'\x08',
		1248	'\x0E'...'\x1F',
		1249	'\x7F',
		1250	code_pages.Codepoint.invalid,
		1251	=> '�',
1029	// \r is seemingly ignored by the RC compiler so skipping it when printing source lines	1252	// \r is seemingly ignored by the RC compiler so skipping it when printing source lines
1030	// could help avoid confusing output (e.g. RC\rDATA if printed verbatim would show up	1253	// could help avoid confusing output (e.g. RC\rDATA if printed verbatim would show up
1031	// in the console as DATA but the compiler reads it as RCDATA)	1254	// in the console as DATA but the compiler reads it as RCDATA)
...	@@ -1033,44 +1256,8 @@ inline fn writeSourceByte(writer: anytype, byte: u8) !void {	...	@@ -1033,44 +1256,8 @@ inline fn writeSourceByte(writer: anytype, byte: u8) !void {
1033	// NOTE: This is irrelevant when using the clang preprocessor, because unpaired \r	1256	// NOTE: This is irrelevant when using the clang preprocessor, because unpaired \r
1034	// characters get converted to \n, but may become relevant if another	1257	// characters get converted to \n, but may become relevant if another
1035	// preprocessor is used instead.	1258	// preprocessor is used instead.
1036	'\r' => {},	1259	'\r' => null,
1037	'\t', '\x0B', '\x0C' => try writer.writeByte(' '),	1260	'\t', '\x0B', '\x0C' => ' ',
1038	else => try writer.writeByte(byte),	1261	else => \|v\| v,
1039	}
1040	}
1041
1042	pub fn writeLinesFromStream(writer: anytype, input: anytype, start_line: usize, end_line: usize) !void {
1043	var line_num: usize = 1;
1044	var last_byte: u8 = 0;
1045	while (try readByteOrEof(input)) \|byte\| {
1046	switch (byte) {
1047	'\n', '\r' => {
1048	if (!utils.isLineEndingPair(last_byte, byte)) {
1049	if (line_num == end_line) return;
1050	if (line_num >= start_line) try writeSourceByte(writer, byte);
1051	line_num += 1;
1052	} else {
1053	// reset last_byte to a non-line ending so that
1054	// consecutive CRLF pairs don't get treated as one
1055	// long line ending 'pair'
1056	last_byte = 0;
1057	continue;
1058	}
1059	},
1060	else => {
1061	if (line_num >= start_line) try writeSourceByte(writer, byte);
1062	},
1063	}
1064	last_byte = byte;
1065	}
1066	if (line_num != end_line) {
1067	return error.LinesNotFound;
1068	}
1069	}
1070
1071	pub fn readByteOrEof(reader: anytype) !?u8 {
1072	return reader.readByte() catch \|err\| switch (err) {
1073	error.EndOfStream => return null,
1074	else => \|e\| return e,
1075	};	1262	};
1076	}	1263	}

lib/compiler/resinator/lang.zig+1-1

...	@@ -119,7 +119,7 @@ test tagToId {	...	@@ -119,7 +119,7 @@ test tagToId {
119	}	119	}
120		120
121	test "exhaustive tagToId" {	121	test "exhaustive tagToId" {
122	inline for (@typeInfo(LanguageId).Enum.fields) \|field\| {	122	inline for (@typeInfo(LanguageId).@"enum".fields) \|field\| {
123	const id = tagToId(field.name) catch \|err\| {	123	const id = tagToId(field.name) catch \|err\| {
124	std.debug.print("tag: {s}\n", .{field.name});	124	std.debug.print("tag: {s}\n", .{field.name});
125	return err;	125	return err;

lib/compiler/resinator/lex.zig+134-113

...	@@ -8,7 +8,7 @@ const std = @import("std");	...	@@ -8,7 +8,7 @@ const std = @import("std");
8	const ErrorDetails = @import("errors.zig").ErrorDetails;	8	const ErrorDetails = @import("errors.zig").ErrorDetails;
9	const columnWidth = @import("literals.zig").columnWidth;	9	const columnWidth = @import("literals.zig").columnWidth;
10	const code_pages = @import("code_pages.zig");	10	const code_pages = @import("code_pages.zig");
11	const CodePage = code_pages.CodePage;	11	const SupportedCodePage = code_pages.SupportedCodePage;
12	const SourceMappings = @import("source_mapping.zig").SourceMappings;	12	const SourceMappings = @import("source_mapping.zig").SourceMappings;
13	const isNonAsciiDigit = @import("utils.zig").isNonAsciiDigit;	13	const isNonAsciiDigit = @import("utils.zig").isNonAsciiDigit;
14		14
...	@@ -62,13 +62,6 @@ pub const Token = struct {	...	@@ -62,13 +62,6 @@ pub const Token = struct {
62	return buffer[self.start..self.end];	62	return buffer[self.start..self.end];
63	}	63	}
64		64
65	pub fn nameForErrorDisplay(self: Token, buffer: []const u8) []const u8 {
66	return switch (self.id) {
67	.eof => self.id.nameForErrorDisplay(),
68	else => self.slice(buffer),
69	};
70	}
71
72	/// Returns 0-based column	65	/// Returns 0-based column
73	pub fn calculateColumn(token: Token, source: []const u8, tab_columns: usize, maybe_line_start: ?usize) usize {	66	pub fn calculateColumn(token: Token, source: []const u8, tab_columns: usize, maybe_line_start: ?usize) usize {
74	const line_start = maybe_line_start orelse token.getLineStartForColumnCalc(source);	67	const line_start = maybe_line_start orelse token.getLineStartForColumnCalc(source);
...	@@ -214,18 +207,19 @@ pub const Lexer = struct {	...	@@ -214,18 +207,19 @@ pub const Lexer = struct {
214	line_handler: LineHandler,	207	line_handler: LineHandler,
215	at_start_of_line: bool = true,	208	at_start_of_line: bool = true,
216	error_context_token: ?Token = null,	209	error_context_token: ?Token = null,
217	current_code_page: CodePage,	210	current_code_page: SupportedCodePage,
218	default_code_page: CodePage,	211	default_code_page: SupportedCodePage,
219	source_mappings: ?*SourceMappings,	212	source_mappings: ?*SourceMappings,
220	max_string_literal_codepoints: u15,	213	max_string_literal_codepoints: u15,
221	/// Needed to determine whether or not the output code page should	214	/// Needed to determine whether or not the output code page should
222	/// be set in the parser.	215	/// be set in the parser.
223	seen_pragma_code_pages: u2 = 0,	216	seen_pragma_code_pages: u2 = 0,
		217	last_pragma_code_page_token: ?Token = null,
224		218
225	pub const Error = LexError;	219	pub const Error = LexError;
226		220
227	pub const LexerOptions = struct {	221	pub const LexerOptions = struct {
228	default_code_page: CodePage = .windows1252,	222	default_code_page: SupportedCodePage = .windows1252,
229	source_mappings: ?*SourceMappings = null,	223	source_mappings: ?*SourceMappings = null,
230	max_string_literal_codepoints: u15 = default_max_string_literal_codepoints,	224	max_string_literal_codepoints: u15 = default_max_string_literal_codepoints,
231	};	225	};
...	@@ -291,6 +285,8 @@ pub const Lexer = struct {	...	@@ -291,6 +285,8 @@ pub const Lexer = struct {
291	},	285	},
292	// NBSP only counts as whitespace at the start of a line (but	286	// NBSP only counts as whitespace at the start of a line (but
293	// can be intermixed with other whitespace). Who knows why.	287	// can be intermixed with other whitespace). Who knows why.
		288	// TODO: This should either be removed, or it should also include
		289	// the codepoints listed in disjoint_code_page.zig
294	'\xA0' => if (self.at_start_of_line) {	290	'\xA0' => if (self.at_start_of_line) {
295	result.start = self.index + codepoint.byte_len;	291	result.start = self.index + codepoint.byte_len;
296	} else {	292	} else {
...	@@ -305,12 +301,8 @@ pub const Lexer = struct {	...	@@ -305,12 +301,8 @@ pub const Lexer = struct {
305	}	301	}
306	self.at_start_of_line = false;	302	self.at_start_of_line = false;
307	},	303	},
308	// Semi-colon acts as a line-terminator, but in this lexing mode
309	// that's only true if it's at the start of a line.
310	';' => {	304	';' => {
311	if (self.at_start_of_line) {	305	state = .semicolon;
312	state = .semicolon;
313	}
314	self.at_start_of_line = false;	306	self.at_start_of_line = false;
315	},	307	},
316	else => {	308	else => {
...	@@ -345,7 +337,11 @@ pub const Lexer = struct {	...	@@ -345,7 +337,11 @@ pub const Lexer = struct {
345	}	337	}
346	} else { // got EOF	338	} else { // got EOF
347	switch (state) {	339	switch (state) {
348	.start, .semicolon => {},	340	.start => {},
		341	.semicolon => {
		342	// Skip past everything up to the EOF
		343	result.start = self.index;
		344	},
349	.literal => {	345	.literal => {
350	result.id = .literal;	346	result.id = .literal;
351	},	347	},
...	@@ -357,6 +353,10 @@ pub const Lexer = struct {	...	@@ -357,6 +353,10 @@ pub const Lexer = struct {
357	}	353	}
358		354
359	result.end = self.index;	355	result.end = self.index;
		356
		357	// EOF tokens must have their start index match the end index
		358	std.debug.assert(result.id != .eof or result.start == result.end);
		359
360	return result;	360	return result;
361	}	361	}
362		362
...	@@ -796,7 +796,11 @@ pub const Lexer = struct {	...	@@ -796,7 +796,11 @@ pub const Lexer = struct {
796	}	796	}
797	} else { // got EOF	797	} else { // got EOF
798	switch (state) {	798	switch (state) {
799	.start, .semicolon => {},	799	.start => {},
		800	.semicolon => {
		801	// Skip past everything up to the EOF
		802	result.start = self.index;
		803	},
800	.literal_or_quoted_wide_string, .literal, .e, .en, .b, .be, .beg, .begi => {	804	.literal_or_quoted_wide_string, .literal, .e, .en, .b, .be, .beg, .begi => {
801	result.id = .literal;	805	result.id = .literal;
802	},	806	},
...	@@ -835,6 +839,9 @@ pub const Lexer = struct {	...	@@ -835,6 +839,9 @@ pub const Lexer = struct {
835	}	839	}
836	}	840	}
837		841
		842	// EOF tokens must have their start index match the end index
		843	std.debug.assert(result.id != .eof or result.start == result.end);
		844
838	return result;	845	return result;
839	}	846	}
840		847
...	@@ -878,7 +885,7 @@ pub const Lexer = struct {	...	@@ -878,7 +885,7 @@ pub const Lexer = struct {
878	// and miscompilations when used within string literals. We avoid the miscompilation	885	// and miscompilations when used within string literals. We avoid the miscompilation
879	// within string literals and emit a warning, but outside of string literals it makes	886	// within string literals and emit a warning, but outside of string literals it makes
880	// more sense to just disallow these codepoints.	887	// more sense to just disallow these codepoints.
881	0x900, 0xA00, 0xA0D, 0x2000, 0xFFFE, 0xD00 => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return,	888	0x900, 0xA00, 0xA0D, 0x2000, 0xD00, 0xFFFE, 0xFFFF => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return,
882	else => return,	889	else => return,
883	};	890	};
884	self.error_context_token = .{	891	self.error_context_token = .{
...	@@ -899,90 +906,11 @@ pub const Lexer = struct {	...	@@ -899,90 +906,11 @@ pub const Lexer = struct {
899	};	906	};
900	errdefer self.error_context_token = token;	907	errdefer self.error_context_token = token;
901	const full_command = self.buffer[start..end];	908	const full_command = self.buffer[start..end];
902	var command = full_command;
903
904	// Anything besides exactly this is ignored by the Windows RC implementation
905	const expected_directive = "#pragma";
906	if (!std.mem.startsWith(u8, command, expected_directive)) return;
907	command = command[expected_directive.len..];
908
909	if (command.len == 0 or !std.ascii.isWhitespace(command[0])) return;
910	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
911	command = command[1..];
912	}
913
914	// Note: CoDe_PaGeZ is also treated as "code_page" by the Windows RC implementation,
915	// and it will error with 'Missing left parenthesis in code_page #pragma'
916	const expected_extension = "code_page";
917	if (!std.ascii.startsWithIgnoreCase(command, expected_extension)) return;
918	command = command[expected_extension.len..];
919
920	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
921	command = command[1..];
922	}
923
924	if (command.len == 0 or command[0] != '(') {
925	return error.CodePagePragmaMissingLeftParen;
926	}
927	command = command[1..];
928		909
929	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {	910	const code_page = (parsePragmaCodePage(full_command) catch \|err\| switch (err) {
930	command = command[1..];	911	error.NotPragma, error.NotCodePagePragma => return,
931	}	912	else => \|e\| return e,
932		913	}) orelse self.default_code_page;
933	var num_str: []u8 = command[0..0];
934	while (command.len > 0 and (command[0] != ')' and !std.ascii.isWhitespace(command[0]))) {
935	command = command[1..];
936	num_str.len += 1;
937	}
938
939	if (num_str.len == 0) {
940	return error.CodePagePragmaNotInteger;
941	}
942
943	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
944	command = command[1..];
945	}
946
947	if (command.len == 0 or command[0] != ')') {
948	return error.CodePagePragmaMissingRightParen;
949	}
950
951	const code_page = code_page: {
952	if (std.ascii.eqlIgnoreCase("DEFAULT", num_str)) {
953	break :code_page self.default_code_page;
954	}
955
956	// The Win32 compiler behaves fairly strangely around maxInt(u32):
957	// - If the overflowed u32 wraps and becomes a known code page ID, then
958	// it will error/warn with "Codepage not valid: ignored" (depending on /w)
959	// - If the overflowed u32 wraps and does not become a known code page ID,
960	// then it will error with 'constant too big' and 'Codepage not integer'
961	//
962	// Instead of that, we just have a separate error specifically for overflow.
963	const num = parseCodePageNum(num_str) catch \|err\| switch (err) {
964	error.InvalidCharacter => return error.CodePagePragmaNotInteger,
965	error.Overflow => return error.CodePagePragmaOverflow,
966	};
967
968	// Anything that starts with 0 but does not resolve to 0 is treated as invalid, e.g. 01252
969	if (num_str[0] == '0' and num != 0) {
970	return error.CodePagePragmaInvalidCodePage;
971	}
972	// Anything that resolves to 0 is treated as 'not an integer' by the Win32 implementation.
973	else if (num == 0) {
974	return error.CodePagePragmaNotInteger;
975	}
976	// Anything above u16 max is not going to be found since our CodePage enum is backed by a u16.
977	if (num > std.math.maxInt(u16)) {
978	return error.CodePagePragmaInvalidCodePage;
979	}
980
981	break :code_page code_pages.CodePage.getByIdentifierEnsureSupported(@intCast(num)) catch \|err\| switch (err) {
982	error.InvalidCodePage => return error.CodePagePragmaInvalidCodePage,
983	error.UnsupportedCodePage => return error.CodePagePragmaUnsupportedCodePage,
984	};
985	};
986		914
987	// https://learn.microsoft.com/en-us/windows/win32/menurc/pragma-directives	915	// https://learn.microsoft.com/en-us/windows/win32/menurc/pragma-directives
988	// > This pragma is not supported in an included resource file (.rc)	916	// > This pragma is not supported in an included resource file (.rc)
...	@@ -998,24 +926,16 @@ pub const Lexer = struct {	...	@@ -998,24 +926,16 @@ pub const Lexer = struct {
998	}	926	}
999		927
1000	self.seen_pragma_code_pages +\|= 1;	928	self.seen_pragma_code_pages +\|= 1;
		929	self.last_pragma_code_page_token = token;
1001	self.current_code_page = code_page;	930	self.current_code_page = code_page;
1002	}	931	}
1003		932
1004	fn parseCodePageNum(str: []const u8) !u32 {
1005	var x: u32 = 0;
1006	for (str) \|c\| {
1007	const digit = try std.fmt.charToDigit(c, 10);
1008	if (x != 0) x = try std.math.mul(u32, x, 10);
1009	x = try std.math.add(u32, x, digit);
1010	}
1011	return x;
1012	}
1013
1014	pub fn getErrorDetails(self: Self, lex_err: LexError) ErrorDetails {	933	pub fn getErrorDetails(self: Self, lex_err: LexError) ErrorDetails {
1015	const err = switch (lex_err) {	934	const err = switch (lex_err) {
1016	error.UnfinishedStringLiteral => ErrorDetails.Error.unfinished_string_literal,	935	error.UnfinishedStringLiteral => ErrorDetails.Error.unfinished_string_literal,
1017	error.StringLiteralTooLong => return .{	936	error.StringLiteralTooLong => return .{
1018	.err = .string_literal_too_long,	937	.err = .string_literal_too_long,
		938	.code_page = self.current_code_page,
1019	.token = self.error_context_token.?,	939	.token = self.error_context_token.?,
1020	.extra = .{ .number = self.max_string_literal_codepoints },	940	.extra = .{ .number = self.max_string_literal_codepoints },
1021	},	941	},
...	@@ -1037,11 +957,112 @@ pub const Lexer = struct {	...	@@ -1037,11 +957,112 @@ pub const Lexer = struct {
1037	};	957	};
1038	return .{	958	return .{
1039	.err = err,	959	.err = err,
		960	.code_page = self.current_code_page,
1040	.token = self.error_context_token.?,	961	.token = self.error_context_token.?,
1041	};	962	};
1042	}	963	}
1043	};	964	};
1044		965
		966	fn parseCodePageNum(str: []const u8) !u32 {
		967	var x: u32 = 0;
		968	for (str) \|c\| {
		969	const digit = try std.fmt.charToDigit(c, 10);
		970	if (x != 0) x = try std.math.mul(u32, x, 10);
		971	x = try std.math.add(u32, x, digit);
		972	}
		973	return x;
		974	}
		975
		976	/// Returns `null` when the code_page is set to DEFAULT
		977	pub fn parsePragmaCodePage(full_command: []const u8) !?SupportedCodePage {
		978	var command = full_command;
		979
		980	// Anything besides exactly this is ignored by the Windows RC implementation
		981	const expected_directive = "#pragma";
		982	if (!std.mem.startsWith(u8, command, expected_directive)) return error.NotPragma;
		983	command = command[expected_directive.len..];
		984
		985	if (command.len == 0 or !std.ascii.isWhitespace(command[0])) return error.NotCodePagePragma;
		986	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
		987	command = command[1..];
		988	}
		989
		990	// Note: CoDe_PaGeZ is also treated as "code_page" by the Windows RC implementation,
		991	// and it will error with 'Missing left parenthesis in code_page #pragma'
		992	const expected_extension = "code_page";
		993	if (!std.ascii.startsWithIgnoreCase(command, expected_extension)) return error.NotCodePagePragma;
		994	command = command[expected_extension.len..];
		995
		996	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
		997	command = command[1..];
		998	}
		999
		1000	if (command.len == 0 or command[0] != '(') {
		1001	return error.CodePagePragmaMissingLeftParen;
		1002	}
		1003	command = command[1..];
		1004
		1005	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
		1006	command = command[1..];
		1007	}
		1008
		1009	var num_str: []u8 = command[0..0];
		1010	while (command.len > 0 and (command[0] != ')' and !std.ascii.isWhitespace(command[0]))) {
		1011	command = command[1..];
		1012	num_str.len += 1;
		1013	}
		1014
		1015	if (num_str.len == 0) {
		1016	return error.CodePagePragmaNotInteger;
		1017	}
		1018
		1019	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
		1020	command = command[1..];
		1021	}
		1022
		1023	if (command.len == 0 or command[0] != ')') {
		1024	return error.CodePagePragmaMissingRightParen;
		1025	}
		1026
		1027	const code_page: ?SupportedCodePage = code_page: {
		1028	if (std.ascii.eqlIgnoreCase("DEFAULT", num_str)) {
		1029	break :code_page null;
		1030	}
		1031
		1032	// The Win32 compiler behaves fairly strangely around maxInt(u32):
		1033	// - If the overflowed u32 wraps and becomes a known code page ID, then
		1034	// it will error/warn with "Codepage not valid: ignored" (depending on /w)
		1035	// - If the overflowed u32 wraps and does not become a known code page ID,
		1036	// then it will error with 'constant too big' and 'Codepage not integer'
		1037	//
		1038	// Instead of that, we just have a separate error specifically for overflow.
		1039	const num = parseCodePageNum(num_str) catch \|err\| switch (err) {
		1040	error.InvalidCharacter => return error.CodePagePragmaNotInteger,
		1041	error.Overflow => return error.CodePagePragmaOverflow,
		1042	};
		1043
		1044	// Anything that starts with 0 but does not resolve to 0 is treated as invalid, e.g. 01252
		1045	if (num_str[0] == '0' and num != 0) {
		1046	return error.CodePagePragmaInvalidCodePage;
		1047	}
		1048	// Anything that resolves to 0 is treated as 'not an integer' by the Win32 implementation.
		1049	else if (num == 0) {
		1050	return error.CodePagePragmaNotInteger;
		1051	}
		1052	// Anything above u16 max is not going to be found since our CodePage enum is backed by a u16.
		1053	if (num > std.math.maxInt(u16)) {
		1054	return error.CodePagePragmaInvalidCodePage;
		1055	}
		1056
		1057	break :code_page code_pages.getByIdentifierEnsureSupported(@intCast(num)) catch \|err\| switch (err) {
		1058	error.InvalidCodePage => return error.CodePagePragmaInvalidCodePage,
		1059	error.UnsupportedCodePage => return error.CodePagePragmaUnsupportedCodePage,
		1060	};
		1061	};
		1062
		1063	return code_page;
		1064	}
		1065
1045	fn testLexNormal(source: []const u8, expected_tokens: []const Token.Id) !void {	1066	fn testLexNormal(source: []const u8, expected_tokens: []const Token.Id) !void {
1046	var lexer = Lexer.init(source, .{});	1067	var lexer = Lexer.init(source, .{});
1047	if (dumpTokensDuringTests) std.debug.print("\n----------------------\n{s}\n----------------------\n", .{lexer.buffer});	1068	if (dumpTokensDuringTests) std.debug.print("\n----------------------\n{s}\n----------------------\n", .{lexer.buffer});
...	@@ -1074,7 +1095,7 @@ test "normal: string literals" {	...	@@ -1074,7 +1095,7 @@ test "normal: string literals" {
1074		1095
1075	test "superscript chars and code pages" {	1096	test "superscript chars and code pages" {
1076	const firstToken = struct {	1097	const firstToken = struct {
1077	pub fn firstToken(source: []const u8, default_code_page: CodePage, comptime lex_method: Lexer.LexMethod) LexError!Token {	1098	pub fn firstToken(source: []const u8, default_code_page: SupportedCodePage, comptime lex_method: Lexer.LexMethod) LexError!Token {
1078	var lexer = Lexer.init(source, .{ .default_code_page = default_code_page });	1099	var lexer = Lexer.init(source, .{ .default_code_page = default_code_page });
1079	return lexer.next(lex_method);	1100	return lexer.next(lex_method);
1080	}	1101	}

lib/compiler/resinator/literals.zig+280-93

...	@@ -1,6 +1,6 @@	...	@@ -1,6 +1,6 @@
1	const std = @import("std");	1	const std = @import("std");
2	const code_pages = @import("code_pages.zig");	2	const code_pages = @import("code_pages.zig");
3	const CodePage = code_pages.CodePage;	3	const SupportedCodePage = code_pages.SupportedCodePage;
4	const windows1252 = @import("windows1252.zig");	4	const windows1252 = @import("windows1252.zig");
5	const ErrorDetails = @import("errors.zig").ErrorDetails;	5	const ErrorDetails = @import("errors.zig").ErrorDetails;
6	const DiagnosticsContext = @import("errors.zig").DiagnosticsContext;	6	const DiagnosticsContext = @import("errors.zig").DiagnosticsContext;
...	@@ -18,7 +18,7 @@ pub fn isValidNumberDataLiteral(str: []const u8) bool {	...	@@ -18,7 +18,7 @@ pub fn isValidNumberDataLiteral(str: []const u8) bool {
18		18
19	pub const SourceBytes = struct {	19	pub const SourceBytes = struct {
20	slice: []const u8,	20	slice: []const u8,
21	code_page: CodePage,	21	code_page: SupportedCodePage,
22	};	22	};
23		23
24	pub const StringType = enum { ascii, wide };	24	pub const StringType = enum { ascii, wide };
...	@@ -53,7 +53,7 @@ pub const StringType = enum { ascii, wide };	...	@@ -53,7 +53,7 @@ pub const StringType = enum { ascii, wide };
53	/// branches should never actually be hit during this function.	53	/// branches should never actually be hit during this function.
54	pub const IterativeStringParser = struct {	54	pub const IterativeStringParser = struct {
55	source: []const u8,	55	source: []const u8,
56	code_page: CodePage,	56	code_page: SupportedCodePage,
57	/// The type of the string inferred by the prefix (L"" or "")	57	/// The type of the string inferred by the prefix (L"" or "")
58	/// This is what matters for things like the maximum digits in an	58	/// This is what matters for things like the maximum digits in an
59	/// escape sequence, whether or not invalid escape sequences are skipped, etc.	59	/// escape sequence, whether or not invalid escape sequences are skipped, etc.
...	@@ -98,32 +98,55 @@ pub const IterativeStringParser = struct {	...	@@ -98,32 +98,55 @@ pub const IterativeStringParser = struct {
98		98
99	pub const ParsedCodepoint = struct {	99	pub const ParsedCodepoint = struct {
100	codepoint: u21,	100	codepoint: u21,
101	/// Note: If this is true, `codepoint` will be a value with a max of maxInt(u16).	101	/// Note: If this is true, `codepoint` will have an effective maximum value
102	/// This is enforced by using saturating arithmetic, so in e.g. a wide string literal the	102	/// of 0xFFFF, as `codepoint` is calculated using wrapping arithmetic on a u16.
103	/// octal escape sequence \7777777 (2,097,151) will be parsed into the value 0xFFFF (65,535).	103	/// If the value needs to be truncated to a smaller integer (e.g. for ASCII string
104	/// If the value needs to be truncated to a smaller integer (for ASCII string literals), then that	104	/// literals), then that must be done by the caller.
105	/// must be done by the caller.
106	from_escaped_integer: bool = false,	105	from_escaped_integer: bool = false,
		106	/// Denotes that the codepoint is:
		107	/// - Escaped (has a \ in front of it), and
		108	/// - Has a value >= U+10000, meaning it would be encoded as a surrogate
		109	/// pair in UTF-16, and
		110	/// - Is part of a wide string literal
		111	///
		112	/// Normally in wide string literals, invalid escapes are omitted
		113	/// during parsing (the codepoints are not returned at all during
		114	/// the `next` call), but this is a special case in which the
		115	/// escape only applies to the high surrogate pair of the codepoint.
		116	///
		117	/// TODO: Maybe just return the low surrogate codepoint by itself in this case.
		118	escaped_surrogate_pair: bool = false,
107	};	119	};
108		120
109	pub fn next(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint {	121	pub fn next(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint {
110	const result = try self.nextUnchecked();	122	const result = try self.nextUnchecked();
111	if (self.diagnostics != null and result != null and !result.?.from_escaped_integer) {	123	if (self.diagnostics != null and result != null and !result.?.from_escaped_integer) {
112	switch (result.?.codepoint) {	124	switch (result.?.codepoint) {
113	0x900, 0xA00, 0xA0D, 0x2000, 0xFFFE, 0xD00 => {	125	0x0900, 0x0A00, 0x0A0D, 0x2000, 0x0D00 => {
114	const err: ErrorDetails.Error = if (result.?.codepoint == 0xD00)	126	const err: ErrorDetails.Error = if (result.?.codepoint == 0xD00)
115	.rc_would_miscompile_codepoint_skip	127	.rc_would_miscompile_codepoint_skip
116	else	128	else
117	.rc_would_miscompile_codepoint_byte_swap;	129	.rc_would_miscompile_codepoint_whitespace;
118	try self.diagnostics.?.diagnostics.append(ErrorDetails{	130	try self.diagnostics.?.diagnostics.append(ErrorDetails{
119	.err = err,	131	.err = err,
120	.type = .warning,	132	.type = .warning,
		133	.code_page = self.code_page,
121	.token = self.diagnostics.?.token,	134	.token = self.diagnostics.?.token,
122	.extra = .{ .number = result.?.codepoint },	135	.extra = .{ .number = result.?.codepoint },
123	});	136	});
		137	},
		138	0xFFFE, 0xFFFF => {
124	try self.diagnostics.?.diagnostics.append(ErrorDetails{	139	try self.diagnostics.?.diagnostics.append(ErrorDetails{
125	.err = err,	140	.err = .rc_would_miscompile_codepoint_bom,
		141	.type = .warning,
		142	.code_page = self.code_page,
		143	.token = self.diagnostics.?.token,
		144	.extra = .{ .number = result.?.codepoint },
		145	});
		146	try self.diagnostics.?.diagnostics.append(ErrorDetails{
		147	.err = .rc_would_miscompile_codepoint_bom,
126	.type = .note,	148	.type = .note,
		149	.code_page = self.code_page,
127	.token = self.diagnostics.?.token,	150	.token = self.diagnostics.?.token,
128	.print_source_line = false,	151	.print_source_line = false,
129	.extra = .{ .number = result.?.codepoint },	152	.extra = .{ .number = result.?.codepoint },
...	@@ -188,11 +211,13 @@ pub const IterativeStringParser = struct {	...	@@ -188,11 +211,13 @@ pub const IterativeStringParser = struct {
188	try self.diagnostics.?.diagnostics.append(ErrorDetails{	211	try self.diagnostics.?.diagnostics.append(ErrorDetails{
189	.err = .tab_converted_to_spaces,	212	.err = .tab_converted_to_spaces,
190	.type = .warning,	213	.type = .warning,
		214	.code_page = self.code_page,
191	.token = self.diagnostics.?.token,	215	.token = self.diagnostics.?.token,
192	});	216	});
193	try self.diagnostics.?.diagnostics.append(ErrorDetails{	217	try self.diagnostics.?.diagnostics.append(ErrorDetails{
194	.err = .tab_converted_to_spaces,	218	.err = .tab_converted_to_spaces,
195	.type = .note,	219	.type = .note,
		220	.code_page = self.code_page,
196	.token = self.diagnostics.?.token,	221	.token = self.diagnostics.?.token,
197	.print_source_line = false,	222	.print_source_line = false,
198	});	223	});
...	@@ -246,8 +271,9 @@ pub const IterativeStringParser = struct {	...	@@ -246,8 +271,9 @@ pub const IterativeStringParser = struct {
246	switch (c) {	271	switch (c) {
247	'a', 'A' => {	272	'a', 'A' => {
248	self.index += codepoint.byte_len;	273	self.index += codepoint.byte_len;
		274	// might be a bug in RC, but matches its behavior
249	return .{ .codepoint = '\x08' };	275	return .{ .codepoint = '\x08' };
250	}, // might be a bug in RC, but matches its behavior	276	},
251	'n' => {	277	'n' => {
252	self.index += codepoint.byte_len;	278	self.index += codepoint.byte_len;
253	return .{ .codepoint = '\n' };	279	return .{ .codepoint = '\n' };
...	@@ -269,7 +295,65 @@ pub const IterativeStringParser = struct {	...	@@ -269,7 +295,65 @@ pub const IterativeStringParser = struct {
269	backtrack = true;	295	backtrack = true;
270	},	296	},
271	else => switch (self.declared_string_type) {	297	else => switch (self.declared_string_type) {
272	.wide => {}, // invalid escape sequences are skipped in wide strings	298	.wide => {
		299	// All invalid escape sequences are skipped in wide strings,
		300	// but there is a special case around \<tab> where the \
		301	// is skipped but the tab character is processed.
		302	// It's actually a bit weirder than that, though, since
		303	// the preprocessor is the one that does the <tab> -> spaces
		304	// conversion, so it goes something like this:
		305	//
		306	// Before preprocessing: L"\<tab>"
		307	// After preprocessing: L"\ "
		308	//
		309	// So the parser only sees an escaped space character followed
		310	// by some other number of spaces >= 0.
		311	//
		312	// However, our preprocessor keeps tab characters intact, so we emulate
		313	// the above behavior by skipping the \ and then outputting one less
		314	// space than normal for the <tab> character.
		315	if (c == '\t') {
		316	// Only warn about a tab getting converted to spaces once per string
		317	if (self.diagnostics != null and !self.seen_tab) {
		318	try self.diagnostics.?.diagnostics.append(ErrorDetails{
		319	.err = .tab_converted_to_spaces,
		320	.type = .warning,
		321	.code_page = self.code_page,
		322	.token = self.diagnostics.?.token,
		323	});
		324	try self.diagnostics.?.diagnostics.append(ErrorDetails{
		325	.err = .tab_converted_to_spaces,
		326	.type = .note,
		327	.code_page = self.code_page,
		328	.token = self.diagnostics.?.token,
		329	.print_source_line = false,
		330	});
		331	self.seen_tab = true;
		332	}
		333
		334	const cols = columnsUntilTabStop(self.column, 8);
		335	// If the tab character would only be converted to a single space,
		336	// then we can just skip both the \ and the <tab> and move on.
		337	if (cols > 1) {
		338	self.num_pending_spaces = @intCast(cols - 2);
		339	self.index += codepoint.byte_len;
		340	return .{ .codepoint = ' ' };
		341	}
		342	}
		343	// There's a second special case when the codepoint would be encoded
		344	// as a surrogate pair in UTF-16, as the escape 'applies' to the
		345	// high surrogate pair only in this instance. This is a side-effect
		346	// of the Win32 RC compiler preprocessor outputting UTF-16 and the
		347	// compiler itself seemingly working on code units instead of code points
		348	// in this particular instance.
		349	//
		350	// We emulate this behavior by emitting the codepoint, but with a marker
		351	// that indicates that it needs to be handled specially.
		352	if (c >= 0x10000 and c != code_pages.Codepoint.invalid) {
		353	self.index += codepoint.byte_len;
		354	return .{ .codepoint = c, .escaped_surrogate_pair = true };
		355	}
		356	},
273	.ascii => {	357	.ascii => {
274	// we intentionally avoid incrementing self.index	358	// we intentionally avoid incrementing self.index
275	// to handle the current char in the next call,	359	// to handle the current char in the next call,
...	@@ -303,6 +387,9 @@ pub const IterativeStringParser = struct {	...	@@ -303,6 +387,9 @@ pub const IterativeStringParser = struct {
303	},	387	},
304	.escaped_octal => switch (c) {	388	.escaped_octal => switch (c) {
305	'0'...'7' => {	389	'0'...'7' => {
		390	// Note: We use wrapping arithmetic on a u16 here since there's been no observed
		391	// string parsing scenario where an escaped integer with a value >= the u16
		392	// max is interpreted as anything but the truncated u16 value.
306	string_escape_n *%= 8;	393	string_escape_n *%= 8;
307	string_escape_n +%= std.fmt.charToDigit(@intCast(c), 8) catch unreachable;	394	string_escape_n +%= std.fmt.charToDigit(@intCast(c), 8) catch unreachable;
308	string_escape_i += 1;	395	string_escape_i += 1;
...	@@ -367,7 +454,7 @@ pub const IterativeStringParser = struct {	...	@@ -367,7 +454,7 @@ pub const IterativeStringParser = struct {
367	pub const StringParseOptions = struct {	454	pub const StringParseOptions = struct {
368	start_column: usize = 0,	455	start_column: usize = 0,
369	diagnostics: ?DiagnosticsContext = null,	456	diagnostics: ?DiagnosticsContext = null,
370	output_code_page: CodePage = .windows1252,	457	output_code_page: SupportedCodePage,
371	};	458	};
372		459
373	pub fn parseQuotedString(	460	pub fn parseQuotedString(
...	@@ -389,46 +476,52 @@ pub fn parseQuotedString(	...	@@ -389,46 +476,52 @@ pub fn parseQuotedString(
389		476
390	while (try iterative_parser.next()) \|parsed\| {	477	while (try iterative_parser.next()) \|parsed\| {
391	const c = parsed.codepoint;	478	const c = parsed.codepoint;
392	if (parsed.from_escaped_integer) {	479	switch (literal_type) {
393	// We truncate here to get the correct behavior for ascii strings	480	.ascii => switch (options.output_code_page) {
394	try buf.append(std.mem.nativeToLittle(T, @truncate(c)));	481	.windows1252 => {
395	} else {	482	if (parsed.from_escaped_integer) {
396	switch (literal_type) {	483	try buf.append(@truncate(c));
397	.ascii => switch (options.output_code_page) {	484	} else if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {
398	.windows1252 => {	485	try buf.append(best_fit);
399	if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {	486	} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
400	try buf.append(best_fit);	487	try buf.append('?');
401	} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
402	try buf.append('?');
403	} else {
404	try buf.appendSlice("??");
405	}
406	},
407	.utf8 => {
408	var codepoint_to_encode = c;
409	if (c == code_pages.Codepoint.invalid) {
410	codepoint_to_encode = '�';
411	}
412	var utf8_buf: [4]u8 = undefined;
413	const utf8_len = std.unicode.utf8Encode(codepoint_to_encode, &utf8_buf) catch unreachable;
414	try buf.appendSlice(utf8_buf[0..utf8_len]);
415	},
416	else => unreachable, // Unsupported code page
417	},
418	.wide => {
419	if (c == code_pages.Codepoint.invalid) {
420	try buf.append(std.mem.nativeToLittle(u16, '�'));
421	} else if (c < 0x10000) {
422	const short: u16 = @intCast(c);
423	try buf.append(std.mem.nativeToLittle(u16, short));
424	} else {	488	} else {
		489	try buf.appendSlice("??");
		490	}
		491	},
		492	.utf8 => {
		493	var codepoint_to_encode = c;
		494	if (parsed.from_escaped_integer) {
		495	codepoint_to_encode = @as(T, @truncate(c));
		496	}
		497	const escaped_integer_outside_ascii_range = parsed.from_escaped_integer and codepoint_to_encode > 0x7F;
		498	if (escaped_integer_outside_ascii_range or c == code_pages.Codepoint.invalid) {
		499	codepoint_to_encode = '�';
		500	}
		501	var utf8_buf: [4]u8 = undefined;
		502	const utf8_len = std.unicode.utf8Encode(codepoint_to_encode, &utf8_buf) catch unreachable;
		503	try buf.appendSlice(utf8_buf[0..utf8_len]);
		504	},
		505	},
		506	.wide => {
		507	// Parsing any string type as a wide string is handled separately, see parseQuotedStringAsWideString
		508	std.debug.assert(iterative_parser.declared_string_type == .wide);
		509	if (parsed.from_escaped_integer) {
		510	try buf.append(std.mem.nativeToLittle(u16, @truncate(c)));
		511	} else if (c == code_pages.Codepoint.invalid) {
		512	try buf.append(std.mem.nativeToLittle(u16, '�'));
		513	} else if (c < 0x10000) {
		514	const short: u16 = @intCast(c);
		515	try buf.append(std.mem.nativeToLittle(u16, short));
		516	} else {
		517	if (!parsed.escaped_surrogate_pair) {
425	const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;	518	const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;
426	try buf.append(std.mem.nativeToLittle(u16, high));	519	try buf.append(std.mem.nativeToLittle(u16, high));
427	const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
428	try buf.append(std.mem.nativeToLittle(u16, low));
429	}	520	}
430	},	521	const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
431	}	522	try buf.append(std.mem.nativeToLittle(u16, low));
		523	}
		524	},
432	}	525	}
433	}	526	}
434		527
...	@@ -449,9 +542,59 @@ pub fn parseQuotedWideString(allocator: std.mem.Allocator, bytes: SourceBytes, o	...	@@ -449,9 +542,59 @@ pub fn parseQuotedWideString(allocator: std.mem.Allocator, bytes: SourceBytes, o
449	return parseQuotedString(.wide, allocator, bytes, options);	542	return parseQuotedString(.wide, allocator, bytes, options);
450	}	543	}
451		544
		545	/// Parses any string type into a wide string.
		546	/// If the string is declared as a wide string (L""), then it is handled normally.
		547	/// Otherwise, things are fairly normal with the exception of escaped integers.
		548	/// Escaped integers are handled by:
		549	/// - Truncating the escape to a u8
		550	/// - Reinterpeting the u8 as a byte from the output code page
		551	/// - Outputting the codepoint that corresponds to the interpreted byte, or � if no such
		552	/// interpretation is possible
		553	/// For example, if the code page is UTF-8, then while \x80 is a valid start byte, it's
		554	/// interpreted as a single byte, so it ends up being seen as invalid and � is outputted.
		555	/// If the code page is Windows-1252, then \x80 is interpreted to be € which has the
		556	/// codepoint U+20AC, so the UTF-16 encoding of U+20AC is outputted.
452	pub fn parseQuotedStringAsWideString(allocator: std.mem.Allocator, bytes: SourceBytes, options: StringParseOptions) ![:0]u16 {	557	pub fn parseQuotedStringAsWideString(allocator: std.mem.Allocator, bytes: SourceBytes, options: StringParseOptions) ![:0]u16 {
453	std.debug.assert(bytes.slice.len >= 2); // ""	558	std.debug.assert(bytes.slice.len >= 2); // ""
454	return parseQuotedString(.wide, allocator, bytes, options);	559
		560	if (bytes.slice[0] == 'l' or bytes.slice[0] == 'L') {
		561	return parseQuotedWideString(allocator, bytes, options);
		562	}
		563
		564	// Note: We're only handling the case of parsing an ASCII string into a wide string from here on out.
		565	// TODO: The logic below is similar to that in AcceleratorKeyCodepointTranslator, might be worth merging the two
		566
		567	var buf = try std.ArrayList(u16).initCapacity(allocator, bytes.slice.len);
		568	errdefer buf.deinit();
		569
		570	var iterative_parser = IterativeStringParser.init(bytes, options);
		571
		572	while (try iterative_parser.next()) \|parsed\| {
		573	const c = parsed.codepoint;
		574	if (parsed.from_escaped_integer) {
		575	std.debug.assert(c != code_pages.Codepoint.invalid);
		576	const byte_to_interpret: u8 = @truncate(c);
		577	const code_unit_to_encode: u16 = switch (options.output_code_page) {
		578	.windows1252 => windows1252.toCodepoint(byte_to_interpret),
		579	.utf8 => if (byte_to_interpret > 0x7F) '�' else byte_to_interpret,
		580	};
		581	try buf.append(std.mem.nativeToLittle(u16, code_unit_to_encode));
		582	} else if (c == code_pages.Codepoint.invalid) {
		583	try buf.append(std.mem.nativeToLittle(u16, '�'));
		584	} else if (c < 0x10000) {
		585	const short: u16 = @intCast(c);
		586	try buf.append(std.mem.nativeToLittle(u16, short));
		587	} else {
		588	if (!parsed.escaped_surrogate_pair) {
		589	const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;
		590	try buf.append(std.mem.nativeToLittle(u16, high));
		591	}
		592	const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
		593	try buf.append(std.mem.nativeToLittle(u16, low));
		594	}
		595	}
		596
		597	return buf.toOwnedSliceSentinel(0);
455	}	598	}
456		599
457	test "parse quoted ascii string" {	600	test "parse quoted ascii string" {
...	@@ -464,133 +607,155 @@ test "parse quoted ascii string" {	...	@@ -464,133 +607,155 @@ test "parse quoted ascii string" {
464	\\"hello"	607	\\"hello"
465	,	608	,
466	.code_page = .windows1252,	609	.code_page = .windows1252,
467	}, .{}));	610	}, .{
		611	.output_code_page = .windows1252,
		612	}));
468	// hex with 0 digits	613	// hex with 0 digits
469	try std.testing.expectEqualSlices(u8, "\x00", try parseQuotedAsciiString(arena, .{	614	try std.testing.expectEqualSlices(u8, "\x00", try parseQuotedAsciiString(arena, .{
470	.slice =	615	.slice =
471	\\"\x"	616	\\"\x"
472	,	617	,
473	.code_page = .windows1252,	618	.code_page = .windows1252,
474	}, .{}));	619	}, .{
		620	.output_code_page = .windows1252,
		621	}));
475	// hex max of 2 digits	622	// hex max of 2 digits
476	try std.testing.expectEqualSlices(u8, "\xFFf", try parseQuotedAsciiString(arena, .{	623	try std.testing.expectEqualSlices(u8, "\xFFf", try parseQuotedAsciiString(arena, .{
477	.slice =	624	.slice =
478	\\"\XfFf"	625	\\"\XfFf"
479	,	626	,
480	.code_page = .windows1252,	627	.code_page = .windows1252,
481	}, .{}));	628	}, .{
		629	.output_code_page = .windows1252,
		630	}));
482	// octal with invalid octal digit	631	// octal with invalid octal digit
483	try std.testing.expectEqualSlices(u8, "\x019", try parseQuotedAsciiString(arena, .{	632	try std.testing.expectEqualSlices(u8, "\x019", try parseQuotedAsciiString(arena, .{
484	.slice =	633	.slice =
485	\\"\19"	634	\\"\19"
486	,	635	,
487	.code_page = .windows1252,	636	.code_page = .windows1252,
488	}, .{}));	637	}, .{
		638	.output_code_page = .windows1252,
		639	}));
489	// escaped quotes	640	// escaped quotes
490	try std.testing.expectEqualSlices(u8, " \" ", try parseQuotedAsciiString(arena, .{	641	try std.testing.expectEqualSlices(u8, " \" ", try parseQuotedAsciiString(arena, .{
491	.slice =	642	.slice =
492	\\" "" "	643	\\" "" "
493	,	644	,
494	.code_page = .windows1252,	645	.code_page = .windows1252,
495	}, .{}));	646	}, .{
		647	.output_code_page = .windows1252,
		648	}));
496	// backslash right before escaped quotes	649	// backslash right before escaped quotes
497	try std.testing.expectEqualSlices(u8, "\"", try parseQuotedAsciiString(arena, .{	650	try std.testing.expectEqualSlices(u8, "\"", try parseQuotedAsciiString(arena, .{
498	.slice =	651	.slice =
499	\\"\"""	652	\\"\"""
500	,	653	,
501	.code_page = .windows1252,	654	.code_page = .windows1252,
502	}, .{}));	655	}, .{
		656	.output_code_page = .windows1252,
		657	}));
503	// octal overflow	658	// octal overflow
504	try std.testing.expectEqualSlices(u8, "\x01", try parseQuotedAsciiString(arena, .{	659	try std.testing.expectEqualSlices(u8, "\x01", try parseQuotedAsciiString(arena, .{
505	.slice =	660	.slice =
506	\\"\401"	661	\\"\401"
507	,	662	,
508	.code_page = .windows1252,	663	.code_page = .windows1252,
509	}, .{}));	664	}, .{
		665	.output_code_page = .windows1252,
		666	}));
510	// escapes	667	// escapes
511	try std.testing.expectEqualSlices(u8, "\x08\n\r\t\\", try parseQuotedAsciiString(arena, .{	668	try std.testing.expectEqualSlices(u8, "\x08\n\r\t\\", try parseQuotedAsciiString(arena, .{
512	.slice =	669	.slice =
513	\\"\a\n\r\t\\"	670	\\"\a\n\r\t\\"
514	,	671	,
515	.code_page = .windows1252,	672	.code_page = .windows1252,
516	}, .{}));	673	}, .{
		674	.output_code_page = .windows1252,
		675	}));
517	// uppercase escapes	676	// uppercase escapes
518	try std.testing.expectEqualSlices(u8, "\x08\\N\\R\t\\", try parseQuotedAsciiString(arena, .{	677	try std.testing.expectEqualSlices(u8, "\x08\\N\\R\t\\", try parseQuotedAsciiString(arena, .{
519	.slice =	678	.slice =
520	\\"\A\N\R\T\\"	679	\\"\A\N\R\T\\"
521	,	680	,
522	.code_page = .windows1252,	681	.code_page = .windows1252,
523	}, .{}));	682	}, .{
		683	.output_code_page = .windows1252,
		684	}));
524	// backslash on its own	685	// backslash on its own
525	try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(arena, .{	686	try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(arena, .{
526	.slice =	687	.slice =
527	\\"\"	688	\\"\"
528	,	689	,
529	.code_page = .windows1252,	690	.code_page = .windows1252,
530	}, .{}));	691	}, .{
		692	.output_code_page = .windows1252,
		693	}));
531	// unrecognized escapes	694	// unrecognized escapes
532	try std.testing.expectEqualSlices(u8, "\\b", try parseQuotedAsciiString(arena, .{	695	try std.testing.expectEqualSlices(u8, "\\b", try parseQuotedAsciiString(arena, .{
533	.slice =	696	.slice =
534	\\"\b"	697	\\"\b"
535	,	698	,
536	.code_page = .windows1252,	699	.code_page = .windows1252,
537	}, .{}));	700	}, .{
		701	.output_code_page = .windows1252,
		702	}));
538	// escaped carriage returns	703	// escaped carriage returns
539	try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(	704	try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(
540	arena,	705	arena,
541	.{ .slice = "\"\\\r\r\r\r\r\"", .code_page = .windows1252 },	706	.{ .slice = "\"\\\r\r\r\r\r\"", .code_page = .windows1252 },
542	.{},	707	.{ .output_code_page = .windows1252 },
543	));	708	));
544	// escaped newlines	709	// escaped newlines
545	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(	710	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
546	arena,	711	arena,
547	.{ .slice = "\"\\\n\n\n\n\n\"", .code_page = .windows1252 },	712	.{ .slice = "\"\\\n\n\n\n\n\"", .code_page = .windows1252 },
548	.{},	713	.{ .output_code_page = .windows1252 },
549	));	714	));
550	// escaped CRLF pairs	715	// escaped CRLF pairs
551	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(	716	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
552	arena,	717	arena,
553	.{ .slice = "\"\\\r\n\r\n\r\n\r\n\r\n\"", .code_page = .windows1252 },	718	.{ .slice = "\"\\\r\n\r\n\r\n\r\n\r\n\"", .code_page = .windows1252 },
554	.{},	719	.{ .output_code_page = .windows1252 },
555	));	720	));
556	// escaped newlines with other whitespace	721	// escaped newlines with other whitespace
557	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(	722	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
558	arena,	723	arena,
559	.{ .slice = "\"\\\n \t\r\n \r\t\n \t\"", .code_page = .windows1252 },	724	.{ .slice = "\"\\\n \t\r\n \r\t\n \t\"", .code_page = .windows1252 },
560	.{},	725	.{ .output_code_page = .windows1252 },
561	));	726	));
562	// literal tab characters get converted to spaces (dependent on source file columns)	727	// literal tab characters get converted to spaces (dependent on source file columns)
563	try std.testing.expectEqualSlices(u8, " ", try parseQuotedAsciiString(	728	try std.testing.expectEqualSlices(u8, " ", try parseQuotedAsciiString(
564	arena,	729	arena,
565	.{ .slice = "\"\t\"", .code_page = .windows1252 },	730	.{ .slice = "\"\t\"", .code_page = .windows1252 },
566	.{},	731	.{ .output_code_page = .windows1252 },
567	));	732	));
568	try std.testing.expectEqualSlices(u8, "abc ", try parseQuotedAsciiString(	733	try std.testing.expectEqualSlices(u8, "abc ", try parseQuotedAsciiString(
569	arena,	734	arena,
570	.{ .slice = "\"abc\t\"", .code_page = .windows1252 },	735	.{ .slice = "\"abc\t\"", .code_page = .windows1252 },
571	.{},	736	.{ .output_code_page = .windows1252 },
572	));	737	));
573	try std.testing.expectEqualSlices(u8, "abcdefg ", try parseQuotedAsciiString(	738	try std.testing.expectEqualSlices(u8, "abcdefg ", try parseQuotedAsciiString(
574	arena,	739	arena,
575	.{ .slice = "\"abcdefg\t\"", .code_page = .windows1252 },	740	.{ .slice = "\"abcdefg\t\"", .code_page = .windows1252 },
576	.{},	741	.{ .output_code_page = .windows1252 },
577	));	742	));
578	try std.testing.expectEqualSlices(u8, "\\ ", try parseQuotedAsciiString(	743	try std.testing.expectEqualSlices(u8, "\\ ", try parseQuotedAsciiString(
579	arena,	744	arena,
580	.{ .slice = "\"\\\t\"", .code_page = .windows1252 },	745	.{ .slice = "\"\\\t\"", .code_page = .windows1252 },
581	.{},	746	.{ .output_code_page = .windows1252 },
582	));	747	));
583	// literal CR's get dropped	748	// literal CR's get dropped
584	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(	749	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
585	arena,	750	arena,
586	.{ .slice = "\"\r\r\r\r\r\"", .code_page = .windows1252 },	751	.{ .slice = "\"\r\r\r\r\r\"", .code_page = .windows1252 },
587	.{},	752	.{ .output_code_page = .windows1252 },
588	));	753	));
589	// contiguous newlines and whitespace get collapsed to <space><newline>	754	// contiguous newlines and whitespace get collapsed to <space><newline>
590	try std.testing.expectEqualSlices(u8, " \n", try parseQuotedAsciiString(	755	try std.testing.expectEqualSlices(u8, " \n", try parseQuotedAsciiString(
591	arena,	756	arena,
592	.{ .slice = "\"\n\r\r \r\n \t \"", .code_page = .windows1252 },	757	.{ .slice = "\"\n\r\r \r\n \t \"", .code_page = .windows1252 },
593	.{},	758	.{ .output_code_page = .windows1252 },
594	));	759	));
595	}	760	}
596		761
...	@@ -602,32 +767,32 @@ test "parse quoted ascii string with utf8 code page" {	...	@@ -602,32 +767,32 @@ test "parse quoted ascii string with utf8 code page" {
602	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(	767	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
603	arena,	768	arena,
604	.{ .slice = "\"\"", .code_page = .utf8 },	769	.{ .slice = "\"\"", .code_page = .utf8 },
605	.{},	770	.{ .output_code_page = .windows1252 },
606	));	771	));
607	// Codepoints that don't have a Windows-1252 representation get converted to ?	772	// Codepoints that don't have a Windows-1252 representation get converted to ?
608	try std.testing.expectEqualSlices(u8, "?????????", try parseQuotedAsciiString(	773	try std.testing.expectEqualSlices(u8, "?????????", try parseQuotedAsciiString(
609	arena,	774	arena,
610	.{ .slice = "\"кириллица\"", .code_page = .utf8 },	775	.{ .slice = "\"кириллица\"", .code_page = .utf8 },
611	.{},	776	.{ .output_code_page = .windows1252 },
612	));	777	));
613	// Codepoints that have a best fit mapping get converted accordingly,	778	// Codepoints that have a best fit mapping get converted accordingly,
614	// these are box drawing codepoints	779	// these are box drawing codepoints
615	try std.testing.expectEqualSlices(u8, "\x2b\x2d\x2b", try parseQuotedAsciiString(	780	try std.testing.expectEqualSlices(u8, "\x2b\x2d\x2b", try parseQuotedAsciiString(
616	arena,	781	arena,
617	.{ .slice = "\"┌─┐\"", .code_page = .utf8 },	782	.{ .slice = "\"┌─┐\"", .code_page = .utf8 },
618	.{},	783	.{ .output_code_page = .windows1252 },
619	));	784	));
620	// Invalid UTF-8 gets converted to ? depending on well-formedness	785	// Invalid UTF-8 gets converted to ? depending on well-formedness
621	try std.testing.expectEqualSlices(u8, "????", try parseQuotedAsciiString(	786	try std.testing.expectEqualSlices(u8, "????", try parseQuotedAsciiString(
622	arena,	787	arena,
623	.{ .slice = "\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },	788	.{ .slice = "\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },
624	.{},	789	.{ .output_code_page = .windows1252 },
625	));	790	));
626	// Codepoints that would require a UTF-16 surrogate pair get converted to ??	791	// Codepoints that would require a UTF-16 surrogate pair get converted to ??
627	try std.testing.expectEqualSlices(u8, "??", try parseQuotedAsciiString(	792	try std.testing.expectEqualSlices(u8, "??", try parseQuotedAsciiString(
628	arena,	793	arena,
629	.{ .slice = "\"\xF2\xAF\xBA\xB4\"", .code_page = .utf8 },	794	.{ .slice = "\"\xF2\xAF\xBA\xB4\"", .code_page = .utf8 },
630	.{},	795	.{ .output_code_page = .windows1252 },
631	));	796	));
632		797
633	// Output code page changes how invalid UTF-8 gets converted, since it	798	// Output code page changes how invalid UTF-8 gets converted, since it
...	@@ -652,6 +817,18 @@ test "parse quoted ascii string with utf8 code page" {	...	@@ -652,6 +817,18 @@ test "parse quoted ascii string with utf8 code page" {
652	));	817	));
653	}	818	}
654		819
		820	test "parse quoted string with different input/output code pages" {
		821	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
		822	defer arena_allocator.deinit();
		823	const arena = arena_allocator.allocator();
		824
		825	try std.testing.expectEqualSlices(u8, "€��\x60\x7F", try parseQuotedAsciiString(
		826	arena,
		827	.{ .slice = "\"\x80\\x8a\\600\\612\\540\\577\"", .code_page = .windows1252 },
		828	.{ .output_code_page = .utf8 },
		829	));
		830	}
		831
655	test "parse quoted wide string" {	832	test "parse quoted wide string" {
656	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);	833	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
657	defer arena_allocator.deinit();	834	defer arena_allocator.deinit();
...	@@ -662,52 +839,62 @@ test "parse quoted wide string" {	...	@@ -662,52 +839,62 @@ test "parse quoted wide string" {
662	\\L"hello"	839	\\L"hello"
663	,	840	,
664	.code_page = .windows1252,	841	.code_page = .windows1252,
665	}, .{}));	842	}, .{
		843	.output_code_page = .windows1252,
		844	}));
666	// hex with 0 digits	845	// hex with 0 digits
667	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{0x0}, try parseQuotedWideString(arena, .{	846	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{0x0}, try parseQuotedWideString(arena, .{
668	.slice =	847	.slice =
669	\\L"\x"	848	\\L"\x"
670	,	849	,
671	.code_page = .windows1252,	850	.code_page = .windows1252,
672	}, .{}));	851	}, .{
		852	.output_code_page = .windows1252,
		853	}));
673	// hex max of 4 digits	854	// hex max of 4 digits
674	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0xFFFF), std.mem.nativeToLittle(u16, 'f') }, try parseQuotedWideString(arena, .{	855	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0xFFFF), std.mem.nativeToLittle(u16, 'f') }, try parseQuotedWideString(arena, .{
675	.slice =	856	.slice =
676	\\L"\XfFfFf"	857	\\L"\XfFfFf"
677	,	858	,
678	.code_page = .windows1252,	859	.code_page = .windows1252,
679	}, .{}));	860	}, .{
		861	.output_code_page = .windows1252,
		862	}));
680	// octal max of 7 digits	863	// octal max of 7 digits
681	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x9493), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '3') }, try parseQuotedWideString(arena, .{	864	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x9493), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '3') }, try parseQuotedWideString(arena, .{
682	.slice =	865	.slice =
683	\\L"\111222333"	866	\\L"\111222333"
684	,	867	,
685	.code_page = .windows1252,	868	.code_page = .windows1252,
686	}, .{}));	869	}, .{
		870	.output_code_page = .windows1252,
		871	}));
687	// octal overflow	872	// octal overflow
688	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0xFF01)}, try parseQuotedWideString(arena, .{	873	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0xFF01)}, try parseQuotedWideString(arena, .{
689	.slice =	874	.slice =
690	\\L"\777401"	875	\\L"\777401"
691	,	876	,
692	.code_page = .windows1252,	877	.code_page = .windows1252,
693	}, .{}));	878	}, .{
		879	.output_code_page = .windows1252,
		880	}));
694	// literal tab characters get converted to spaces (dependent on source file columns)	881	// literal tab characters get converted to spaces (dependent on source file columns)
695	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("abcdefg "), try parseQuotedWideString(	882	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("abcdefg "), try parseQuotedWideString(
696	arena,	883	arena,
697	.{ .slice = "L\"abcdefg\t\"", .code_page = .windows1252 },	884	.{ .slice = "L\"abcdefg\t\"", .code_page = .windows1252 },
698	.{},	885	.{ .output_code_page = .windows1252 },
699	));	886	));
700	// Windows-1252 conversion	887	// Windows-1252 conversion
701	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("ðð€€€"), try parseQuotedWideString(	888	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("ðð€€€"), try parseQuotedWideString(
702	arena,	889	arena,
703	.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .windows1252 },	890	.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .windows1252 },
704	.{},	891	.{ .output_code_page = .windows1252 },
705	));	892	));
706	// Invalid escape sequences are skipped	893	// Invalid escape sequences are skipped
707	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedWideString(	894	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedWideString(
708	arena,	895	arena,
709	.{ .slice = "L\"\\H\"", .code_page = .windows1252 },	896	.{ .slice = "L\"\\H\"", .code_page = .windows1252 },
710	.{},	897	.{ .output_code_page = .windows1252 },
711	));	898	));
712	}	899	}
713		900
...	@@ -719,18 +906,18 @@ test "parse quoted wide string with utf8 code page" {	...	@@ -719,18 +906,18 @@ test "parse quoted wide string with utf8 code page" {
719	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{}, try parseQuotedWideString(	906	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{}, try parseQuotedWideString(
720	arena,	907	arena,
721	.{ .slice = "L\"\"", .code_page = .utf8 },	908	.{ .slice = "L\"\"", .code_page = .utf8 },
722	.{},	909	.{ .output_code_page = .windows1252 },
723	));	910	));
724	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedWideString(	911	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedWideString(
725	arena,	912	arena,
726	.{ .slice = "L\"кириллица\"", .code_page = .utf8 },	913	.{ .slice = "L\"кириллица\"", .code_page = .utf8 },
727	.{},	914	.{ .output_code_page = .windows1252 },
728	));	915	));
729	// Invalid UTF-8 gets converted to � depending on well-formedness	916	// Invalid UTF-8 gets converted to � depending on well-formedness
730	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("��"), try parseQuotedWideString(	917	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("��"), try parseQuotedWideString(
731	arena,	918	arena,
732	.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },	919	.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },
733	.{},	920	.{ .output_code_page = .windows1252 },
734	));	921	));
735	}	922	}
736		923
...	@@ -742,29 +929,29 @@ test "parse quoted ascii string as wide string" {	...	@@ -742,29 +929,29 @@ test "parse quoted ascii string as wide string" {
742	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedStringAsWideString(	929	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedStringAsWideString(
743	arena,	930	arena,
744	.{ .slice = "\"кириллица\"", .code_page = .utf8 },	931	.{ .slice = "\"кириллица\"", .code_page = .utf8 },
745	.{},	932	.{ .output_code_page = .windows1252 },
746	));	933	));
747	// Whether or not invalid escapes are skipped is still determined by the L prefix	934	// Whether or not invalid escapes are skipped is still determined by the L prefix
748	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("\\H"), try parseQuotedStringAsWideString(	935	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("\\H"), try parseQuotedStringAsWideString(
749	arena,	936	arena,
750	.{ .slice = "\"\\H\"", .code_page = .windows1252 },	937	.{ .slice = "\"\\H\"", .code_page = .windows1252 },
751	.{},	938	.{ .output_code_page = .windows1252 },
752	));	939	));
753	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedStringAsWideString(	940	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedStringAsWideString(
754	arena,	941	arena,
755	.{ .slice = "L\"\\H\"", .code_page = .windows1252 },	942	.{ .slice = "L\"\\H\"", .code_page = .windows1252 },
756	.{},	943	.{ .output_code_page = .windows1252 },
757	));	944	));
758	// Maximum escape sequence value is also determined by the L prefix	945	// Maximum escape sequence value is also determined by the L prefix
759	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x12), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '4') }, try parseQuotedStringAsWideString(	946	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x12), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '4') }, try parseQuotedStringAsWideString(
760	arena,	947	arena,
761	.{ .slice = "\"\\x1234\"", .code_page = .windows1252 },	948	.{ .slice = "\"\\x1234\"", .code_page = .windows1252 },
762	.{},	949	.{ .output_code_page = .windows1252 },
763	));	950	));
764	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0x1234)}, try parseQuotedStringAsWideString(	951	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0x1234)}, try parseQuotedStringAsWideString(
765	arena,	952	arena,
766	.{ .slice = "L\"\\x1234\"", .code_page = .windows1252 },	953	.{ .slice = "L\"\\x1234\"", .code_page = .windows1252 },
767	.{},	954	.{ .output_code_page = .windows1252 },
768	));	955	));
769	}	956	}
770		957

lib/compiler/resinator/main.zig+25-9

...	@@ -7,6 +7,7 @@ const Diagnostics = @import("errors.zig").Diagnostics;	...	@@ -7,6 +7,7 @@ const Diagnostics = @import("errors.zig").Diagnostics;
7	const cli = @import("cli.zig");	7	const cli = @import("cli.zig");
8	const preprocess = @import("preprocess.zig");	8	const preprocess = @import("preprocess.zig");
9	const renderErrorMessage = @import("utils.zig").renderErrorMessage;	9	const renderErrorMessage = @import("utils.zig").renderErrorMessage;
		10	const hasDisjointCodePage = @import("disjoint_code_page.zig").hasDisjointCodePage;
10	const aro = @import("aro");	11	const aro = @import("aro");
11		12
12	pub fn main() !void {	13	pub fn main() !void {
...	@@ -179,16 +180,30 @@ pub fn main() !void {	...	@@ -179,16 +180,30 @@ pub fn main() !void {
179	// Note: We still want to run this when no-preprocess is set because:	180	// Note: We still want to run this when no-preprocess is set because:
180	// 1. We want to print accurate line numbers after removing multiline comments	181	// 1. We want to print accurate line numbers after removing multiline comments
181	// 2. We want to be able to handle an already-preprocessed input with #line commands in it	182	// 2. We want to be able to handle an already-preprocessed input with #line commands in it
182	var mapping_results = try parseAndRemoveLineCommands(allocator, full_input, full_input, .{ .initial_filename = options.input_filename });	183	var mapping_results = parseAndRemoveLineCommands(allocator, full_input, full_input, .{ .initial_filename = options.input_filename }) catch \|err\| switch (err) {
183	defer mapping_results.mappings.deinit(allocator);	184	error.InvalidLineCommand => {
184		185	// TODO: Maybe output the invalid line command
185	const final_input = removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings) catch \|err\| switch (err) {	186	try renderErrorMessage(stderr.writer(), stderr_config, .err, "invalid line command in the preprocessed source", .{});
186	error.InvalidSourceMappingCollapse => {	187	if (options.preprocess == .no) {
187	try error_handler.emitMessage(allocator, .err, "failed during comment removal; this is a known bug", .{});	188	try renderErrorMessage(stderr.writer(), stderr_config, .note, "line commands must be of the format: #line <num> \"<path>\"", .{});
		189	} else {
		190	try renderErrorMessage(stderr.writer(), stderr_config, .note, "this is likely to be a bug, please report it", .{});
		191	}
188	std.process.exit(1);	192	std.process.exit(1);
189	},	193	},
190	else => \|e\| return e,	194	error.LineNumberOverflow => {
		195	// TODO: Better error message
		196	try renderErrorMessage(stderr.writer(), stderr_config, .err, "line number count exceeded maximum of {}", .{std.math.maxInt(usize)});
		197	std.process.exit(1);
		198	},
		199	error.OutOfMemory => \|e\| return e,
191	};	200	};
		201	defer mapping_results.mappings.deinit(allocator);
		202
		203	const default_code_page = options.default_code_page orelse .windows1252;
		204	const has_disjoint_code_page = hasDisjointCodePage(mapping_results.result, &mapping_results.mappings, default_code_page);
		205
		206	const final_input = try removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings);
192		207
193	var output_file = std.fs.cwd().createFile(options.output_filename, .{}) catch \|err\| {	208	var output_file = std.fs.cwd().createFile(options.output_filename, .{}) catch \|err\| {
194	try error_handler.emitMessage(allocator, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) });	209	try error_handler.emitMessage(allocator, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) });
...	@@ -211,7 +226,8 @@ pub fn main() !void {	...	@@ -211,7 +226,8 @@ pub fn main() !void {
211	.extra_include_paths = options.extra_include_paths.items,	226	.extra_include_paths = options.extra_include_paths.items,
212	.system_include_paths = include_paths,	227	.system_include_paths = include_paths,
213	.default_language_id = options.default_language_id,	228	.default_language_id = options.default_language_id,
214	.default_code_page = options.default_code_page orelse .windows1252,	229	.default_code_page = default_code_page,
		230	.disjoint_code_page = has_disjoint_code_page,
215	.verbose = options.verbose,	231	.verbose = options.verbose,
216	.null_terminate_string_table_strings = options.null_terminate_string_table_strings,	232	.null_terminate_string_table_strings = options.null_terminate_string_table_strings,
217	.max_string_literal_codepoints = options.max_string_literal_codepoints,	233	.max_string_literal_codepoints = options.max_string_literal_codepoints,
...	@@ -513,7 +529,7 @@ fn diagnosticsToErrorBundle(	...	@@ -513,7 +529,7 @@ fn diagnosticsToErrorBundle(
513	};	529	};
514	if (err_details.print_source_line) {	530	if (err_details.print_source_line) {
515	const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);	531	const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);
516	const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len);	532	const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len, source);
517	src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len);	533	src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len);
518	src_loc.span_main = @intCast(visual_info.point_offset);	534	src_loc.span_main = @intCast(visual_info.point_offset);
519	src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len);	535	src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len);

lib/compiler/resinator/parse.zig+174-69

 @@ -4,9 +4,10 @@ const Token = @import("lex.zig").Token;
 const Node = @import("ast.zig").Node;
 const Tree = @import("ast.zig").Tree;
 const CodePageLookup = @import("ast.zig").CodePageLookup;
-const Resource = @import("rc.zig").Resource;
+const ResourceType = @import("rc.zig").ResourceType;
 const Allocator = std.mem.Allocator;
 const ErrorDetails = @import("errors.zig").ErrorDetails;
+const ErrorDetailsWithoutCodePage = @import("errors.zig").ErrorDetailsWithoutCodePage;
 const Diagnostics = @import("errors.zig").Diagnostics;
 const SourceBytes = @import("literals.zig").SourceBytes;
 const Compiler = @import("compile.zig").Compiler;
 @@ -30,6 +31,7 @@ pub const Parser = struct {
     pub const Options = struct {
         warn_instead_of_error_on_invalid_code_page: bool = false,
+        disjoint_code_page: bool = false,
     };
     pub fn init(lexer: *Lexer, options: Options) Parser {
 @@ -47,6 +49,7 @@ pub const Parser = struct {
         diagnostics: *Diagnostics,
         input_code_page_lookup: CodePageLookup,
         output_code_page_lookup: CodePageLookup,
+        warned_about_disjoint_code_page: bool,
     };
     pub fn parse(self: *Self, allocator: Allocator, diagnostics: *Diagnostics) Error!*Tree {
 @@ -61,6 +64,7 @@ pub const Parser = struct {
             .diagnostics = diagnostics,
             .input_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page),
             .output_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page),
+            .warned_about_disjoint_code_page = false,
         };
         const parsed_root = try self.parseRoot();
 @@ -116,7 +120,7 @@ pub const Parser = struct {
             const maybe_common_resource_attribute = try self.lookaheadToken(.normal);
             if (maybe_common_resource_attribute.id == .literal and rc.CommonResourceAttributes.map.has(maybe_common_resource_attribute.slice(self.lexer.buffer))) {
                 try common_resource_attributes.append(self.state.arena, maybe_common_resource_attribute);
-                self.nextToken(.normal) catch unreachable;
+                try self.nextToken(.normal);
             } else {
                 break;
             }
 @@ -130,8 +134,13 @@ pub const Parser = struct {
     /// optional statements (if any). If there are no optional statements, the
     /// current token is unchanged.
     /// The returned slice is allocated by the parser's arena
-    fn parseOptionalStatements(self: *Self, resource: Resource) ![]*Node {
+    fn parseOptionalStatements(self: *Self, resource: ResourceType) ![]*Node {
         var optional_statements: std.ArrayListUnmanaged(*Node) = .empty;
+        const num_statement_types = @typeInfo(rc.OptionalStatements).@"enum".fields.len;
+        var statement_type_has_duplicates = [_]bool{false} ** num_statement_types;
+        var last_statement_per_type = [_]?*Node{null} ** num_statement_types;
         while (true) {
             const lookahead_token = try self.lookaheadToken(.normal);
             if (lookahead_token.id != .literal) break;
 @@ -140,7 +149,13 @@ pub const Parser = struct {
                 .dialog, .dialogex => rc.OptionalStatements.dialog_map.get(slice) orelse break,
                 else => break,
             };
-            self.nextToken(.normal) catch unreachable;
+            try self.nextToken(.normal);
+            const type_i = @intFromEnum(optional_statement_type);
+            if (last_statement_per_type[type_i] != null) {
+                statement_type_has_duplicates[type_i] = true;
+            }
             switch (optional_statement_type) {
                 .language => {
                     const language = try self.parseLanguageStatement();
 @@ -166,7 +181,7 @@ pub const Parser = struct {
                     try self.nextToken(.normal);
                     const value = self.state.token;
                     if (!value.isStringLiteral()) {
-                        return self.addErrorDetailsAndFail(ErrorDetails{
+                        return self.addErrorDetailsAndFail(.{
                             .err = .expected_something_else,
                             .token = value,
                             .extra = .{ .expected_types = .{
 @@ -223,7 +238,7 @@ pub const Parser = struct {
                     try self.nextToken(.normal);
                     const typeface = self.state.token;
                     if (!typeface.isStringLiteral()) {
-                        return self.addErrorDetailsAndFail(ErrorDetails{
+                        return self.addErrorDetailsAndFail(.{
                             .err = .expected_something_else,
                             .token = typeface,
                             .extra = .{ .expected_types = .{
 @@ -272,7 +287,42 @@ pub const Parser = struct {
                     try optional_statements.append(self.state.arena, &node.base);
                 },
             }
+            last_statement_per_type[type_i] = optional_statements.items[optional_statements.items.len - 1];
         }
+        for (optional_statements.items) |optional_statement| {
+            const type_i = type_i: {
+                switch (optional_statement.id) {
+                    .simple_statement => {
+                        const simple_statement: *Node.SimpleStatement = @alignCast(@fieldParentPtr("base", optional_statement));
+                        const statement_identifier = simple_statement.identifier;
+                        const slice = statement_identifier.slice(self.lexer.buffer);
+                        const optional_statement_type = rc.OptionalStatements.map.get(slice) orelse
+                            rc.OptionalStatements.dialog_map.get(slice).?;
+                        break :type_i @intFromEnum(optional_statement_type);
+                    },
+                    .font_statement => {
+                        break :type_i @intFromEnum(rc.OptionalStatements.font);
+                    },
+                    .language_statement => {
+                        break :type_i @intFromEnum(rc.OptionalStatements.language);
+                    },
+                    else => unreachable,
+                }
+            };
+            if (!statement_type_has_duplicates[type_i]) continue;
+            if (optional_statement == last_statement_per_type[type_i].?) continue;
+            try self.addErrorDetails(.{
+                .err = .duplicate_optional_statement_skipped,
+                .type = .warning,
+                .token = optional_statement.getFirstToken(),
+                .token_span_start = optional_statement.getFirstToken(),
+                .token_span_end = optional_statement.getLastToken(),
+            });
+        }
         return optional_statements.toOwnedSlice(self.state.arena);
     }
 @@ -311,12 +361,13 @@ pub const Parser = struct {
                     const maybe_end_token = try self.lookaheadToken(.normal);
                     switch (maybe_end_token.id) {
                         .end => {
-                            self.nextToken(.normal) catch unreachable;
+                            try self.nextToken(.normal);
                             break;
                         },
                         .eof => {
-                            return self.addErrorDetailsAndFail(ErrorDetails{
+                            return self.addErrorDetailsWithCodePageAndFail(.{
                                 .err = .unfinished_string_table_block,
+                                .code_page = self.lexer.current_code_page,
                                 .token = maybe_end_token,
                             });
                         },
 @@ -328,7 +379,7 @@ pub const Parser = struct {
                     try self.nextToken(.normal);
                     if (self.state.token.id != .quoted_ascii_string and self.state.token.id != .quoted_wide_string) {
-                        return self.addErrorDetailsAndFail(ErrorDetails{
+                        return self.addErrorDetailsAndFail(.{
                             .err = .expected_something_else,
                             .token = self.state.token,
                             .extra = .{ .expected_types = .{ .string_literal = true } },
 @@ -345,7 +396,7 @@ pub const Parser = struct {
                 }
                 if (strings.items.len == 0) {
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .expected_token, // TODO: probably a more specific error message
                         .token = self.state.token,
                         .extra = .{ .expected = .number },
 @@ -374,7 +425,12 @@ pub const Parser = struct {
         // of projects. So, we have special compatibility for this particular case.
         const maybe_eof = try self.lookaheadToken(.whitespace_delimiter_only);
         if (maybe_eof.id == .eof) {
-            // TODO: emit warning
+            try self.addErrorDetails(.{
+                .err = .dangling_literal_at_eof,
+                .type = .warning,
+                .token = first_token,
+            });
             var context = try self.state.arena.alloc(Token, 2);
             context[0] = first_token;
             context[1] = maybe_eof;
 @@ -413,12 +469,12 @@ pub const Parser = struct {
             if (maybe_ordinal == null) {
                 const would_be_win32_rc_ordinal = res.NameOrOrdinal.maybeNonAsciiOrdinalFromString(id_bytes);
                 if (would_be_win32_rc_ordinal) |win32_rc_ordinal| {
-                    try self.addErrorDetails(ErrorDetails{
+                    try self.addErrorDetails(.{
                         .err = .id_must_be_ordinal,
                         .token = id_token,
                         .extra = .{ .resource = resource },
                     });
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .win32_non_ascii_ordinal,
                         .token = id_token,
                         .type = .note,
 @@ -426,7 +482,7 @@ pub const Parser = struct {
                         .extra = .{ .number = win32_rc_ordinal.ordinal },
                     });
                 } else {
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .id_must_be_ordinal,
                         .token = id_token,
                         .extra = .{ .resource = resource },
 @@ -451,7 +507,7 @@ pub const Parser = struct {
                     const lookahead = try self.lookaheadToken(.normal);
                     switch (lookahead.id) {
                         .end, .eof => {
-                            self.nextToken(.normal) catch unreachable;
+                            try self.nextToken(.normal);
                             break;
                         },
                         else => {},
 @@ -739,19 +795,19 @@ pub const Parser = struct {
                 const maybe_begin = try self.lookaheadToken(.normal);
                 if (maybe_begin.id == .begin) {
-                    self.nextToken(.normal) catch unreachable;
+                    try self.nextToken(.normal);
                     if (!resource.canUseRawData()) {
-                        try self.addErrorDetails(ErrorDetails{
+                        try self.addErrorDetails(.{
                             .err = .resource_type_cant_use_raw_data,
-                            .token = maybe_begin,
+                            .token = self.state.token,
                             .extra = .{ .resource = resource },
                         });
-                        return self.addErrorDetailsAndFail(ErrorDetails{
+                        return self.addErrorDetailsAndFail(.{
                             .err = .resource_type_cant_use_raw_data,
                             .type = .note,
                             .print_source_line = false,
-                            .token = maybe_begin,
+                            .token = self.state.token,
                         });
                     }
 @@ -802,11 +858,12 @@ pub const Parser = struct {
             const maybe_end_token = try self.lookaheadToken(.normal);
             switch (maybe_end_token.id) {
                 .comma => {
+                    try self.nextToken(.normal);
                     // comma as the first token in a raw data block is an error
                     if (raw_data.items.len == 0) {
-                        return self.addErrorDetailsAndFail(ErrorDetails{
+                        return self.addErrorDetailsAndFail(.{
                             .err = .expected_something_else,
-                            .token = maybe_end_token,
+                            .token = self.state.token,
                             .extra = .{ .expected_types = .{
                                 .number = true,
                                 .number_expression = true,
 @@ -815,16 +872,16 @@ pub const Parser = struct {
                         });
                     }
                     // otherwise just skip over commas
-                    self.nextToken(.normal) catch unreachable;
                     continue;
                 },
                 .end => {
-                    self.nextToken(.normal) catch unreachable;
+                    try self.nextToken(.normal);
                     break;
                 },
                 .eof => {
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsWithCodePageAndFail(.{
                         .err = .unfinished_raw_data_block,
+                        .code_page = self.lexer.current_code_page,
                         .token = maybe_end_token,
                     });
                 },
 @@ -836,10 +893,12 @@ pub const Parser = struct {
             if (expression.isNumberExpression()) {
                 const maybe_close_paren = try self.lookaheadToken(.normal);
                 if (maybe_close_paren.id == .close_paren) {
+                    // advance to ensure that the code page lookup is populated for this token
+                    try self.nextToken(.normal);
                     // <number expression>) is an error
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .expected_token,
-                        .token = maybe_close_paren,
+                        .token = self.state.token,
                         .extra = .{ .expected = .operator },
                     });
                 }
 @@ -852,10 +911,10 @@ pub const Parser = struct {
     /// begin on the next token.
     /// After return, the current token will be the token immediately before the end of the
     /// control statement (or unchanged if the function returns null).
-    fn parseControlStatement(self: *Self, resource: Resource) Error!?*Node {
+    fn parseControlStatement(self: *Self, resource: ResourceType) Error!?*Node {
         const control_token = try self.lookaheadToken(.normal);
         const control = rc.Control.map.get(control_token.slice(self.lexer.buffer)) orelse return null;
-        self.nextToken(.normal) catch unreachable;
+        try self.nextToken(.normal);
         try self.skipAnyCommas();
 @@ -867,7 +926,7 @@ pub const Parser = struct {
                     text = self.state.token;
                 },
                 else => {
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .expected_something_else,
                         .token = self.state.token,
                         .extra = .{ .expected_types = .{
 @@ -920,14 +979,16 @@ pub const Parser = struct {
             // the style parameter.
             const lookahead_token = try self.lookaheadToken(.normal);
             if (lookahead_token.id != .comma and lookahead_token.id != .eof) {
-                try self.addErrorDetails(.{
+                try self.addErrorDetailsWithCodePage(.{
                     .err = .rc_could_miscompile_control_params,
                     .type = .warning,
+                    .code_page = self.lexer.current_code_page,
                     .token = lookahead_token,
                 });
-                try self.addErrorDetails(.{
+                try self.addErrorDetailsWithCodePage(.{
                     .err = .rc_could_miscompile_control_params,
                     .type = .note,
+                    .code_page = self.lexer.current_code_page,
                     .token = style.?.getFirstToken(),
                     .token_span_end = style.?.getLastToken(),
                 });
 @@ -987,7 +1048,7 @@ pub const Parser = struct {
     fn parseToolbarButtonStatement(self: *Self) Error!?*Node {
         const keyword_token = try self.lookaheadToken(.normal);
         const button_type = rc.ToolbarButton.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null;
-        self.nextToken(.normal) catch unreachable;
+        try self.nextToken(.normal);
         switch (button_type) {
             .separator => {
 @@ -1014,10 +1075,10 @@ pub const Parser = struct {
     /// begin on the next token.
     /// After return, the current token will be the token immediately before the end of the
     /// menuitem statement (or unchanged if the function returns null).
-    fn parseMenuItemStatement(self: *Self, resource: Resource, top_level_menu_id_token: Token, nesting_level: u32) Error!?*Node {
+    fn parseMenuItemStatement(self: *Self, resource: ResourceType, top_level_menu_id_token: Token, nesting_level: u32) Error!?*Node {
         const menuitem_token = try self.lookaheadToken(.normal);
         const menuitem = rc.MenuItem.map.get(menuitem_token.slice(self.lexer.buffer)) orelse return null;
-        self.nextToken(.normal) catch unreachable;
+        try self.nextToken(.normal);
         if (nesting_level > max_nested_menu_level) {
             try self.addErrorDetails(.{
 @@ -1050,7 +1111,7 @@ pub const Parser = struct {
                     } else {
                         const text = self.state.token;
                         if (!text.isStringLiteral()) {
-                            return self.addErrorDetailsAndFail(ErrorDetails{
+                            return self.addErrorDetailsAndFail(.{
                                 .err = .expected_something_else,
                                 .token = text,
                                 .extra = .{ .expected_types = .{
 @@ -1070,7 +1131,7 @@ pub const Parser = struct {
                             if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) {
                                 break;
                             }
-                            self.nextToken(.normal) catch unreachable;
+                            try self.nextToken(.normal);
                             try options.append(self.state.arena, option_token);
                             try self.skipAnyCommas();
                         }
 @@ -1089,7 +1150,7 @@ pub const Parser = struct {
                     try self.nextToken(.normal);
                     const text = self.state.token;
                     if (!text.isStringLiteral()) {
-                        return self.addErrorDetailsAndFail(ErrorDetails{
+                        return self.addErrorDetailsAndFail(.{
                             .err = .expected_something_else,
                             .token = text,
                             .extra = .{ .expected_types = .{
 @@ -1105,7 +1166,7 @@ pub const Parser = struct {
                         if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) {
                             break;
                         }
-                        self.nextToken(.normal) catch unreachable;
+                        try self.nextToken(.normal);
                         try options.append(self.state.arena, option_token);
                         try self.skipAnyCommas();
                     }
 @@ -1146,7 +1207,7 @@ pub const Parser = struct {
                 try self.nextToken(.normal);
                 const text = self.state.token;
                 if (!text.isStringLiteral()) {
-                    return self.addErrorDetailsAndFail(ErrorDetails{
+                    return self.addErrorDetailsAndFail(.{
                         .err = .expected_something_else,
                         .token = text,
                         .extra = .{ .expected_types = .{
 @@ -1257,7 +1318,7 @@ pub const Parser = struct {
     fn parseVersionStatement(self: *Self) Error!?*Node {
         const type_token = try self.lookaheadToken(.normal);
         const statement_type = rc.VersionInfo.map.get(type_token.slice(self.lexer.buffer)) orelse return null;
-        self.nextToken(.normal) catch unreachable;
+        try self.nextToken(.normal);
         switch (statement_type) {
             .file_version, .product_version => {
                 var parts_buffer: [4]*Node = undefined;
 @@ -1301,7 +1362,7 @@ pub const Parser = struct {
     fn parseVersionBlockOrValue(self: *Self, top_level_version_id_token: Token, nesting_level: u32) Error!?*Node {
         const keyword_token = try self.lookaheadToken(.normal);
         const keyword = rc.VersionBlock.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null;
-        self.nextToken(.normal) catch unreachable;
+        try self.nextToken(.normal);
         if (nesting_level > max_nested_version_level) {
             try self.addErrorDetails(.{
 @@ -1541,7 +1602,7 @@ pub const Parser = struct {
             }
         };
-        pub fn toErrorDetails(options: ParseExpressionOptions, token: Token) ErrorDetails {
+        pub fn toErrorDetails(options: ParseExpressionOptions, token: Token) ErrorDetailsWithoutCodePage {
             // TODO: expected_types_override interaction with is_known_to_be_number_expression?
             const expected_types = options.expected_types_override orelse ErrorDetails.ExpectedTypes{
                 .number = options.allowed_types.number,
 @@ -1549,7 +1610,7 @@ pub const Parser = struct {
                 .string_literal = options.allowed_types.string and !options.is_known_to_be_number_expression,
                 .literal = options.allowed_types.literal and !options.is_known_to_be_number_expression,
             };
-            return ErrorDetails{
+            return .{
                 .err = .expected_something_else,
                 .token = token,
                 .extra = .{ .expected_types = expected_types },
 @@ -1690,7 +1751,7 @@ pub const Parser = struct {
         try self.addErrorDetails(options.toErrorDetails(self.state.token));
         if (is_close_paren_expression) {
-            try self.addErrorDetails(ErrorDetails{
+            try self.addErrorDetails(.{
                 .err = .close_paren_expression,
                 .type = .note,
                 .token = self.state.token,
 @@ -1698,7 +1759,7 @@ pub const Parser = struct {
             });
         }
         if (is_unary_plus_expression) {
-            try self.addErrorDetails(ErrorDetails{
+            try self.addErrorDetails(.{
                 .err = .unary_plus_expression,
                 .type = .note,
                 .token = self.state.token,
 @@ -1739,7 +1800,7 @@ pub const Parser = struct {
             });
             if (!rhs_node.isNumberExpression()) {
-                return self.addErrorDetailsAndFail(ErrorDetails{
+                return self.addErrorDetailsAndFail(.{
                     .err = .expected_something_else,
                     .token = rhs_node.getFirstToken(),
                     .token_span_end = rhs_node.getLastToken(),
 @@ -1781,16 +1842,39 @@ pub const Parser = struct {
     fn parseOptionalTokenAdvanced(self: *Self, id: Token.Id, comptime method: Lexer.LexMethod) Error!bool {
         const maybe_token = try self.lookaheadToken(method);
         if (maybe_token.id != id) return false;
-        self.nextToken(method) catch unreachable;
+        try self.nextToken(method);
         return true;
     }
-    fn addErrorDetails(self: *Self, details: ErrorDetails) Allocator.Error!void {
+    fn addErrorDetailsWithCodePage(self: *Self, details: ErrorDetails) Allocator.Error!void {
         try self.state.diagnostics.append(details);
     }
-    fn addErrorDetailsAndFail(self: *Self, details: ErrorDetails) Error {
+    fn addErrorDetailsWithCodePageAndFail(self: *Self, details: ErrorDetails) Error {
-        try self.addErrorDetails(details);
+        try self.addErrorDetailsWithCodePage(details);
+        return error.ParseError;
+    }
+    /// Code page is looked up in input_code_page_lookup using the token, meaning the token
+    /// must come from nextToken (i.e. it can't be a lookahead token).
+    fn addErrorDetails(self: *Self, details_without_code_page: ErrorDetailsWithoutCodePage) Allocator.Error!void {
+        const details = ErrorDetails{
+            .err = details_without_code_page.err,
+            .code_page = self.state.input_code_page_lookup.getForToken(details_without_code_page.token),
+            .token = details_without_code_page.token,
+            .token_span_start = details_without_code_page.token_span_start,
+            .token_span_end = details_without_code_page.token_span_end,
+            .type = details_without_code_page.type,
+            .print_source_line = details_without_code_page.print_source_line,
+            .extra = details_without_code_page.extra,
+        };
+        try self.addErrorDetailsWithCodePage(details);
+    }
+    /// Code page is looked up in input_code_page_lookup using the token, meaning the token
+    /// must come from nextToken (i.e. it can't be a lookahead token).
+    fn addErrorDetailsAndFail(self: *Self, details_without_code_page: ErrorDetailsWithoutCodePage) Error {
+        try self.addErrorDetails(details_without_code_page);
         return error.ParseError;
     }
 @@ -1798,35 +1882,34 @@ pub const Parser = struct {
         self.state.token = token: while (true) {
             const token = self.lexer.next(method) catch |err| switch (err) {
                 error.CodePagePragmaInIncludedFile => {
-                    // The Win32 RC compiler silently ignores such `#pragma code_point` directives,
+                    // The Win32 RC compiler silently ignores such `#pragma code_page` directives,
                     // but we want to both ignore them *and* emit a warning
-                    try self.addErrorDetails(.{
+                    var details = self.lexer.getErrorDetails(err);
-                        .err = .code_page_pragma_in_included_file,
+                    details.type = .warning;
-                        .type = .warning,
+                    try self.addErrorDetailsWithCodePage(details);
-                        .token = self.lexer.error_context_token.?,
-                    });
                     continue;
                 },
                 error.CodePagePragmaInvalidCodePage => {
                     var details = self.lexer.getErrorDetails(err);
                     if (!self.options.warn_instead_of_error_on_invalid_code_page) {
-                        return self.addErrorDetailsAndFail(details);
+                        return self.addErrorDetailsWithCodePageAndFail(details);
                     }
                     details.type = .warning;
-                    try self.addErrorDetails(details);
+                    try self.addErrorDetailsWithCodePage(details);
                     continue;
                 },
                 error.InvalidDigitCharacterInNumberLiteral => {
                     const details = self.lexer.getErrorDetails(err);
-                    try self.addErrorDetails(details);
+                    try self.addErrorDetailsWithCodePage(details);
-                    return self.addErrorDetailsAndFail(.{
+                    return self.addErrorDetailsWithCodePageAndFail(.{
                         .err = details.err,
                         .type = .note,
+                        .code_page = self.lexer.current_code_page,
                         .token = details.token,
                         .print_source_line = false,
                     });
                 },
-                else => return self.addErrorDetailsAndFail(self.lexer.getErrorDetails(err)),
+                else => return self.addErrorDetailsWithCodePageAndFail(self.lexer.getErrorDetails(err)),
             };
             break :token token;
         };
 @@ -1835,7 +1918,29 @@ pub const Parser = struct {
         // But only set the output code page to the current code page if we are past the first code_page pragma in the file.
         // Otherwise, we want to fill the lookup using the default code page so that lookups still work for lines that
         // don't have an explicit output code page set.
-        const output_code_page = if (self.lexer.seen_pragma_code_pages > 1) self.lexer.current_code_page else self.state.output_code_page_lookup.default_code_page;
+        const is_disjoint_code_page = self.options.disjoint_code_page and self.lexer.seen_pragma_code_pages == 1;
+        const output_code_page = if (is_disjoint_code_page)
+            self.state.output_code_page_lookup.default_code_page
+        else
+            self.lexer.current_code_page;
+        if (is_disjoint_code_page and !self.state.warned_about_disjoint_code_page) {
+            try self.addErrorDetailsWithCodePage(.{
+                .err = .disjoint_code_page,
+                .type = .warning,
+                .code_page = self.state.input_code_page_lookup.getForLineNum(self.lexer.last_pragma_code_page_token.?.line_number),
+                .token = self.lexer.last_pragma_code_page_token.?,
+            });
+            try self.addErrorDetailsWithCodePage(.{
+                .err = .disjoint_code_page,
+                .type = .note,
+                .code_page = self.state.input_code_page_lookup.getForLineNum(self.lexer.last_pragma_code_page_token.?.line_number),
+                .token = self.lexer.last_pragma_code_page_token.?,
+                .print_source_line = false,
+            });
+            self.state.warned_about_disjoint_code_page = true;
+        }
         try self.state.output_code_page_lookup.setForToken(self.state.token, output_code_page);
     }
 @@ -1846,7 +1951,7 @@ pub const Parser = struct {
                 // Ignore this error and get the next valid token, we'll deal with this
                 // properly when getting the token for real
                 error.CodePagePragmaInIncludedFile => continue,
-                else => return self.addErrorDetailsAndFail(self.state.lookahead_lexer.getErrorDetails(err)),
+                else => return self.addErrorDetailsWithCodePageAndFail(self.state.lookahead_lexer.getErrorDetails(err)),
             };
         };
     }
 @@ -1860,7 +1965,7 @@ pub const Parser = struct {
         switch (self.state.token.id) {
             .literal => {},
             else => {
-                return self.addErrorDetailsAndFail(ErrorDetails{
+                return self.addErrorDetailsAndFail(.{
                     .err = .expected_token,
                     .token = self.state.token,
                     .extra = .{ .expected = .literal },
 @@ -1871,7 +1976,7 @@ pub const Parser = struct {
     fn check(self: *Self, expected_token_id: Token.Id) !void {
         if (self.state.token.id != expected_token_id) {
-            return self.addErrorDetailsAndFail(ErrorDetails{
+            return self.addErrorDetailsAndFail(.{
                 .err = .expected_token,
                 .token = self.state.token,
                 .extra = .{ .expected = expected_token_id },
 @@ -1879,14 +1984,14 @@ pub const Parser = struct {
         }
     }
-    fn checkResource(self: *Self) !Resource {
+    fn checkResource(self: *Self) !ResourceType {
         switch (self.state.token.id) {
-            .literal => return Resource.fromString(.{
+            .literal => return ResourceType.fromString(.{
                 .slice = self.state.token.slice(self.lexer.buffer),
                 .code_page = self.lexer.current_code_page,
             }),
             else => {
-                return self.addErrorDetailsAndFail(ErrorDetails{
+                return self.addErrorDetailsAndFail(.{
                     .err = .expected_token,
                     .token = self.state.token,
                     .extra = .{ .expected = .literal },

lib/compiler/resinator/preprocess.zig+1

...	@@ -96,6 +96,7 @@ pub fn appendAroArgs(arena: Allocator, argv: *std.ArrayList([]const u8), options	...	@@ -96,6 +96,7 @@ pub fn appendAroArgs(arena: Allocator, argv: *std.ArrayList([]const u8), options
96	"--emulate=msvc",	96	"--emulate=msvc",
97	"-nostdinc",	97	"-nostdinc",
98	"-DRC_INVOKED",	98	"-DRC_INVOKED",
		99	"-D_WIN32", // undocumented, but defined by default
99	});	100	});
100	for (options.extra_include_paths.items) \|extra_include_path\| {	101	for (options.extra_include_paths.items) \|extra_include_path\| {
101	try argv.append("-I");	102	try argv.append("-I");

lib/compiler/resinator/rc.zig+7-7

...	@@ -5,7 +5,7 @@ const SourceBytes = @import("literals.zig").SourceBytes;	...	@@ -5,7 +5,7 @@ const SourceBytes = @import("literals.zig").SourceBytes;
5		5
6	// https://learn.microsoft.com/en-us/windows/win32/menurc/about-resource-files	6	// https://learn.microsoft.com/en-us/windows/win32/menurc/about-resource-files
7		7
8	pub const Resource = enum {	8	pub const ResourceType = enum {
9	accelerators,	9	accelerators,
10	bitmap,	10	bitmap,
11	cursor,	11	cursor,
...	@@ -48,7 +48,7 @@ pub const Resource = enum {	...	@@ -48,7 +48,7 @@ pub const Resource = enum {
48	manifest_num,	48	manifest_num,
49		49
50	const map = std.StaticStringMapWithEql(	50	const map = std.StaticStringMapWithEql(
51	Resource,	51	ResourceType,
52	std.static_string_map.eqlAsciiIgnoreCase,	52	std.static_string_map.eqlAsciiIgnoreCase,
53	).initComptime(.{	53	).initComptime(.{
54	.{ "ACCELERATORS", .accelerators },	54	.{ "ACCELERATORS", .accelerators },
...	@@ -72,7 +72,7 @@ pub const Resource = enum {	...	@@ -72,7 +72,7 @@ pub const Resource = enum {
72	.{ "VXD", .vxd },	72	.{ "VXD", .vxd },
73	});	73	});
74		74
75	pub fn fromString(bytes: SourceBytes) Resource {	75	pub fn fromString(bytes: SourceBytes) ResourceType {
76	const maybe_ordinal = res.NameOrOrdinal.maybeOrdinalFromString(bytes);	76	const maybe_ordinal = res.NameOrOrdinal.maybeOrdinalFromString(bytes);
77	if (maybe_ordinal) \|ordinal\| {	77	if (maybe_ordinal) \|ordinal\| {
78	if (ordinal.ordinal >= 256) return .user_defined;	78	if (ordinal.ordinal >= 256) return .user_defined;
...	@@ -81,8 +81,8 @@ pub const Resource = enum {	...	@@ -81,8 +81,8 @@ pub const Resource = enum {
81	return map.get(bytes.slice) orelse .user_defined;	81	return map.get(bytes.slice) orelse .user_defined;
82	}	82	}
83		83
84	// TODO: Some comptime validation that RT <-> Resource conversion is synced?	84	// TODO: Some comptime validation that RT <-> ResourceType conversion is synced?
85	pub fn fromRT(rt: res.RT) Resource {	85	pub fn fromRT(rt: res.RT) ResourceType {
86	return switch (rt) {	86	return switch (rt) {
87	.ACCELERATOR => .accelerators,	87	.ACCELERATOR => .accelerators,
88	.ANICURSOR => .anicursor_num,	88	.ANICURSOR => .anicursor_num,
...	@@ -111,7 +111,7 @@ pub const Resource = enum {	...	@@ -111,7 +111,7 @@ pub const Resource = enum {
111	};	111	};
112	}	112	}
113		113
114	pub fn canUseRawData(resource: Resource) bool {	114	pub fn canUseRawData(resource: ResourceType) bool {
115	return switch (resource) {	115	return switch (resource) {
116	.user_defined,	116	.user_defined,
117	.html,	117	.html,
...	@@ -125,7 +125,7 @@ pub const Resource = enum {	...	@@ -125,7 +125,7 @@ pub const Resource = enum {
125	};	125	};
126	}	126	}
127		127
128	pub fn nameForErrorDisplay(resource: Resource) []const u8 {	128	pub fn nameForErrorDisplay(resource: ResourceType) []const u8 {
129	return switch (resource) {	129	return switch (resource) {
130	// zig fmt: off	130	// zig fmt: off
131	.accelerators, .bitmap, .cursor, .dialog, .dialogex, .dlginclude, .dlginit, .font,	131	.accelerators, .bitmap, .cursor, .dialog, .dialogex, .dlginclude, .dlginit, .font,

lib/compiler/resinator/res.zig+175-38

 @@ -1,10 +1,10 @@
 const std = @import("std");
 const rc = @import("rc.zig");
-const Resource = rc.Resource;
+const ResourceType = rc.ResourceType;
 const CommonResourceAttributes = rc.CommonResourceAttributes;
 const Allocator = std.mem.Allocator;
 const windows1252 = @import("windows1252.zig");
-const CodePage = @import("code_pages.zig").CodePage;
+const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
 const literals = @import("literals.zig");
 const SourceBytes = literals.SourceBytes;
 const Codepoint = @import("code_pages.zig").Codepoint;
 @@ -40,7 +40,7 @@ pub const RT = enum(u8) {
     /// Returns null if the resource type is user-defined
     /// Asserts that the resource is not `stringtable`
-    pub fn fromResource(resource: Resource) ?RT {
+    pub fn fromResource(resource: ResourceType) ?RT {
         return switch (resource) {
             .accelerators => .ACCELERATOR,
             .bitmap => .BITMAP,
 @@ -162,6 +162,27 @@ pub const Language = packed struct(u16) {
     pub fn asInt(self: Language) u16 {
         return @bitCast(self);
     }
+    pub fn format(
+        language: Language,
+        comptime fmt: []const u8,
+        options: std.fmt.FormatOptions,
+        out_stream: anytype,
+    ) !void {
+        _ = fmt;
+        _ = options;
+        const language_id = language.asInt();
+        const language_name = language_name: {
+            if (std.meta.intToEnum(lang.LanguageId, language_id)) |lang_enum_val| {
+                break :language_name @tagName(lang_enum_val);
+            } else |_| {}
+            if (language_id == lang.LOCALE_CUSTOM_UNSPECIFIED) {
+                break :language_name "LOCALE_CUSTOM_UNSPECIFIED";
+            }
+            break :language_name "<UNKNOWN>";
+        };
+        try out_stream.print("{s} (0x{X})", .{ language_name, language_id });
+    }
 };
 /// https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-dlgitemtemplate#remarks
 @@ -423,6 +444,50 @@ pub const NameOrOrdinal = union(enum) {
             .name => return null,
         }
     }
+    pub fn format(
+        self: NameOrOrdinal,
+        comptime fmt: []const u8,
+        options: std.fmt.FormatOptions,
+        out_stream: anytype,
+    ) !void {
+        _ = fmt;
+        _ = options;
+        switch (self) {
+            .name => |name| {
+                try out_stream.print("{s}", .{std.unicode.fmtUtf16Le(name)});
+            },
+            .ordinal => |ordinal| {
+                try out_stream.print("{d}", .{ordinal});
+            },
+        }
+    }
+    fn formatResourceType(
+        self: NameOrOrdinal,
+        comptime fmt: []const u8,
+        options: std.fmt.FormatOptions,
+        out_stream: anytype,
+    ) !void {
+        _ = fmt;
+        _ = options;
+        switch (self) {
+            .name => |name| {
+                try out_stream.print("{s}", .{std.unicode.fmtUtf16Le(name)});
+            },
+            .ordinal => |ordinal| {
+                if (std.enums.tagName(RT, @enumFromInt(ordinal))) |predefined_type_name| {
+                    try out_stream.print("{s}", .{predefined_type_name});
+                } else {
+                    try out_stream.print("{d}", .{ordinal});
+                }
+            },
+        }
+    }
+    pub fn fmtResourceType(type_value: NameOrOrdinal) std.fmt.Formatter(formatResourceType) {
+        return .{ .data = type_value };
+    }
 };
 fn expectNameOrOrdinal(expected: NameOrOrdinal, actual: NameOrOrdinal) !void {
 @@ -603,12 +668,33 @@ pub const AcceleratorModifiers = struct {
 const AcceleratorKeyCodepointTranslator = struct {
     string_type: literals.StringType,
+    output_code_page: SupportedCodePage,
     pub fn translate(self: @This(), maybe_parsed: ?literals.IterativeStringParser.ParsedCodepoint) ?u21 {
         const parsed = maybe_parsed orelse return null;
         if (parsed.codepoint == Codepoint.invalid) return 0xFFFD;
-        if (parsed.from_escaped_integer and self.string_type == .ascii) {
+        if (parsed.from_escaped_integer) {
-            return windows1252.toCodepoint(@truncate(parsed.codepoint));
+            switch (self.string_type) {
+                .ascii => {
+                    const truncated: u8 = @truncate(parsed.codepoint);
+                    switch (self.output_code_page) {
+                        .utf8 => switch (truncated) {
+                            0...0x7F => return truncated,
+                            else => return 0xFFFD,
+                        },
+                        .windows1252 => return windows1252.toCodepoint(truncated),
+                    }
+                },
+                .wide => {
+                    const truncated: u16 = @truncate(parsed.codepoint);
+                    return truncated;
+                },
+            }
+        }
+        if (parsed.escaped_surrogate_pair) {
+            // The codepoint of only the low surrogate
+            const low = @as(u16, @intCast(parsed.codepoint & 0x3FF)) + 0xDC00;
+            return low;
         }
         return parsed.codepoint;
     }
 @@ -623,14 +709,17 @@ pub fn parseAcceleratorKeyString(bytes: SourceBytes, is_virt: bool, options: lit
     }
     var parser = literals.IterativeStringParser.init(bytes, options);
-    var translator = AcceleratorKeyCodepointTranslator{ .string_type = parser.declared_string_type };
+    var translator = AcceleratorKeyCodepointTranslator{
+        .string_type = parser.declared_string_type,
+        .output_code_page = options.output_code_page,
+    };
     const first_codepoint = translator.translate(try parser.next()) orelse return error.EmptyAccelerator;
     // 0 is treated as a terminator, so this is equivalent to an empty string
     if (first_codepoint == 0) return error.EmptyAccelerator;
     if (first_codepoint == '^') {
-        // Note: Emitting this warning unconditonally whenever ^ is the first character
+        // Note: Emitting this warning unconditionally whenever ^ is the first character
         //       matches the Win32 RC behavior, but it's questionable whether or not
         //       the warning should be emitted for ^^ since that results in the ASCII
         //       character ^ being written to the .res.
 @@ -638,11 +727,18 @@ pub fn parseAcceleratorKeyString(bytes: SourceBytes, is_virt: bool, options: lit
             try options.diagnostics.?.diagnostics.append(.{
                 .err = .ascii_character_not_equivalent_to_virtual_key_code,
                 .type = .warning,
+                .code_page = bytes.code_page,
                 .token = options.diagnostics.?.token,
             });
         }
         const c = translator.translate(try parser.next()) orelse return error.InvalidControlCharacter;
+        const third_codepoint = translator.translate(try parser.next());
+        // 0 is treated as a terminator, so a 0 in the third position is fine but
+        // anything else is too many codepoints for an accelerator
+        if (third_codepoint != null and third_codepoint.? != 0) return error.InvalidControlCharacter;
         switch (c) {
             '^' => return '^', // special case
             'a'...'z', 'A'...'Z' => return std.ascii.toUpper(@intCast(c)) - 0x40,
 @@ -699,44 +795,44 @@ test "accelerator keys" {
     try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString(
         .{ .slice = "\"^a\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString(
         .{ .slice = "\"^A\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 26), try parseAcceleratorKeyString(
         .{ .slice = "\"^Z\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, '^'), try parseAcceleratorKeyString(
         .{ .slice = "\"^^\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString(
         .{ .slice = "\"a\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0x6162), try parseAcceleratorKeyString(
         .{ .slice = "\"ab\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 'C'), try parseAcceleratorKeyString(
         .{ .slice = "\"c\"", .code_page = .windows1252 },
         true,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0x6363), try parseAcceleratorKeyString(
         .{ .slice = "\"cc\"", .code_page = .windows1252 },
         true,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // \x00 or any escape that evaluates to zero acts as a terminator, everything past it
 @@ -744,93 +840,93 @@ test "accelerator keys" {
     try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString(
         .{ .slice = "\"a\\0bcdef\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // \x80 is € in Windows-1252, which is Unicode codepoint 20AC
     try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
         .{ .slice = "\"\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // This depends on the code page, though, with codepage 65001, \x80
     // on its own is invalid UTF-8 so it gets converted to the replacement character
     try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
         .{ .slice = "\"\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
         .{ .slice = "\"\x80\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // This also behaves the same with escaped characters
     try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
         .{ .slice = "\"\\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // Even with utf8 code page
     try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
         .{ .slice = "\"\\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
         .{ .slice = "\"\\x80\\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // Wide string with the actual characters behaves like the ASCII string version
     try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
         .{ .slice = "L\"\x80\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // But wide string with escapes behaves differently
     try std.testing.expectEqual(@as(u16, 0x8080), try parseAcceleratorKeyString(
         .{ .slice = "L\"\\x80\\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // and invalid escapes within wide strings get skipped
     try std.testing.expectEqual(@as(u16, 'z'), try parseAcceleratorKeyString(
         .{ .slice = "L\"\\Hz\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // any non-A-Z codepoints are illegal
     try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
         .{ .slice = "\"^\x83\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
         .{ .slice = "\"^1\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectError(error.InvalidControlCharacter, parseAcceleratorKeyString(
         .{ .slice = "\"^\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectError(error.EmptyAccelerator, parseAcceleratorKeyString(
         .{ .slice = "\"\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
         .{ .slice = "\"hello\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
         .{ .slice = "\"^\x80\"", .code_page = .windows1252 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // Invalid UTF-8 gets converted to 0xFFFD, multiple invalids get shifted and added together
 @@ -838,40 +934,81 @@ test "accelerator keys" {
     try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString(
         .{ .slice = "\"\x80\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString(
         .{ .slice = "L\"\x80\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // Codepoints >= 0x10000
     try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString(
         .{ .slice = "\"\xF0\x90\x84\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString(
         .{ .slice = "L\"\xF0\x90\x84\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectEqual(@as(u16, 0x9C01), try parseAcceleratorKeyString(
         .{ .slice = "\"\xF4\x80\x80\x81\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     // anything before or after a codepoint >= 0x10000 causes an error
     try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
         .{ .slice = "\"a\xF0\x90\x80\x80\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
     ));
     try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
         .{ .slice = "\"\xF0\x90\x80\x80a\"", .code_page = .utf8 },
         false,
-        .{},
+        .{ .output_code_page = .windows1252 },
+    ));
+    // Misc special cases
+    try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
+        .{ .slice = "\"\\777\"", .code_page = .utf8 },
+        false,
+        .{ .output_code_page = .utf8 },
+    ));
+    try std.testing.expectEqual(@as(u16, 0xFFFF), try parseAcceleratorKeyString(
+        .{ .slice = "L\"\\7777777\"", .code_page = .utf8 },
+        false,
+        .{ .output_code_page = .utf8 },
+    ));
+    try std.testing.expectEqual(@as(u16, 0x01), try parseAcceleratorKeyString(
+        .{ .slice = "L\"\\200001\"", .code_page = .utf8 },
+        false,
+        .{ .output_code_page = .utf8 },
+    ));
+    // Escape of a codepoint >= 0x10000 omits the high surrogate pair
+    try std.testing.expectEqual(@as(u16, 0xDF48), try parseAcceleratorKeyString(
+        .{ .slice = "L\"\\𐍈\"", .code_page = .utf8 },
+        false,
+        .{ .output_code_page = .utf8 },
+    ));
+    // Invalid escape code is skipped, allows for 2 codepoints afterwards
+    try std.testing.expectEqual(@as(u16, 0x7878), try parseAcceleratorKeyString(
+        .{ .slice = "L\"\\kxx\"", .code_page = .utf8 },
+        false,
+        .{ .output_code_page = .utf8 },
+    ));
+    // Escape of a codepoint >= 0x10000 allows for a codepoint afterwards
+    try std.testing.expectEqual(@as(u16, 0x4878), try parseAcceleratorKeyString(
+        .{ .slice = "L\"\\𐍈x\"", .code_page = .utf8 },
+        false,
+        .{ .output_code_page = .utf8 },
+    ));
+    // Input code page of 1252, output code page of utf-8
+    try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
+        .{ .slice = "\"\\270\"", .code_page = .windows1252 },
+        false,
+        .{ .output_code_page = .utf8 },
     ));
 }

lib/compiler/resinator/source_mapping.zig+438-41

...	@@ -38,7 +38,7 @@ pub const ParseAndRemoveLineCommandsOptions = struct {	...	@@ -38,7 +38,7 @@ pub const ParseAndRemoveLineCommandsOptions = struct {
38	///	38	///
39	/// If `options.initial_filename` is provided, that filename is guaranteed to be	39	/// If `options.initial_filename` is provided, that filename is guaranteed to be
40	/// within the `mappings.files` table and `root_filename_offset` will be set appropriately.	40	/// within the `mappings.files` table and `root_filename_offset` will be set appropriately.
41	pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) !ParseLineCommandsResult {	41	pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) error{ OutOfMemory, InvalidLineCommand, LineNumberOverflow }!ParseLineCommandsResult {
42	var parse_result = ParseLineCommandsResult{	42	var parse_result = ParseLineCommandsResult{
43	.result = undefined,	43	.result = undefined,
44	.mappings = .{},	44	.mappings = .{},
...	@@ -53,12 +53,41 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:	...	@@ -53,12 +53,41 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
53	parse_result.mappings.root_filename_offset = try parse_result.mappings.files.put(allocator, initial_filename);	53	parse_result.mappings.root_filename_offset = try parse_result.mappings.files.put(allocator, initial_filename);
54	}	54	}
55		55
		56	// This implementation attempts to be comment and string aware in order
		57	// to avoid errant #line <num> "<filename>" within multiline comments
		58	// leading to problems in the source mapping after comments are removed,
		59	// but it is not a perfect implementation (intentionally).
		60	//
		61	// The current implementation does not handle cases like
		62	// /* foo */ #line ...
		63	// #line ... // foo
		64	// #line ... /* foo ...
		65	// etc
		66	//
		67	// (the first example will not be recognized as a #line command, the second
		68	// and third will error with InvalidLineCommand)
		69	//
		70	// This is fine, though, since #line commands are generated by the
		71	// preprocessor so in normal circumstances they will be well-formed and
		72	// consistent. The only realistic way the imperfect implementation could
		73	// affect a 'real' use-case would be someone taking the output of a
		74	// preprocessor, editing it manually to add comments before/after #line
		75	// commands, and then running it through resinator with /:no-preprocess.
		76
56	std.debug.assert(buf.len >= source.len);	77	std.debug.assert(buf.len >= source.len);
57	var result = UncheckedSliceWriter{ .slice = buf };	78	var result = UncheckedSliceWriter{ .slice = buf };
58	const State = enum {	79	const State = enum {
59	line_start,	80	line_start,
60	preprocessor,	81	preprocessor,
61	non_preprocessor,	82	non_preprocessor,
		83	forward_slash,
		84	line_comment,
		85	multiline_comment,
		86	multiline_comment_end,
		87	single_quoted,
		88	single_quoted_escape,
		89	double_quoted,
		90	double_quoted_escape,
62	};	91	};
63	var state: State = .line_start;	92	var state: State = .line_start;
64	var index: usize = 0;	93	var index: usize = 0;
...	@@ -66,8 +95,8 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:	...	@@ -66,8 +95,8 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
66	var preprocessor_start: usize = 0;	95	var preprocessor_start: usize = 0;
67	var line_number: usize = 1;	96	var line_number: usize = 1;
68	while (index < source.len) : (index += 1) {	97	while (index < source.len) : (index += 1) {
69	const c = source[index];	98	var c = source[index];
70	switch (state) {	99	state: switch (state) {
71	.line_start => switch (c) {	100	.line_start => switch (c) {
72	'#' => {	101	'#' => {
73	preprocessor_start = index;	102	preprocessor_start = index;
...	@@ -93,6 +122,27 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:	...	@@ -93,6 +122,27 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
93	pending_start = index;	122	pending_start = index;
94	}	123	}
95	},	124	},
		125	'/' => {
		126	if (!current_mapping.ignore_contents) {
		127	result.writeSlice(source[pending_start orelse index .. index + 1]);
		128	pending_start = null;
		129	}
		130	state = .forward_slash;
		131	},
		132	'\'' => {
		133	if (!current_mapping.ignore_contents) {
		134	result.writeSlice(source[pending_start orelse index .. index + 1]);
		135	pending_start = null;
		136	}
		137	state = .single_quoted;
		138	},
		139	'"' => {
		140	if (!current_mapping.ignore_contents) {
		141	result.writeSlice(source[pending_start orelse index .. index + 1]);
		142	pending_start = null;
		143	}
		144	state = .double_quoted;
		145	},
96	else => {	146	else => {
97	state = .non_preprocessor;	147	state = .non_preprocessor;
98	if (pending_start != null) {	148	if (pending_start != null) {
...	@@ -107,25 +157,246 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:	...	@@ -107,25 +157,246 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
107	}	157	}
108	},	158	},
109	},	159	},
		160	.forward_slash => switch (c) {
		161	'\r', '\n' => {
		162	const is_crlf = formsLineEndingPair(source, c, index + 1);
		163	if (!current_mapping.ignore_contents) {
		164	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		165
		166	result.write(c);
		167	if (is_crlf) result.write(source[index + 1]);
		168	line_number += 1;
		169	}
		170	if (is_crlf) index += 1;
		171	state = .line_start;
		172	pending_start = null;
		173	},
		174	'/' => {
		175	if (!current_mapping.ignore_contents) {
		176	result.write(c);
		177	}
		178	state = .line_comment;
		179	},
		180	'*' => {
		181	if (!current_mapping.ignore_contents) {
		182	result.write(c);
		183	}
		184	state = .multiline_comment;
		185	},
		186	else => {
		187	if (!current_mapping.ignore_contents) {
		188	result.write(c);
		189	}
		190	state = .non_preprocessor;
		191	},
		192	},
		193	.line_comment => switch (c) {
		194	'\r', '\n' => {
		195	const is_crlf = formsLineEndingPair(source, c, index + 1);
		196	if (!current_mapping.ignore_contents) {
		197	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		198
		199	result.write(c);
		200	if (is_crlf) result.write(source[index + 1]);
		201	line_number += 1;
		202	}
		203	if (is_crlf) index += 1;
		204	state = .line_start;
		205	pending_start = null;
		206	},
		207	else => {
		208	if (!current_mapping.ignore_contents) {
		209	result.write(c);
		210	}
		211	},
		212	},
		213	.multiline_comment => switch (c) {
		214	'\r', '\n' => {
		215	const is_crlf = formsLineEndingPair(source, c, index + 1);
		216	if (!current_mapping.ignore_contents) {
		217	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		218
		219	result.write(c);
		220	if (is_crlf) result.write(source[index + 1]);
		221	line_number += 1;
		222	}
		223	if (is_crlf) index += 1;
		224	pending_start = null;
		225	},
		226	'*' => {
		227	if (!current_mapping.ignore_contents) {
		228	result.write(c);
		229	}
		230	state = .multiline_comment_end;
		231	},
		232	else => {
		233	if (!current_mapping.ignore_contents) {
		234	result.write(c);
		235	}
		236	},
		237	},
		238	.multiline_comment_end => switch (c) {
		239	'\r', '\n' => {
		240	const is_crlf = formsLineEndingPair(source, c, index + 1);
		241	if (!current_mapping.ignore_contents) {
		242	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		243
		244	result.write(c);
		245	if (is_crlf) result.write(source[index + 1]);
		246	line_number += 1;
		247	}
		248	if (is_crlf) index += 1;
		249	state = .multiline_comment;
		250	pending_start = null;
		251	},
		252	'/' => {
		253	if (!current_mapping.ignore_contents) {
		254	result.write(c);
		255	}
		256	state = .non_preprocessor;
		257	},
		258	'*' => {
		259	if (!current_mapping.ignore_contents) {
		260	result.write(c);
		261	}
		262	// stay in multiline_comment_end state
		263	},
		264	else => {
		265	if (!current_mapping.ignore_contents) {
		266	result.write(c);
		267	}
		268	state = .multiline_comment;
		269	},
		270	},
		271	.single_quoted => switch (c) {
		272	'\r', '\n' => {
		273	const is_crlf = formsLineEndingPair(source, c, index + 1);
		274	if (!current_mapping.ignore_contents) {
		275	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		276
		277	result.write(c);
		278	if (is_crlf) result.write(source[index + 1]);
		279	line_number += 1;
		280	}
		281	if (is_crlf) index += 1;
		282	state = .line_start;
		283	pending_start = null;
		284	},
		285	'\\' => {
		286	if (!current_mapping.ignore_contents) {
		287	result.write(c);
		288	}
		289	state = .single_quoted_escape;
		290	},
		291	'\'' => {
		292	if (!current_mapping.ignore_contents) {
		293	result.write(c);
		294	}
		295	state = .non_preprocessor;
		296	},
		297	else => {
		298	if (!current_mapping.ignore_contents) {
		299	result.write(c);
		300	}
		301	},
		302	},
		303	.single_quoted_escape => switch (c) {
		304	'\r', '\n' => {
		305	const is_crlf = formsLineEndingPair(source, c, index + 1);
		306	if (!current_mapping.ignore_contents) {
		307	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		308
		309	result.write(c);
		310	if (is_crlf) result.write(source[index + 1]);
		311	line_number += 1;
		312	}
		313	if (is_crlf) index += 1;
		314	state = .line_start;
		315	pending_start = null;
		316	},
		317	else => {
		318	if (!current_mapping.ignore_contents) {
		319	result.write(c);
		320	}
		321	state = .single_quoted;
		322	},
		323	},
		324	.double_quoted => switch (c) {
		325	'\r', '\n' => {
		326	const is_crlf = formsLineEndingPair(source, c, index + 1);
		327	if (!current_mapping.ignore_contents) {
		328	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		329
		330	result.write(c);
		331	if (is_crlf) result.write(source[index + 1]);
		332	line_number += 1;
		333	}
		334	if (is_crlf) index += 1;
		335	state = .line_start;
		336	pending_start = null;
		337	},
		338	'\\' => {
		339	if (!current_mapping.ignore_contents) {
		340	result.write(c);
		341	}
		342	state = .double_quoted_escape;
		343	},
		344	'"' => {
		345	if (!current_mapping.ignore_contents) {
		346	result.write(c);
		347	}
		348	state = .non_preprocessor;
		349	},
		350	else => {
		351	if (!current_mapping.ignore_contents) {
		352	result.write(c);
		353	}
		354	},
		355	},
		356	.double_quoted_escape => switch (c) {
		357	'\r', '\n' => {
		358	const is_crlf = formsLineEndingPair(source, c, index + 1);
		359	if (!current_mapping.ignore_contents) {
		360	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
		361
		362	result.write(c);
		363	if (is_crlf) result.write(source[index + 1]);
		364	line_number += 1;
		365	}
		366	if (is_crlf) index += 1;
		367	state = .line_start;
		368	pending_start = null;
		369	},
		370	else => {
		371	if (!current_mapping.ignore_contents) {
		372	result.write(c);
		373	}
		374	state = .double_quoted;
		375	},
		376	},
110	.preprocessor => switch (c) {	377	.preprocessor => switch (c) {
111	'\r', '\n' => {	378	'\r', '\n' => {
112	// Now that we have the full line we can decide what to do with it	379	// Now that we have the full line we can decide what to do with it
113	const preprocessor_str = source[preprocessor_start..index];	380	const preprocessor_str = source[preprocessor_start..index];
114	const is_crlf = formsLineEndingPair(source, c, index + 1);
115	if (std.mem.startsWith(u8, preprocessor_str, "#line")) {	381	if (std.mem.startsWith(u8, preprocessor_str, "#line")) {
116	try handleLineCommand(allocator, preprocessor_str, &current_mapping);	382	try handleLineCommand(allocator, preprocessor_str, &current_mapping);
		383	const is_crlf = formsLineEndingPair(source, c, index + 1);
		384	if (is_crlf) index += 1;
		385	state = .line_start;
		386	pending_start = null;
117	} else {	387	} else {
118	if (!current_mapping.ignore_contents) {	388	// Backtrack and reparse the line in the non_preprocessor state,
119	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);	389	// since it's possible that this line contains a multiline comment
120		390	// start, etc.
121	const line_ending_len: usize = if (is_crlf) 2 else 1;	391	state = .non_preprocessor;
122	result.writeSlice(source[pending_start.? .. index + line_ending_len]);	392	index = pending_start.?;
123	line_number += 1;	393	pending_start = null;
124	}	394	// TODO: This is a hacky way to implement this, c needs to be
		395	// updated since we're using continue :state here
		396	c = source[index];
		397	// continue to avoid the index += 1 of the while loop
		398	continue :state .non_preprocessor;
125	}	399	}
126	if (is_crlf) index += 1;
127	state = .line_start;
128	pending_start = null;
129	},	400	},
130	else => {},	401	else => {},
131	},	402	},
...	@@ -143,6 +414,24 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:	...	@@ -143,6 +414,24 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
143	state = .line_start;	414	state = .line_start;
144	pending_start = null;	415	pending_start = null;
145	},	416	},
		417	'/' => {
		418	if (!current_mapping.ignore_contents) {
		419	result.write(c);
		420	}
		421	state = .forward_slash;
		422	},
		423	'\'' => {
		424	if (!current_mapping.ignore_contents) {
		425	result.write(c);
		426	}
		427	state = .single_quoted;
		428	},
		429	'"' => {
		430	if (!current_mapping.ignore_contents) {
		431	result.write(c);
		432	}
		433	state = .double_quoted;
		434	},
146	else => {	435	else => {
147	if (!current_mapping.ignore_contents) {	436	if (!current_mapping.ignore_contents) {
148	result.write(c);	437	result.write(c);
...	@@ -153,7 +442,16 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:	...	@@ -153,7 +442,16 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
153	} else {	442	} else {
154	switch (state) {	443	switch (state) {
155	.line_start => {},	444	.line_start => {},
156	.non_preprocessor => {	445	.forward_slash,
		446	.line_comment,
		447	.multiline_comment,
		448	.multiline_comment_end,
		449	.single_quoted,
		450	.single_quoted_escape,
		451	.double_quoted,
		452	.double_quoted_escape,
		453	.non_preprocessor,
		454	=> {
157	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);	455	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
158	},	456	},
159	.preprocessor => {	457	.preprocessor => {
...	@@ -207,34 +505,40 @@ pub fn handleLineEnd(allocator: Allocator, post_processed_line_number: usize, ma	...	@@ -207,34 +505,40 @@ pub fn handleLineEnd(allocator: Allocator, post_processed_line_number: usize, ma
207		505
208	try mapping.set(post_processed_line_number, current_mapping.line_num, filename_offset);	506	try mapping.set(post_processed_line_number, current_mapping.line_num, filename_offset);
209		507
210	current_mapping.line_num += 1;	508	current_mapping.line_num = std.math.add(usize, current_mapping.line_num, 1) catch return error.LineNumberOverflow;
211	current_mapping.pending = false;	509	current_mapping.pending = false;
212	}	510	}
213		511
214	// TODO: Might want to provide diagnostics on invalid line commands instead of just returning	512	// TODO: Might want to provide diagnostics on invalid line commands instead of just returning
215	pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{OutOfMemory}!void {	513	pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{ OutOfMemory, InvalidLineCommand }!void {
216	// TODO: Are there other whitespace characters that should be included?	514	// TODO: Are there other whitespace characters that should be included?
217	var tokenizer = std.mem.tokenizeAny(u8, line_command, " \t");	515	var tokenizer = std.mem.tokenizeAny(u8, line_command, " \t");
218	const line_directive = tokenizer.next() orelse return; // #line	516	const line_directive = tokenizer.next() orelse return error.InvalidLineCommand; // #line
219	if (!std.mem.eql(u8, line_directive, "#line")) return;	517	if (!std.mem.eql(u8, line_directive, "#line")) return error.InvalidLineCommand;
220	const linenum_str = tokenizer.next() orelse return;	518	const linenum_str = tokenizer.next() orelse return error.InvalidLineCommand;
221	const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return;	519	const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return error.InvalidLineCommand;
		520	if (linenum == 0) return error.InvalidLineCommand;
222		521
223	var filename_literal = tokenizer.rest();	522	var filename_literal = tokenizer.rest();
224	while (filename_literal.len > 0 and std.ascii.isWhitespace(filename_literal[filename_literal.len - 1])) {	523	while (filename_literal.len > 0 and std.ascii.isWhitespace(filename_literal[filename_literal.len - 1])) {
225	filename_literal.len -= 1;	524	filename_literal.len -= 1;
226	}	525	}
227	if (filename_literal.len < 2) return;	526	if (filename_literal.len < 2) return error.InvalidLineCommand;
228	const is_quoted = filename_literal[0] == '"' and filename_literal[filename_literal.len - 1] == '"';	527	const is_quoted = filename_literal[0] == '"' and filename_literal[filename_literal.len - 1] == '"';
229	if (!is_quoted) return;	528	if (!is_quoted) return error.InvalidLineCommand;
230	const filename = parseFilename(allocator, filename_literal[1 .. filename_literal.len - 1]) catch \|err\| switch (err) {	529	const unquoted_filename = filename_literal[1 .. filename_literal.len - 1];
		530
		531	// Ignore <builtin> and <command line>
		532	if (std.mem.eql(u8, unquoted_filename, "<builtin>") or std.mem.eql(u8, unquoted_filename, "<command line>")) return;
		533
		534	const filename = parseFilename(allocator, unquoted_filename) catch \|err\| switch (err) {
231	error.OutOfMemory => \|e\| return e,	535	error.OutOfMemory => \|e\| return e,
232	else => return,	536	else => return error.InvalidLineCommand,
233	};	537	};
234	defer allocator.free(filename);	538	defer allocator.free(filename);
235		539
236	// \x00 bytes in the filename is incompatible with how StringTable works	540	// \x00 bytes in the filename is incompatible with how StringTable works
237	if (std.mem.indexOfScalar(u8, filename, '\x00') != null) return;	541	if (std.mem.indexOfScalar(u8, filename, '\x00') != null) return error.InvalidLineCommand;
238		542
239	current_mapping.line_num = linenum;	543	current_mapping.line_num = linenum;
240	current_mapping.filename.clearRetainingCapacity();	544	current_mapping.filename.clearRetainingCapacity();
...	@@ -494,8 +798,12 @@ pub const SourceMappings = struct {	...	@@ -494,8 +798,12 @@ pub const SourceMappings = struct {
494	if (node.key.filename_offset != filename_offset) {	798	if (node.key.filename_offset != filename_offset) {
495	break :need_new_node true;	799	break :need_new_node true;
496	}	800	}
497	const exist_delta = @as(i64, @intCast(node.key.corresponding_start_line)) - @as(i64, @intCast(node.key.start_line));	801	// TODO: These use i65 to avoid truncation when any of the line number values
498	const cur_delta = @as(i64, @intCast(corresponding_line_num)) - @as(i64, @intCast(line_num));	802	// use all 64 bits of the usize. In reality, line numbers can't really
		803	// get that large so limiting the line number and using a smaller iX
		804	// type here might be a better solution.
		805	const exist_delta = @as(i65, @intCast(node.key.corresponding_start_line)) - @as(i65, @intCast(node.key.start_line));
		806	const cur_delta = @as(i65, @intCast(corresponding_line_num)) - @as(i65, @intCast(line_num));
499	if (exist_delta != cur_delta) {	807	if (exist_delta != cur_delta) {
500	break :need_new_node true;	808	break :need_new_node true;
501	}	809	}
...	@@ -578,15 +886,8 @@ pub const SourceMappings = struct {	...	@@ -578,15 +886,8 @@ pub const SourceMappings = struct {
578	inorder_node.key.start_line -= span_diff;	886	inorder_node.key.start_line -= span_diff;
579		887
580	// This can only really happen if there are #line commands within	888	// This can only really happen if there are #line commands within
581	// a multiline comment, which in theory should be skipped over.	889	// a multiline comment, which should be skipped over.
582	// However, currently, parseAndRemoveLineCommands is not aware of	890	std.debug.assert(prev.key.start_line <= inorder_node.key.start_line);
583	// comments at all.
584	//
585	// TODO: Make parseAndRemoveLineCommands aware of comments/strings
586	// and turn this into an assertion
587	if (prev.key.start_line > inorder_node.key.start_line) {
588	return error.InvalidSourceMappingCollapse;
589	}
590	prev = inorder_node;	891	prev = inorder_node;
591	}	892	}
592	self.end_line -= span_diff;	893	self.end_line -= span_diff;
...	@@ -594,7 +895,7 @@ pub const SourceMappings = struct {	...	@@ -594,7 +895,7 @@ pub const SourceMappings = struct {
594		895
595	/// Returns true if the line is from the main/root file (i.e. not a file that has been	896	/// Returns true if the line is from the main/root file (i.e. not a file that has been
596	/// `#include`d).	897	/// `#include`d).
597	pub fn isRootFile(self: *SourceMappings, line_num: usize) bool {	898	pub fn isRootFile(self: *const SourceMappings, line_num: usize) bool {
598	const source = self.get(line_num) orelse return false;	899	const source = self.get(line_num) orelse return false;
599	return source.filename_offset == self.root_filename_offset;	900	return source.filename_offset == self.root_filename_offset;
600	}	901	}
...	@@ -803,9 +1104,6 @@ test "in place" {	...	@@ -803,9 +1104,6 @@ test "in place" {
803	}	1104	}
804		1105
805	test "line command within a multiline comment" {	1106	test "line command within a multiline comment" {
806	// TODO: Enable once parseAndRemoveLineCommands is comment-aware
807	if (true) return error.SkipZigTest;
808
809	try testParseAndRemoveLineCommands(	1107	try testParseAndRemoveLineCommands(
810	\\/*	1108	\\/*
811	\\#line 1 "irrelevant.rc"	1109	\\#line 1 "irrelevant.rc"
...	@@ -825,4 +1123,103 @@ test "line command within a multiline comment" {	...	@@ -825,4 +1123,103 @@ test "line command within a multiline comment" {
825	\\	1123	\\
826	\\*/	1124	\\*/
827	, .{ .initial_filename = "blah.rc" });	1125	, .{ .initial_filename = "blah.rc" });
		1126
		1127	// * but without / directly after
		1128	try testParseAndRemoveLineCommands(
		1129	\\/** /
		1130	\\#line 1 "irrelevant.rc"
		1131	\\*/
		1132	, &[_]ExpectedSourceSpan{
		1133	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
		1134	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
		1135	.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
		1136	},
		1137	\\/** /
		1138	\\#line 1 "irrelevant.rc"
		1139	\\*/
		1140	, .{ .initial_filename = "blah.rc" });
		1141
		1142	// / and /
		1143	try testParseAndRemoveLineCommands(
		1144	\\/**
		1145	\\#line 1 "irrelevant.rc"
		1146	\\**/
		1147	\\foo
		1148	, &[_]ExpectedSourceSpan{
		1149	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
		1150	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
		1151	.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
		1152	.{ .start_line = 20, .end_line = 20, .filename = "blah.rc" },
		1153	},
		1154	\\/**
		1155	\\#line 1 "irrelevant.rc"
		1156	\\**/
		1157	\\#line 20 "blah.rc"
		1158	\\foo
		1159	, .{ .initial_filename = "blah.rc" });
		1160	}
		1161
		1162	test "whitespace preservation" {
		1163	try testParseAndRemoveLineCommands(
		1164	\\ /
		1165	\\/
		1166	, &[_]ExpectedSourceSpan{
		1167	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
		1168	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
		1169	},
		1170	\\ /
		1171	\\/
		1172	, .{ .initial_filename = "blah.rc" });
		1173	}
		1174
		1175	test "preprocessor line with a multiline comment after" {
		1176	try testParseAndRemoveLineCommands(
		1177	\\#pragma test /*
		1178	\\
		1179	\\*/
		1180	, &[_]ExpectedSourceSpan{
		1181	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
		1182	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
		1183	.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
		1184	},
		1185	\\#pragma test /*
		1186	\\
		1187	\\*/
		1188	, .{ .initial_filename = "blah.rc" });
		1189	}
		1190
		1191	test "comment after line command" {
		1192	var mut_source = "#line 1 \"blah.rc\" /".;
		1193	try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
		1194	}
		1195
		1196	test "line command with 0 as line number" {
		1197	var mut_source = "#line 0 \"blah.rc\"".*;
		1198	try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
		1199	}
		1200
		1201	test "line number limits" {
		1202	// TODO: Avoid usize for line numbers
		1203	if (@sizeOf(usize) != 8) return error.SkipZigTest;
		1204
		1205	// greater than i64 max
		1206	try testParseAndRemoveLineCommands(
		1207	\\
		1208	, &[_]ExpectedSourceSpan{
		1209	.{ .start_line = 11111111111111111111, .end_line = 11111111111111111111, .filename = "blah.rc" },
		1210	},
		1211	\\#line 11111111111111111111 "blah.rc"
		1212	, .{ .initial_filename = "blah.rc" });
		1213
		1214	// equal to u64 max, overflows on line number increment
		1215	{
		1216	var mut_source = "#line 18446744073709551615 \"blah.rc\"".*;
		1217	try std.testing.expectError(error.LineNumberOverflow, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
		1218	}
		1219
		1220	// greater than u64 max
		1221	{
		1222	var mut_source = "#line 18446744073709551616 \"blah.rc\"".*;
		1223	try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
		1224	}
828	}	1225	}