Commit 160445ef316f76dfddfa17b11c4919a3e14f486d | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-01-20 23:16:47-05:00
committer	GitHub <noreply@github.com> 2025-01-20 23:16:47-05:00
log	160445ef316f76dfddfa17b11c4919a3e14f486d
tree	55f13daf40012512ac591a0350d9b91643fbdaf1
parent	0d6b17b6a5cd4d179417f8a13302558b05f1f2ca
parent	289e9c3507d622bd42a6dee4df8a69dd71a7dfe6
signature	Signed by PGP key B5690EEEBB952194

Merge pull request #22522 from squeek502/resinator-sync

resinator: Sync with upstream

17 files changed, 2145 insertions(+), 949 deletions(-)

lib
- compiler
  - resinator
    ast.zig
    bmp.zig
    cli.zig
    code_pages.zig
    comments.zig
    compile.zig
    disjoint_code_page.zig
    errors.zig
    lang.zig
    lex.zig
    literals.zig
    main.zig
    parse.zig
    preprocess.zig
    rc.zig
    res.zig
    source_mapping.zig

lib/compiler/resinator/ast.zig+49-49

...	...	@@ -1,7 +1,7 @@
1	1	const std = @import("std");
2	2	const Allocator = std.mem.Allocator;
3	3	const Token = @import("lex.zig").Token;
4		const CodePage = @import("code_pages.zig").CodePage;
	4	const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
5	5
6	6	pub const Tree = struct {
7	7	node: *Node,
...	...	@@ -28,11 +28,11 @@ pub const Tree = struct {
28	28	};
29	29
30	30	pub const CodePageLookup = struct {
31		lookup: std.ArrayListUnmanaged(CodePage) = .empty,
	31	lookup: std.ArrayListUnmanaged(SupportedCodePage) = .empty,
32	32	allocator: Allocator,
33		default_code_page: CodePage,
	33	default_code_page: SupportedCodePage,
34	34
35		pub fn init(allocator: Allocator, default_code_page: CodePage) CodePageLookup {
	35	pub fn init(allocator: Allocator, default_code_page: SupportedCodePage) CodePageLookup {
36	36	return .{
37	37	.allocator = allocator,
38	38	.default_code_page = default_code_page,
...	...	@@ -44,7 +44,7 @@ pub const CodePageLookup = struct {
44	44	}
45	45
46	46	/// line_num is 1-indexed
47		pub fn setForLineNum(self: *CodePageLookup, line_num: usize, code_page: CodePage) !void {
	47	pub fn setForLineNum(self: *CodePageLookup, line_num: usize, code_page: SupportedCodePage) !void {
48	48	const index = line_num - 1;
49	49	if (index >= self.lookup.items.len) {
50	50	const new_size = line_num;
...	...	@@ -66,16 +66,16 @@ pub const CodePageLookup = struct {
66	66	self.lookup.items[index] = code_page;
67	67	}
68	68
69		pub fn setForToken(self: *CodePageLookup, token: Token, code_page: CodePage) !void {
	69	pub fn setForToken(self: *CodePageLookup, token: Token, code_page: SupportedCodePage) !void {
70	70	return self.setForLineNum(token.line_number, code_page);
71	71	}
72	72
73	73	/// line_num is 1-indexed
74		pub fn getForLineNum(self: CodePageLookup, line_num: usize) CodePage {
	74	pub fn getForLineNum(self: CodePageLookup, line_num: usize) SupportedCodePage {
75	75	return self.lookup.items[line_num - 1];
76	76	}
77	77
78		pub fn getForToken(self: CodePageLookup, token: Token) CodePage {
	78	pub fn getForToken(self: CodePageLookup, token: Token) SupportedCodePage {
79	79	return self.getForLineNum(token.line_number);
80	80	}
81	81	};
...	...	@@ -85,21 +85,21 @@ test "CodePageLookup" {
85	85	defer lookup.deinit();
86	86
87	87	try lookup.setForLineNum(5, .utf8);
88		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(1));
89		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(2));
90		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(3));
91		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(4));
92		try std.testing.expectEqual(CodePage.utf8, lookup.getForLineNum(5));
	88	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(1));
	89	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(2));
	90	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(3));
	91	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(4));
	92	try std.testing.expectEqual(SupportedCodePage.utf8, lookup.getForLineNum(5));
93	93	try std.testing.expectEqual(@as(usize, 5), lookup.lookup.items.len);
94	94
95	95	try lookup.setForLineNum(7, .windows1252);
96		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(1));
97		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(2));
98		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(3));
99		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(4));
100		try std.testing.expectEqual(CodePage.utf8, lookup.getForLineNum(5));
101		try std.testing.expectEqual(CodePage.utf8, lookup.getForLineNum(6));
102		try std.testing.expectEqual(CodePage.windows1252, lookup.getForLineNum(7));
	96	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(1));
	97	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(2));
	98	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(3));
	99	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(4));
	100	try std.testing.expectEqual(SupportedCodePage.utf8, lookup.getForLineNum(5));
	101	try std.testing.expectEqual(SupportedCodePage.utf8, lookup.getForLineNum(6));
	102	try std.testing.expectEqual(SupportedCodePage.windows1252, lookup.getForLineNum(7));
103	103	try std.testing.expectEqual(@as(usize, 7), lookup.lookup.items.len);
104	104	}
105	105
...	...	@@ -734,31 +734,31 @@ pub const Node = struct {
734	734	switch (node.id) {
735	735	.root => {
736	736	try writer.writeAll("\n");
737		const root: *Node.Root = @alignCast(@fieldParentPtr("base", node));
	737	const root: *const Node.Root = @alignCast(@fieldParentPtr("base", node));
738	738	for (root.body) \|body_node\| {
739	739	try body_node.dump(tree, writer, indent + 1);
740	740	}
741	741	},
742	742	.resource_external => {
743		const resource: *Node.ResourceExternal = @alignCast(@fieldParentPtr("base", node));
	743	const resource: *const Node.ResourceExternal = @alignCast(@fieldParentPtr("base", node));
744	744	try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ resource.id.slice(tree.source), resource.type.slice(tree.source), resource.common_resource_attributes.len });
745	745	try resource.filename.dump(tree, writer, indent + 1);
746	746	},
747	747	.resource_raw_data => {
748		const resource: *Node.ResourceRawData = @alignCast(@fieldParentPtr("base", node));
	748	const resource: *const Node.ResourceRawData = @alignCast(@fieldParentPtr("base", node));
749	749	try writer.print(" {s} {s} [{d} common_resource_attributes] raw data: {}\n", .{ resource.id.slice(tree.source), resource.type.slice(tree.source), resource.common_resource_attributes.len, resource.raw_data.len });
750	750	for (resource.raw_data) \|data_expression\| {
751	751	try data_expression.dump(tree, writer, indent + 1);
752	752	}
753	753	},
754	754	.literal => {
755		const literal: *Node.Literal = @alignCast(@fieldParentPtr("base", node));
	755	const literal: *const Node.Literal = @alignCast(@fieldParentPtr("base", node));
756	756	try writer.writeAll(" ");
757	757	try writer.writeAll(literal.token.slice(tree.source));
758	758	try writer.writeAll("\n");
759	759	},
760	760	.binary_expression => {
761		const binary: *Node.BinaryExpression = @alignCast(@fieldParentPtr("base", node));
	761	const binary: *const Node.BinaryExpression = @alignCast(@fieldParentPtr("base", node));
762	762	try writer.writeAll(" ");
763	763	try writer.writeAll(binary.operator.slice(tree.source));
764	764	try writer.writeAll("\n");
...	...	@@ -766,7 +766,7 @@ pub const Node = struct {
766	766	try binary.right.dump(tree, writer, indent + 1);
767	767	},
768	768	.grouped_expression => {
769		const grouped: *Node.GroupedExpression = @alignCast(@fieldParentPtr("base", node));
	769	const grouped: *const Node.GroupedExpression = @alignCast(@fieldParentPtr("base", node));
770	770	try writer.writeAll("\n");
771	771	try writer.writeByteNTimes(' ', indent);
772	772	try writer.writeAll(grouped.open_token.slice(tree.source));
...	...	@@ -777,7 +777,7 @@ pub const Node = struct {
777	777	try writer.writeAll("\n");
778	778	},
779	779	.not_expression => {
780		const not: *Node.NotExpression = @alignCast(@fieldParentPtr("base", node));
	780	const not: *const Node.NotExpression = @alignCast(@fieldParentPtr("base", node));
781	781	try writer.writeAll(" ");
782	782	try writer.writeAll(not.not_token.slice(tree.source));
783	783	try writer.writeAll(" ");
...	...	@@ -785,7 +785,7 @@ pub const Node = struct {
785	785	try writer.writeAll("\n");
786	786	},
787	787	.accelerators => {
788		const accelerators: *Node.Accelerators = @alignCast(@fieldParentPtr("base", node));
	788	const accelerators: *const Node.Accelerators = @alignCast(@fieldParentPtr("base", node));
789	789	try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ accelerators.id.slice(tree.source), accelerators.type.slice(tree.source), accelerators.common_resource_attributes.len });
790	790	for (accelerators.optional_statements) \|statement\| {
791	791	try statement.dump(tree, writer, indent + 1);
...	...	@@ -801,7 +801,7 @@ pub const Node = struct {
801	801	try writer.writeAll("\n");
802	802	},
803	803	.accelerator => {
804		const accelerator: *Node.Accelerator = @alignCast(@fieldParentPtr("base", node));
	804	const accelerator: *const Node.Accelerator = @alignCast(@fieldParentPtr("base", node));
805	805	for (accelerator.type_and_options, 0..) \|option, i\| {
806	806	if (i != 0) try writer.writeAll(",");
807	807	try writer.writeByte(' ');
...	...	@@ -812,7 +812,7 @@ pub const Node = struct {
812	812	try accelerator.idvalue.dump(tree, writer, indent + 1);
813	813	},
814	814	.dialog => {
815		const dialog: *Node.Dialog = @alignCast(@fieldParentPtr("base", node));
	815	const dialog: *const Node.Dialog = @alignCast(@fieldParentPtr("base", node));
816	816	try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ dialog.id.slice(tree.source), dialog.type.slice(tree.source), dialog.common_resource_attributes.len });
817	817	inline for (.{ "x", "y", "width", "height" }) \|arg\| {
818	818	try writer.writeByteNTimes(' ', indent + 1);
...	...	@@ -838,7 +838,7 @@ pub const Node = struct {
838	838	try writer.writeAll("\n");
839	839	},
840	840	.control_statement => {
841		const control: *Node.ControlStatement = @alignCast(@fieldParentPtr("base", node));
	841	const control: *const Node.ControlStatement = @alignCast(@fieldParentPtr("base", node));
842	842	try writer.print(" {s}", .{control.type.slice(tree.source)});
843	843	if (control.text) \|text\| {
844	844	try writer.print(" text: {s}", .{text.slice(tree.source)});
...	...	@@ -874,7 +874,7 @@ pub const Node = struct {
874	874	}
875	875	},
876	876	.toolbar => {
877		const toolbar: *Node.Toolbar = @alignCast(@fieldParentPtr("base", node));
	877	const toolbar: *const Node.Toolbar = @alignCast(@fieldParentPtr("base", node));
878	878	try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ toolbar.id.slice(tree.source), toolbar.type.slice(tree.source), toolbar.common_resource_attributes.len });
879	879	inline for (.{ "button_width", "button_height" }) \|arg\| {
880	880	try writer.writeByteNTimes(' ', indent + 1);
...	...	@@ -892,7 +892,7 @@ pub const Node = struct {
892	892	try writer.writeAll("\n");
893	893	},
894	894	.menu => {
895		const menu: *Node.Menu = @alignCast(@fieldParentPtr("base", node));
	895	const menu: *const Node.Menu = @alignCast(@fieldParentPtr("base", node));
896	896	try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ menu.id.slice(tree.source), menu.type.slice(tree.source), menu.common_resource_attributes.len });
897	897	for (menu.optional_statements) \|statement\| {
898	898	try statement.dump(tree, writer, indent + 1);
...	...	@@ -913,16 +913,16 @@ pub const Node = struct {
913	913	try writer.writeAll("\n");
914	914	},
915	915	.menu_item => {
916		const menu_item: *Node.MenuItem = @alignCast(@fieldParentPtr("base", node));
	916	const menu_item: *const Node.MenuItem = @alignCast(@fieldParentPtr("base", node));
917	917	try writer.print(" {s} {s} [{d} options]\n", .{ menu_item.menuitem.slice(tree.source), menu_item.text.slice(tree.source), menu_item.option_list.len });
918	918	try menu_item.result.dump(tree, writer, indent + 1);
919	919	},
920	920	.menu_item_separator => {
921		const menu_item: *Node.MenuItemSeparator = @alignCast(@fieldParentPtr("base", node));
	921	const menu_item: *const Node.MenuItemSeparator = @alignCast(@fieldParentPtr("base", node));
922	922	try writer.print(" {s} {s}\n", .{ menu_item.menuitem.slice(tree.source), menu_item.separator.slice(tree.source) });
923	923	},
924	924	.menu_item_ex => {
925		const menu_item: *Node.MenuItemEx = @alignCast(@fieldParentPtr("base", node));
	925	const menu_item: *const Node.MenuItemEx = @alignCast(@fieldParentPtr("base", node));
926	926	try writer.print(" {s} {s}\n", .{ menu_item.menuitem.slice(tree.source), menu_item.text.slice(tree.source) });
927	927	inline for (.{ "id", "type", "state" }) \|arg\| {
928	928	if (@field(menu_item, arg)) \|val_node\| {
...	...	@@ -933,7 +933,7 @@ pub const Node = struct {
933	933	}
934	934	},
935	935	.popup => {
936		const popup: *Node.Popup = @alignCast(@fieldParentPtr("base", node));
	936	const popup: *const Node.Popup = @alignCast(@fieldParentPtr("base", node));
937	937	try writer.print(" {s} {s} [{d} options]\n", .{ popup.popup.slice(tree.source), popup.text.slice(tree.source), popup.option_list.len });
938	938	try writer.writeByteNTimes(' ', indent);
939	939	try writer.writeAll(popup.begin_token.slice(tree.source));
...	...	@@ -946,7 +946,7 @@ pub const Node = struct {
946	946	try writer.writeAll("\n");
947	947	},
948	948	.popup_ex => {
949		const popup: *Node.PopupEx = @alignCast(@fieldParentPtr("base", node));
	949	const popup: *const Node.PopupEx = @alignCast(@fieldParentPtr("base", node));
950	950	try writer.print(" {s} {s}\n", .{ popup.popup.slice(tree.source), popup.text.slice(tree.source) });
951	951	inline for (.{ "id", "type", "state", "help_id" }) \|arg\| {
952	952	if (@field(popup, arg)) \|val_node\| {
...	...	@@ -966,7 +966,7 @@ pub const Node = struct {
966	966	try writer.writeAll("\n");
967	967	},
968	968	.version_info => {
969		const version_info: *Node.VersionInfo = @alignCast(@fieldParentPtr("base", node));
	969	const version_info: *const Node.VersionInfo = @alignCast(@fieldParentPtr("base", node));
970	970	try writer.print(" {s} {s} [{d} common_resource_attributes]\n", .{ version_info.id.slice(tree.source), version_info.versioninfo.slice(tree.source), version_info.common_resource_attributes.len });
971	971	for (version_info.fixed_info) \|fixed_info\| {
972	972	try fixed_info.dump(tree, writer, indent + 1);
...	...	@@ -982,14 +982,14 @@ pub const Node = struct {
982	982	try writer.writeAll("\n");
983	983	},
984	984	.version_statement => {
985		const version_statement: *Node.VersionStatement = @alignCast(@fieldParentPtr("base", node));
	985	const version_statement: *const Node.VersionStatement = @alignCast(@fieldParentPtr("base", node));
986	986	try writer.print(" {s}\n", .{version_statement.type.slice(tree.source)});
987	987	for (version_statement.parts) \|part\| {
988	988	try part.dump(tree, writer, indent + 1);
989	989	}
990	990	},
991	991	.block => {
992		const block: *Node.Block = @alignCast(@fieldParentPtr("base", node));
	992	const block: *const Node.Block = @alignCast(@fieldParentPtr("base", node));
993	993	try writer.print(" {s} {s}\n", .{ block.identifier.slice(tree.source), block.key.slice(tree.source) });
994	994	for (block.values) \|value\| {
995	995	try value.dump(tree, writer, indent + 1);
...	...	@@ -1005,14 +1005,14 @@ pub const Node = struct {
1005	1005	try writer.writeAll("\n");
1006	1006	},
1007	1007	.block_value => {
1008		const block_value: *Node.BlockValue = @alignCast(@fieldParentPtr("base", node));
	1008	const block_value: *const Node.BlockValue = @alignCast(@fieldParentPtr("base", node));
1009	1009	try writer.print(" {s} {s}\n", .{ block_value.identifier.slice(tree.source), block_value.key.slice(tree.source) });
1010	1010	for (block_value.values) \|value\| {
1011	1011	try value.dump(tree, writer, indent + 1);
1012	1012	}
1013	1013	},
1014	1014	.block_value_value => {
1015		const block_value: *Node.BlockValueValue = @alignCast(@fieldParentPtr("base", node));
	1015	const block_value: *const Node.BlockValueValue = @alignCast(@fieldParentPtr("base", node));
1016	1016	if (block_value.trailing_comma) {
1017	1017	try writer.writeAll(" ,");
1018	1018	}
...	...	@@ -1020,7 +1020,7 @@ pub const Node = struct {
1020	1020	try block_value.expression.dump(tree, writer, indent + 1);
1021	1021	},
1022	1022	.string_table => {
1023		const string_table: *Node.StringTable = @alignCast(@fieldParentPtr("base", node));
	1023	const string_table: *const Node.StringTable = @alignCast(@fieldParentPtr("base", node));
1024	1024	try writer.print(" {s} [{d} common_resource_attributes]\n", .{ string_table.type.slice(tree.source), string_table.common_resource_attributes.len });
1025	1025	for (string_table.optional_statements) \|statement\| {
1026	1026	try statement.dump(tree, writer, indent + 1);
...	...	@@ -1037,19 +1037,19 @@ pub const Node = struct {
1037	1037	},
1038	1038	.string_table_string => {
1039	1039	try writer.writeAll("\n");
1040		const string: *Node.StringTableString = @alignCast(@fieldParentPtr("base", node));
	1040	const string: *const Node.StringTableString = @alignCast(@fieldParentPtr("base", node));
1041	1041	try string.id.dump(tree, writer, indent + 1);
1042	1042	try writer.writeByteNTimes(' ', indent + 1);
1043	1043	try writer.print("{s}\n", .{string.string.slice(tree.source)});
1044	1044	},
1045	1045	.language_statement => {
1046		const language: *Node.LanguageStatement = @alignCast(@fieldParentPtr("base", node));
	1046	const language: *const Node.LanguageStatement = @alignCast(@fieldParentPtr("base", node));
1047	1047	try writer.print(" {s}\n", .{language.language_token.slice(tree.source)});
1048	1048	try language.primary_language_id.dump(tree, writer, indent + 1);
1049	1049	try language.sublanguage_id.dump(tree, writer, indent + 1);
1050	1050	},
1051	1051	.font_statement => {
1052		const font: *Node.FontStatement = @alignCast(@fieldParentPtr("base", node));
	1052	const font: *const Node.FontStatement = @alignCast(@fieldParentPtr("base", node));
1053	1053	try writer.print(" {s} typeface: {s}\n", .{ font.identifier.slice(tree.source), font.typeface.slice(tree.source) });
1054	1054	try writer.writeByteNTimes(' ', indent + 1);
1055	1055	try writer.writeAll("point_size:\n");
...	...	@@ -1063,12 +1063,12 @@ pub const Node = struct {
1063	1063	}
1064	1064	},
1065	1065	.simple_statement => {
1066		const statement: *Node.SimpleStatement = @alignCast(@fieldParentPtr("base", node));
	1066	const statement: *const Node.SimpleStatement = @alignCast(@fieldParentPtr("base", node));
1067	1067	try writer.print(" {s}\n", .{statement.identifier.slice(tree.source)});
1068	1068	try statement.value.dump(tree, writer, indent + 1);
1069	1069	},
1070	1070	.invalid => {
1071		const invalid: *Node.Invalid = @alignCast(@fieldParentPtr("base", node));
	1071	const invalid: *const Node.Invalid = @alignCast(@fieldParentPtr("base", node));
1072	1072	try writer.print(" context.len: {}\n", .{invalid.context.len});
1073	1073	for (invalid.context) \|context_token\| {
1074	1074	try writer.writeByteNTimes(' ', indent + 1);

lib/compiler/resinator/bmp.zig+10-3

...	...	@@ -60,9 +60,16 @@ pub const BitmapInfo = struct {
60	60	}
61	61
62	62	pub fn getBitmasksByteLen(self: *const BitmapInfo) u8 {
63		return switch (self.compression) {
64		.BI_BITFIELDS => 12,
65		.BI_ALPHABITFIELDS => 16,
	63	// Only BITMAPINFOHEADER (3.1) has trailing bytes for the BITFIELDS
	64	// The 2.0 format doesn't have a compression field and 4.0+ has dedicated
	65	// fields for the masks in the header.
	66	const dib_version = BitmapHeader.Version.get(self.dib_header_size);
	67	return switch (dib_version) {
	68	.@"nt3.1" => switch (self.compression) {
	69	.BI_BITFIELDS => 12,
	70	.BI_ALPHABITFIELDS => 16,
	71	else => 0,
	72	},
66	73	else => 0,
67	74	};
68	75	}

lib/compiler/resinator/cli.zig+92-35

...	...	@@ -1,5 +1,6 @@
1	1	const std = @import("std");
2		const CodePage = @import("code_pages.zig").CodePage;
	2	const code_pages = @import("code_pages.zig");
	3	const SupportedCodePage = code_pages.SupportedCodePage;
3	4	const lang = @import("lang.zig");
4	5	const res = @import("res.zig");
5	6	const Allocator = std.mem.Allocator;
...	...	@@ -14,6 +15,8 @@ pub const usage_string_after_command_name =
14	15	\\The sequence -- can be used to signify when to stop parsing options.
15	16	\\This is necessary when the input path begins with a forward slash.
16	17	\\
	18	\\Supported option prefixes are /, -, and --, so e.g. /h, -h, and --h all work.
	19	\\
17	20	\\Supported Win32 RC Options:
18	21	\\ /?, /h Print this help and exit.
19	22	\\ /v Verbose (print progress messages).
...	...	@@ -56,8 +59,6 @@ pub const usage_string_after_command_name =
56	59	\\ the .rc includes or otherwise depends on.
57	60	\\ /:depfile-fmt <value> Output format of the depfile, if /:depfile is set.
58	61	\\ json (default) A top-level JSON array of paths
59		\\ /:mingw-includes <path> Path to a directory containing MinGW include files. If
60		\\ not specified, bundled MinGW include files will be used.
61	62	\\
62	63	\\Note: For compatibility reasons, all custom options start with :
63	64	\\
...	...	@@ -136,7 +137,7 @@ pub const Options = struct {
136	137	ignore_include_env_var: bool = false,
137	138	preprocess: Preprocess = .yes,
138	139	default_language_id: ?u16 = null,
139		default_code_page: ?CodePage = null,
	140	default_code_page: ?SupportedCodePage = null,
140	141	verbose: bool = false,
141	142	symbols: std.StringArrayHashMapUnmanaged(SymbolValue) = .empty,
142	143	null_terminate_string_table_strings: bool = false,
...	...	@@ -148,7 +149,6 @@ pub const Options = struct {
148	149	auto_includes: AutoIncludes = .any,
149	150	depfile_path: ?[]const u8 = null,
150	151	depfile_fmt: DepfileFormat = .json,
151		mingw_includes_dir: ?[]const u8 = null,
152	152
153	153	pub const AutoIncludes = enum { any, msvc, gnu, none };
154	154	pub const DepfileFormat = enum { json };
...	...	@@ -243,9 +243,6 @@ pub const Options = struct {
243	243	if (self.depfile_path) \|depfile_path\| {
244	244	self.allocator.free(depfile_path);
245	245	}
246		if (self.mingw_includes_dir) \|mingw_includes_dir\| {
247		self.allocator.free(mingw_includes_dir);
248		}
249	246	}
250	247
251	248	pub fn dumpVerbose(self: *const Options, writer: anytype) !void {
...	...	@@ -358,6 +355,29 @@ pub const Arg = struct {
358	355	};
359	356	}
360	357
	358	pub fn looksLikeFilepath(self: Arg) bool {
	359	const meets_min_requirements = self.prefix == .slash and isSupportedInputExtension(std.fs.path.extension(self.full));
	360	if (!meets_min_requirements) return false;
	361
	362	const could_be_fo_option = could_be_fo_option: {
	363	var window_it = std.mem.window(u8, self.full[1..], 2, 1);
	364	while (window_it.next()) \|window\| {
	365	if (std.ascii.eqlIgnoreCase(window, "fo")) break :could_be_fo_option true;
	366	// If we see '/' before "fo", then it's not possible for this to be a valid
	367	// `/fo` option.
	368	if (window[0] == '/') break;
	369	}
	370	break :could_be_fo_option false;
	371	};
	372	if (!could_be_fo_option) return true;
	373
	374	// It's still possible for a file path to look like a /fo option but not actually
	375	// be one, e.g. `/foo/bar.rc`. As a last ditch effort to reduce false negatives,
	376	// check if the file path exists and, if so, then we ignore the 'could be /fo option'-ness
	377	std.fs.accessAbsolute(self.full, .{}) catch return false;
	378	return true;
	379	}
	380
361	381	pub const Value = struct {
362	382	slice: []const u8,
363	383	index_increment: u2 = 1,
...	...	@@ -432,6 +452,16 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
432	452	}
433	453	}
434	454
	455	const args_remaining = args.len - arg_i;
	456	if (args_remaining <= 2 and arg.looksLikeFilepath()) {
	457	var err_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i };
	458	var msg_writer = err_details.msg.writer(allocator);
	459	try msg_writer.writeAll("this argument was inferred to be a filepath, so argument parsing was terminated");
	460	try diagnostics.append(err_details);
	461
	462	break;
	463	}
	464
435	465	while (arg.name().len > 0) {
436	466	const arg_name = arg.name();
437	467	// Note: These cases should be in order from longest to shortest, since
...	...	@@ -440,24 +470,6 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
440	470	if (std.ascii.startsWithIgnoreCase(arg_name, ":no-preprocess")) {
441	471	options.preprocess = .no;
442	472	arg.name_offset += ":no-preprocess".len;
443		} else if (std.ascii.startsWithIgnoreCase(arg_name, ":mingw-includes")) {
444		const value = arg.value(":mingw-includes".len, arg_i, args) catch {
445		var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() };
446		var msg_writer = err_details.msg.writer(allocator);
447		try msg_writer.print("missing value after {s}{s} option", .{ arg.prefixSlice(), arg.optionWithoutPrefix(":mingw-includes".len) });
448		try diagnostics.append(err_details);
449		arg_i += 1;
450		break :next_arg;
451		};
452		if (options.mingw_includes_dir) \|overwritten_path\| {
453		allocator.free(overwritten_path);
454		options.mingw_includes_dir = null;
455		}
456		const path = try allocator.dupe(u8, value.slice);
457		errdefer allocator.free(path);
458		options.mingw_includes_dir = path;
459		arg_i += value.index_increment;
460		continue :next_arg;
461	473	} else if (std.ascii.startsWithIgnoreCase(arg_name, ":auto-includes")) {
462	474	const value = arg.value(":auto-includes".len, arg_i, args) catch {
463	475	var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = arg.missingSpan() };
...	...	@@ -769,7 +781,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
769	781	arg_i += value.index_increment;
770	782	continue :next_arg;
771	783	};
772		options.default_code_page = CodePage.getByIdentifierEnsureSupported(code_page_id) catch \|err\| switch (err) {
	784	options.default_code_page = code_pages.getByIdentifierEnsureSupported(code_page_id) catch \|err\| switch (err) {
773	785	error.InvalidCodePage => {
774	786	var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) };
775	787	var msg_writer = err_details.msg.writer(allocator);
...	...	@@ -782,7 +794,7 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
782	794	var err_details = Diagnostics.ErrorDetails{ .arg_index = arg_i, .arg_span = value.argSpan(arg) };
783	795	var msg_writer = err_details.msg.writer(allocator);
784	796	try msg_writer.print("unsupported code page: {s} (id={})", .{
785		@tagName(CodePage.getByIdentifier(code_page_id) catch unreachable),
	797	@tagName(code_pages.getByIdentifier(code_page_id) catch unreachable),
786	798	code_page_id,
787	799	});
788	800	try diagnostics.append(err_details);
...	...	@@ -900,18 +912,20 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
900	912
901	913	const positionals = args[arg_i..];
902	914
903		if (positionals.len < 1) {
	915	if (positionals.len == 0) {
904	916	var err_details = Diagnostics.ErrorDetails{ .print_args = false, .arg_index = arg_i };
905	917	var msg_writer = err_details.msg.writer(allocator);
906	918	try msg_writer.writeAll("missing input filename");
907	919	try diagnostics.append(err_details);
908	920
909		const last_arg = args[args.len - 1];
910		if (arg_i > 0 and last_arg.len > 0 and last_arg[0] == '/' and std.ascii.endsWithIgnoreCase(last_arg, ".rc")) {
911		var note_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i - 1 };
912		var note_writer = note_details.msg.writer(allocator);
913		try note_writer.writeAll("if this argument was intended to be the input filename, then -- should be specified in front of it to exclude it from option parsing");
914		try diagnostics.append(note_details);
	921	if (args.len > 0) {
	922	const last_arg = args[args.len - 1];
	923	if (arg_i > 0 and last_arg.len > 0 and last_arg[0] == '/' and std.ascii.endsWithIgnoreCase(last_arg, ".rc")) {
	924	var note_details = Diagnostics.ErrorDetails{ .type = .note, .print_args = true, .arg_index = arg_i - 1 };
	925	var note_writer = note_details.msg.writer(allocator);
	926	try note_writer.writeAll("if this argument was intended to be the input filename, then -- should be specified in front of it to exclude it from option parsing");
	927	try diagnostics.append(note_details);
	928	}
915	929	}
916	930
917	931	// This is a fatal enough problem to justify an early return, since
...	...	@@ -969,6 +983,12 @@ pub fn parse(allocator: Allocator, args: []const []const u8, diagnostics: *Diagn
969	983	return options;
970	984	}
971	985
	986	pub fn isSupportedInputExtension(ext: []const u8) bool {
	987	if (std.ascii.eqlIgnoreCase(ext, ".rc")) return true;
	988	if (std.ascii.eqlIgnoreCase(ext, ".rcpp")) return true;
	989	return false;
	990	}
	991
972	992	/// Returns true if the str is a valid C identifier for use in a #define/#undef macro
973	993	pub fn isValidIdentifier(str: []const u8) bool {
974	994	for (str, 0..) \|c, i\| switch (c) {
...	...	@@ -1271,6 +1291,43 @@ test "parse errors: basic" {
1271	1291	);
1272	1292	}
1273	1293
	1294	test "inferred absolute filepaths" {
	1295	{
	1296	var options = try testParseWarning(&.{ "/fo", "foo.res", "/home/absolute/path.rc" },
	1297	\\<cli>: note: this argument was inferred to be a filepath, so argument parsing was terminated
	1298	\\ ... /home/absolute/path.rc
	1299	\\ ^~~~~~~~~~~~~~~~~~~~~~
	1300	\\
	1301	);
	1302	defer options.deinit();
	1303	}
	1304	{
	1305	var options = try testParseWarning(&.{ "/home/absolute/path.rc", "foo.res" },
	1306	\\<cli>: note: this argument was inferred to be a filepath, so argument parsing was terminated
	1307	\\ ... /home/absolute/path.rc ...
	1308	\\ ^~~~~~~~~~~~~~~~~~~~~~
	1309	\\
	1310	);
	1311	defer options.deinit();
	1312	}
	1313	{
	1314	// Only the last two arguments are checked, so the /h is parsed as an option
	1315	var options = try testParse(&.{ "/home/absolute/path.rc", "foo.rc", "foo.res" });
	1316	defer options.deinit();
	1317
	1318	try std.testing.expect(options.print_help_and_exit);
	1319	}
	1320	{
	1321	var options = try testParse(&.{ "/xvFO/some/absolute/path.res", "foo.rc" });
	1322	defer options.deinit();
	1323
	1324	try std.testing.expectEqual(true, options.verbose);
	1325	try std.testing.expectEqual(true, options.ignore_include_env_var);
	1326	try std.testing.expectEqualStrings("foo.rc", options.input_source.filename);
	1327	try std.testing.expectEqualStrings("/some/absolute/path.res", options.output_source.filename);
	1328	}
	1329	}
	1330
1274	1331	test "parse errors: /ln" {
1275	1332	try testParseError(&.{ "/ln", "invalid", "foo.rc" },
1276	1333	\\<cli>: error: invalid language tag: invalid

lib/compiler/resinator/code_pages.zig+78-139

...	...	@@ -1,86 +1,30 @@
1	1	const std = @import("std");
2	2	const windows1252 = @import("windows1252.zig");
3	3
4		// TODO: Parts of this comment block may be more relevant to string/NameOrOrdinal parsing
5		// than it is to the stuff in this file.
6		//
7		// ‰ representations for context:
8		// Win-1252 89
9		// UTF-8 E2 80 B0
10		// UTF-16 20 30
11		//
12		// With code page 65001:
13		// ‰ RCDATA { "‰" L"‰" }
14		// File encoded as Windows-1252:
15		// ‰ => <U+FFFD REPLACEMENT CHARACTER> as u16
16		// "‰" => 0x3F ('?')
17		// L"‰" => <U+FFFD REPLACEMENT CHARACTER> as u16
18		// File encoded as UTF-8:
19		// ‰ => <U+2030 ‰> as u16
20		// "‰" => 0x89 ('‰' encoded as Windows-1252)
21		// L"‰" => <U+2030 ‰> as u16
22		//
23		// With code page 1252:
24		// ‰ RCDATA { "‰" L"‰" }
25		// File encoded as Windows-1252:
26		// ‰ => <U+2030 ‰> as u16
27		// "‰" => 0x89 ('‰' encoded as Windows-1252)
28		// L"‰" => <U+2030 ‰> as u16
29		// File encoded as UTF-8:
30		// ‰ => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16
31		// ^ first byte of utf8 representation
32		// ^ second byte of UTF-8 representation (0x80), but interpretted as
33		// Windows-1252 ('€') and then converted to UTF-16 (<U+20AC>)
34		// ^ third byte of utf8 representation
35		// "‰" => 0xE2, 0x80, 0xB0 (the bytes of the UTF-8 representation)
36		// L"‰" => 0xE2 as u16, 0x20AC as u16, 0xB0 as u16 (see '‰ =>' explanation)
37		//
38		// With code page 1252:
39		// <0x90> RCDATA { "<0x90>" L"<0x90>" }
40		// File encoded as Windows-1252:
41		// <0x90> => 0x90 as u16
42		// "<0x90>" => 0x90
43		// L"<0x90>" => 0x90 as u16
44		// File encoded as UTF-8:
45		// <0x90> => 0xC2 as u16, 0x90 as u16
46		// "<0x90>" => 0xC2, 0x90 (the bytes of the UTF-8 representation of <U+0090>)
47		// L"<0x90>" => 0xC2 as u16, 0x90 as u16
48		//
49		// Within a raw data block, file encoded as Windows-1252 (Â is <0xC2>):
50		// "Âa" L"Âa" "\xC2ad" L"\xC2AD"
51		// With code page 1252:
52		// C2 61 C2 00 61 00 C2 61 64 AD C2
53		// Â^ a^ Â~~~^ a~~~^ .^ a^ d^ ^~~~~\xC2AD
54		// \xC2~`
55		// With code page 65001:
56		// 3F 61 FD FF 61 00 C2 61 64 AD C2
57		// ^. a^ ^~~~. a~~~^ ^. a^ d^ ^~~~~\xC2AD
58		// `. `. `~\xC2
59		// `. `.~<0xC2>a is not well-formed UTF-8 (0xC2 expects a continutation byte after it).
60		// `. Because 'a' is a valid first byte of a UTF-8 sequence, it is not included in the
61		// `. invalid sequence so only the <0xC2> gets converted to <U+FFFD>.
62		// `~Same as ^ but converted to '?' instead.
63		//
64		// Within a raw data block, file encoded as Windows-1252 (ð is <0xF0>, € is <0x80>):
65		// "ð€a" L"ð€a"
66		// With code page 1252:
67		// F0 80 61 F0 00 AC 20 61 00
68		// ð^ €^ a^ ð~~~^ €~~~^ a~~~^
69		// With code page 65001:
70		// 3F 61 FD FF 61 00
71		// ^. a^ ^~~~. a~~~^
72		// `. `.
73		// `. `.~<0xF0><0x80> is not well-formed UTF-8, and <0x80> is not a valid first byte, so
74		// `. both bytes are considered an invalid sequence and get converted to '<U+FFFD>'
75		// `~Same as ^ but converted to '?' instead.
76
77	4	/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
78		pub const CodePage = enum(u16) {
79		// supported
	5	pub const SupportedCodePage = enum(u16) {
80	6	windows1252 = 1252, // windows-1252 ANSI Latin 1; Western European (Windows)
81	7	utf8 = 65001, // utf-8 Unicode (UTF-8)
82	8
83		// unsupported but valid
	9	pub fn codepointAt(code_page: SupportedCodePage, index: usize, bytes: []const u8) ?Codepoint {
	10	if (index >= bytes.len) return null;
	11	switch (code_page) {
	12	.windows1252 => {
	13	// All byte values have a representation, so just convert the byte
	14	return Codepoint{
	15	.value = windows1252.toCodepoint(bytes[index]),
	16	.byte_len = 1,
	17	};
	18	},
	19	.utf8 => {
	20	return Utf8.WellFormedDecoder.decode(bytes[index..]);
	21	},
	22	}
	23	}
	24	};
	25
	26	/// https://learn.microsoft.com/en-us/windows/win32/intl/code-page-identifiers
	27	pub const UnsupportedCodePage = enum(u16) {
84	28	ibm037 = 37, // IBM037 IBM EBCDIC US-Canada
85	29	ibm437 = 437, // IBM437 OEM United States
86	30	ibm500 = 500, // IBM500 IBM EBCDIC International
...	...	@@ -231,50 +175,45 @@ pub const CodePage = enum(u16) {
231	175	x_iscii_gu = 57010, // x-iscii-gu ISCII Gujarati
232	176	x_iscii_pa = 57011, // x-iscii-pa ISCII Punjabi
233	177	utf7 = 65000, // utf-7 Unicode (UTF-7)
	178	};
234	179
235		pub fn codepointAt(code_page: CodePage, index: usize, bytes: []const u8) ?Codepoint {
236		if (index >= bytes.len) return null;
237		switch (code_page) {
238		.windows1252 => {
239		// All byte values have a representation, so just convert the byte
240		return Codepoint{
241		.value = windows1252.toCodepoint(bytes[index]),
242		.byte_len = 1,
243		};
244		},
245		.utf8 => {
246		return Utf8.WellFormedDecoder.decode(bytes[index..]);
247		},
248		else => unreachable,
249		}
250		}
251
252		pub fn isSupported(code_page: CodePage) bool {
253		return switch (code_page) {
254		.windows1252, .utf8 => true,
255		else => false,
256		};
257		}
	180	pub const CodePage = blk: {
	181	const fields = @typeInfo(SupportedCodePage).@"enum".fields ++ @typeInfo(UnsupportedCodePage).@"enum".fields;
	182	break :blk @Type(.{ .@"enum" = .{
	183	.tag_type = u16,
	184	.decls = &.{},
	185	.fields = fields,
	186	.is_exhaustive = true,
	187	} });
	188	};
258	189
259		pub fn getByIdentifier(identifier: u16) !CodePage {
260		// There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but
261		// this should be fine, especially since this function likely won't be called much.
262		inline for (@typeInfo(CodePage).@"enum".fields) \|enumField\| {
263		if (identifier == enumField.value) {
264		return @field(CodePage, enumField.name);
265		}
	190	pub fn isSupported(code_page: CodePage) bool {
	191	inline for (@typeInfo(SupportedCodePage).@"enum".fields) \|enumField\| {
	192	if (@intFromEnum(code_page) == @intFromEnum(@field(SupportedCodePage, enumField.name))) {
	193	return true;
266	194	}
267		return error.InvalidCodePage;
268	195	}
	196	return false;
	197	}
269	198
270		pub fn getByIdentifierEnsureSupported(identifier: u16) !CodePage {
271		const code_page = try getByIdentifier(identifier);
272		switch (isSupported(code_page)) {
273		true => return code_page,
274		false => return error.UnsupportedCodePage,
	199	pub fn getByIdentifier(identifier: u16) !CodePage {
	200	// There's probably a more efficient way to do this (e.g. ComptimeHashMap?) but
	201	// this should be fine, especially since this function likely won't be called much.
	202	inline for (@typeInfo(CodePage).@"enum".fields) \|enumField\| {
	203	if (identifier == enumField.value) {
	204	return @field(CodePage, enumField.name);
275	205	}
276	206	}
277		};
	207	return error.InvalidCodePage;
	208	}
	209
	210	pub fn getByIdentifierEnsureSupported(identifier: u16) !SupportedCodePage {
	211	const code_page = try getByIdentifier(identifier);
	212	return if (isSupported(code_page))
	213	@enumFromInt(@intFromEnum(code_page))
	214	else
	215	error.UnsupportedCodePage;
	216	}
278	217
279	218	pub const Utf8 = struct {
280	219	/// Implements decoding with rejection of ill-formed UTF-8 sequences based on section
...	...	@@ -378,20 +317,20 @@ test "codepointAt invalid utf8" {
378	317	try std.testing.expectEqual(Codepoint{
379	318	.value = Codepoint.invalid,
380	319	.byte_len = 1,
381		}, CodePage.utf8.codepointAt(0, invalid_utf8).?);
	320	}, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
382	321	try std.testing.expectEqual(Codepoint{
383	322	.value = Codepoint.invalid,
384	323	.byte_len = 2,
385		}, CodePage.utf8.codepointAt(1, invalid_utf8).?);
	324	}, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
386	325	try std.testing.expectEqual(Codepoint{
387	326	.value = Codepoint.invalid,
388	327	.byte_len = 1,
389		}, CodePage.utf8.codepointAt(3, invalid_utf8).?);
	328	}, SupportedCodePage.utf8.codepointAt(3, invalid_utf8).?);
390	329	try std.testing.expectEqual(Codepoint{
391	330	.value = Codepoint.invalid,
392	331	.byte_len = 1,
393		}, CodePage.utf8.codepointAt(4, invalid_utf8).?);
394		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(5, invalid_utf8));
	332	}, SupportedCodePage.utf8.codepointAt(4, invalid_utf8).?);
	333	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(5, invalid_utf8));
395	334	}
396	335
397	336	{
...	...	@@ -399,12 +338,12 @@ test "codepointAt invalid utf8" {
399	338	try std.testing.expectEqual(Codepoint{
400	339	.value = Codepoint.invalid,
401	340	.byte_len = 2,
402		}, CodePage.utf8.codepointAt(0, invalid_utf8).?);
	341	}, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
403	342	try std.testing.expectEqual(Codepoint{
404	343	.value = Codepoint.invalid,
405	344	.byte_len = 1,
406		}, CodePage.utf8.codepointAt(2, invalid_utf8).?);
407		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(3, invalid_utf8));
	345	}, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
	346	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(3, invalid_utf8));
408	347	}
409	348
410	349	{
...	...	@@ -412,8 +351,8 @@ test "codepointAt invalid utf8" {
412	351	try std.testing.expectEqual(Codepoint{
413	352	.value = Codepoint.invalid,
414	353	.byte_len = 1,
415		}, CodePage.utf8.codepointAt(0, invalid_utf8).?);
416		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, invalid_utf8));
	354	}, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
	355	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(1, invalid_utf8));
417	356	}
418	357
419	358	{
...	...	@@ -421,8 +360,8 @@ test "codepointAt invalid utf8" {
421	360	try std.testing.expectEqual(Codepoint{
422	361	.value = Codepoint.invalid,
423	362	.byte_len = 2,
424		}, CodePage.utf8.codepointAt(0, invalid_utf8).?);
425		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8));
	363	}, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
	364	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
426	365	}
427	366
428	367	{
...	...	@@ -430,12 +369,12 @@ test "codepointAt invalid utf8" {
430	369	try std.testing.expectEqual(Codepoint{
431	370	.value = Codepoint.invalid,
432	371	.byte_len = 1,
433		}, CodePage.utf8.codepointAt(0, invalid_utf8).?);
	372	}, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
434	373	try std.testing.expectEqual(Codepoint{
435	374	.value = Codepoint.invalid,
436	375	.byte_len = 1,
437		}, CodePage.utf8.codepointAt(1, invalid_utf8).?);
438		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, invalid_utf8));
	376	}, SupportedCodePage.utf8.codepointAt(1, invalid_utf8).?);
	377	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, invalid_utf8));
439	378	}
440	379
441	380	{
...	...	@@ -444,11 +383,11 @@ test "codepointAt invalid utf8" {
444	383	try std.testing.expectEqual(Codepoint{
445	384	.value = Codepoint.invalid,
446	385	.byte_len = 2,
447		}, CodePage.utf8.codepointAt(0, invalid_utf8).?);
	386	}, SupportedCodePage.utf8.codepointAt(0, invalid_utf8).?);
448	387	try std.testing.expectEqual(Codepoint{
449	388	.value = Codepoint.invalid,
450	389	.byte_len = 1,
451		}, CodePage.utf8.codepointAt(2, invalid_utf8).?);
	390	}, SupportedCodePage.utf8.codepointAt(2, invalid_utf8).?);
452	391	}
453	392	}
454	393
...	...	@@ -459,19 +398,19 @@ test "codepointAt utf8 encoded" {
459	398	try std.testing.expectEqual(Codepoint{
460	399	.value = '²',
461	400	.byte_len = 2,
462		}, CodePage.utf8.codepointAt(0, utf8_encoded).?);
463		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, utf8_encoded));
	401	}, SupportedCodePage.utf8.codepointAt(0, utf8_encoded).?);
	402	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, utf8_encoded));
464	403
465	404	// with code page windows1252
466	405	try std.testing.expectEqual(Codepoint{
467	406	.value = '\xC2',
468	407	.byte_len = 1,
469		}, CodePage.windows1252.codepointAt(0, utf8_encoded).?);
	408	}, SupportedCodePage.windows1252.codepointAt(0, utf8_encoded).?);
470	409	try std.testing.expectEqual(Codepoint{
471	410	.value = '\xB2',
472	411	.byte_len = 1,
473		}, CodePage.windows1252.codepointAt(1, utf8_encoded).?);
474		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(2, utf8_encoded));
	412	}, SupportedCodePage.windows1252.codepointAt(1, utf8_encoded).?);
	413	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(2, utf8_encoded));
475	414	}
476	415
477	416	test "codepointAt windows1252 encoded" {
...	...	@@ -481,15 +420,15 @@ test "codepointAt windows1252 encoded" {
481	420	try std.testing.expectEqual(Codepoint{
482	421	.value = Codepoint.invalid,
483	422	.byte_len = 1,
484		}, CodePage.utf8.codepointAt(0, windows1252_encoded).?);
485		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.utf8.codepointAt(2, windows1252_encoded));
	423	}, SupportedCodePage.utf8.codepointAt(0, windows1252_encoded).?);
	424	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.utf8.codepointAt(2, windows1252_encoded));
486	425
487	426	// with code page windows1252
488	427	try std.testing.expectEqual(Codepoint{
489	428	.value = '\xB2',
490	429	.byte_len = 1,
491		}, CodePage.windows1252.codepointAt(0, windows1252_encoded).?);
492		try std.testing.expectEqual(@as(?Codepoint, null), CodePage.windows1252.codepointAt(1, windows1252_encoded));
	430	}, SupportedCodePage.windows1252.codepointAt(0, windows1252_encoded).?);
	431	try std.testing.expectEqual(@as(?Codepoint, null), SupportedCodePage.windows1252.codepointAt(1, windows1252_encoded));
493	432	}
494	433
495	434	pub const Codepoint = struct {

lib/compiler/resinator/comments.zig+25

...	...	@@ -174,6 +174,21 @@ pub fn removeComments(source: []const u8, buf: []u8, source_mappings: ?*SourceMa
174	174	},
175	175	},
176	176	}
	177	} else {
	178	switch (state) {
	179	.start,
	180	.line_comment,
	181	.multiline_comment,
	182	.multiline_comment_end,
	183	.single_quoted,
	184	.single_quoted_escape,
	185	.double_quoted,
	186	.double_quoted_escape,
	187	=> {},
	188	.forward_slash => {
	189	result.writeSlice(source[pending_start.?..index]);
	190	},
	191	}
177	192	}
178	193	return result.getWritten();
179	194	}
...	...	@@ -334,6 +349,16 @@ test "comments appended to a line" {
334	349	);
335	350	}
336	351
	352	test "forward slash only" {
	353	try testRemoveComments(
	354	\\ /
	355	\\/
	356	,
	357	\\ /
	358	\\/
	359	);
	360	}
	361
337	362	test "remove comments with mappings" {
338	363	const allocator = std.testing.allocator;
339	364	var mut_source = "blah/\rcommented line\r/blah".*;

lib/compiler/resinator/compile.zig+165-147

...	...	@@ -4,7 +4,7 @@ const Allocator = std.mem.Allocator;
4	4	const Node = @import("ast.zig").Node;
5	5	const lex = @import("lex.zig");
6	6	const Parser = @import("parse.zig").Parser;
7		const Resource = @import("rc.zig").Resource;
	7	const ResourceType = @import("rc.zig").ResourceType;
8	8	const Token = @import("lex.zig").Token;
9	9	const literals = @import("literals.zig");
10	10	const Number = literals.Number;
...	...	@@ -21,7 +21,7 @@ const WORD = std.os.windows.WORD;
21	21	const DWORD = std.os.windows.DWORD;
22	22	const utils = @import("utils.zig");
23	23	const NameOrOrdinal = res.NameOrOrdinal;
24		const CodePage = @import("code_pages.zig").CodePage;
	24	const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
25	25	const CodePageLookup = @import("ast.zig").CodePageLookup;
26	26	const SourceMappings = @import("source_mapping.zig").SourceMappings;
27	27	const windows1252 = @import("windows1252.zig");
...	...	@@ -39,7 +39,10 @@ pub const CompileOptions = struct {
39	39	/// freed by the caller.
40	40	/// TODO: Maybe a dedicated struct for this purpose so that it's a bit nicer to work with.
41	41	dependencies_list: ?*std.ArrayList([]const u8) = null,
42		default_code_page: CodePage = .windows1252,
	42	default_code_page: SupportedCodePage = .windows1252,
	43	/// If true, the first #pragma code_page directive only sets the input code page, but not the output code page.
	44	/// This check must be done before comments are removed from the file.
	45	disjoint_code_page: bool = false,
43	46	ignore_include_env_var: bool = false,
44	47	extra_include_paths: []const []const u8 = &.{},
45	48	/// This is just an API convenience to allow separately passing 'system' (i.e. those
...	...	@@ -66,6 +69,7 @@ pub fn compile(allocator: Allocator, source: []const u8, writer: anytype, option
66	69	});
67	70	var parser = Parser.init(&lexer, .{
68	71	.warn_instead_of_error_on_invalid_code_page = options.warn_instead_of_error_on_invalid_code_page,
	72	.disjoint_code_page = options.disjoint_code_page,
69	73	});
70	74	var tree = try parser.parse(allocator, options.diagnostics);
71	75	defer tree.deinit();
...	...	@@ -98,6 +102,7 @@ pub fn compile(allocator: Allocator, source: []const u8, writer: anytype, option
98	102	.end = 0,
99	103	.line_number = 1,
100	104	},
	105	.code_page = .utf8,
101	106	.print_source_line = false,
102	107	.extra = .{ .file_open_error = .{
103	108	.err = ErrorDetails.FileOpenError.enumFromError(err),
...	...	@@ -213,7 +218,12 @@ pub const Compiler = struct {
213	218	try self.addErrorDetails(.{
214	219	.err = .result_contains_fontdir,
215	220	.type = .hint,
216		.token = undefined,
	221	.token = .{
	222	.id = .invalid,
	223	.start = 0,
	224	.end = 0,
	225	.line_number = 1,
	226	},
217	227	});
218	228	}
219	229	// once we've written every else out, we can write out the finalized STRINGTABLE resources
...	...	@@ -301,7 +311,10 @@ pub const Compiler = struct {
301	311	// UTF-8, we can parse either string type directly to UTF-8.
302	312	var parser = literals.IterativeStringParser.init(bytes, .{
303	313	.start_column = column,
304		.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token },
	314	.diagnostics = self.errContext(literal_node.token),
	315	// TODO: Re-evaluate this. It's not been tested whether or not using the actual
	316	// output code page would make more sense.
	317	.output_code_page = .windows1252,
305	318	});
306	319
307	320	while (try parser.nextUnchecked()) \|parsed\| {
...	...	@@ -401,56 +414,55 @@ pub const Compiler = struct {
401	414	return first_error orelse error.FileNotFound;
402	415	}
403	416
	417	/// Returns a Windows-1252 encoded string regardless of the current output code page.
	418	/// All codepoints are encoded as a maximum of 2 bytes, where unescaped codepoints
	419	/// >= 0x10000 are encoded as `??` and everything else is encoded as 1 byte.
404	420	pub fn parseDlgIncludeString(self: *Compiler, token: Token) ![]u8 {
405		// For the purposes of parsing, we want to strip the L prefix
406		// if it exists since we want escaped integers to be limited to
407		// their ascii string range.
408		//
409		// We keep track of whether or not there was an L prefix, though,
410		// since there's more weirdness to come.
411		var bytes = self.sourceBytesForToken(token);
412		var was_wide_string = false;
413		if (bytes.slice[0] == 'L' or bytes.slice[0] == 'l') {
414		was_wide_string = true;
415		bytes.slice = bytes.slice[1..];
416		}
	421	const bytes = self.sourceBytesForToken(token);
	422	const output_code_page = self.output_code_pages.getForToken(token);
417	423
418	424	var buf = try std.ArrayList(u8).initCapacity(self.allocator, bytes.slice.len);
419	425	errdefer buf.deinit();
420	426
421	427	var iterative_parser = literals.IterativeStringParser.init(bytes, .{
422	428	.start_column = token.calculateColumn(self.source, 8, null),
423		.diagnostics = .{ .diagnostics = self.diagnostics, .token = token },
	429	.diagnostics = self.errContext(token),
	430	// TODO: Potentially re-evaluate this, it's not been tested whether or not
	431	// using the actual output code page would make more sense.
	432	.output_code_page = .windows1252,
424	433	});
425	434
426		// No real idea what's going on here, but this matches the rc.exe behavior
	435	// This is similar to the logic in parseQuotedString, but ends up with everything
	436	// encoded as Windows-1252. This effectively consolidates the two-step process
	437	// of rc.exe into one step, since rc.exe's preprocessor converts to UTF-16 (this
	438	// is when invalid sequences are replaced by the replacement character (U+FFFD)),
	439	// and then that's run through the parser. Our preprocessor keeps things in their
	440	// original encoding, meaning we emulate the <encoding> -> UTF-16 -> Windows-1252
	441	// results all at once.
427	442	while (try iterative_parser.next()) \|parsed\| {
428	443	const c = parsed.codepoint;
429		switch (was_wide_string) {
430		true => {
431		switch (c) {
432		0...0x7F, 0xA0...0xFF => try buf.append(@intCast(c)),
433		0x80...0x9F => {
434		if (windows1252.bestFitFromCodepoint(c)) \|_\| {
435		try buf.append(@intCast(c));
436		} else {
437		try buf.append('?');
438		}
439		},
440		else => {
441		if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {
442		try buf.append(best_fit);
443		} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
444		try buf.append('?');
445		} else {
446		try buf.appendSlice("??");
447		}
448		},
	444	switch (iterative_parser.declared_string_type) {
	445	.wide => {
	446	if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {
	447	try buf.append(best_fit);
	448	} else if (c < 0x10000 or c == code_pages.Codepoint.invalid or parsed.escaped_surrogate_pair) {
	449	try buf.append('?');
	450	} else {
	451	try buf.appendSlice("??");
449	452	}
450	453	},
451		false => {
	454	.ascii => {
452	455	if (parsed.from_escaped_integer) {
453		try buf.append(@truncate(c));
	456	const truncated: u8 = @truncate(c);
	457	switch (output_code_page) {
	458	.utf8 => switch (truncated) {
	459	0...0x7F => try buf.append(truncated),
	460	else => try buf.append('?'),
	461	},
	462	.windows1252 => {
	463	try buf.append(truncated);
	464	},
	465	}
454	466	} else {
455	467	if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {
456	468	try buf.append(best_fit);
...	...	@@ -484,8 +496,12 @@ pub const Compiler = struct {
484	496	const parsed_filename_terminated = std.mem.sliceTo(parsed_filename, 0);
485	497
486	498	header.applyMemoryFlags(node.common_resource_attributes, self.source);
	499	// This is effectively limited by `max_string_literal_codepoints` which is a u15.
	500	// Each codepoint within a DLGINCLUDE string is encoded as a maximum of
	501	// 2 bytes, which means that the maximum byte length of a DLGINCLUDE string is
	502	// (including the NUL terminator): 32,767 * 2 + 1 = 65,535 or exactly the u16 max.
487	503	header.data_size = @intCast(parsed_filename_terminated.len + 1);
488		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	504	try header.write(writer, self.errContext(node.id));
489	505	try writer.writeAll(parsed_filename_terminated);
490	506	try writer.writeByte(0);
491	507	try writeDataPadding(writer, header.data_size);
...	...	@@ -568,7 +584,7 @@ pub const Compiler = struct {
568	584	header.applyMemoryFlags(node.common_resource_attributes, self.source);
569	585	header.data_size = @intCast(try file.getEndPos());
570	586
571		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	587	try header.write(writer, self.errContext(node.id));
572	588	try file.seekTo(0);
573	589	try writeResourceData(writer, file.reader(), header.data_size);
574	590	return;
...	...	@@ -644,7 +660,7 @@ pub const Compiler = struct {
644	660	.version = self.state.version,
645	661	.characteristics = self.state.characteristics,
646	662	};
647		try image_header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	663	try image_header.write(writer, self.errContext(node.id));
648	664
649	665	// From https://learn.microsoft.com/en-us/windows/win32/menurc/localheader:
650	666	// > The LOCALHEADER structure is the first data written to the RT_CURSOR
...	...	@@ -817,12 +833,26 @@ pub const Compiler = struct {
817	833
818	834	header.data_size = icon_dir.getResDataSize();
819	835
820		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	836	try header.write(writer, self.errContext(node.id));
821	837	try icon_dir.writeResData(writer, first_icon_id);
822	838	try writeDataPadding(writer, header.data_size);
823	839	return;
824	840	},
825		.RCDATA, .HTML, .MANIFEST, .MESSAGETABLE, .DLGINIT, .PLUGPLAY => {
	841	.RCDATA,
	842	.HTML,
	843	.MESSAGETABLE,
	844	.DLGINIT,
	845	.PLUGPLAY,
	846	.VXD,
	847	// Note: All of the below can only be specified by using a number
	848	// as the resource type.
	849	.MANIFEST,
	850	.CURSOR,
	851	.ICON,
	852	.ANICURSOR,
	853	.ANIICON,
	854	.FONTDIR,
	855	=> {
826	856	header.applyMemoryFlags(node.common_resource_attributes, self.source);
827	857	},
828	858	.BITMAP => {
...	...	@@ -855,47 +885,32 @@ pub const Compiler = struct {
855	885	} else if (bitmap_info.getActualPaletteByteLen() < bitmap_info.getExpectedPaletteByteLen()) {
856	886	const num_padding_bytes = bitmap_info.getExpectedPaletteByteLen() - bitmap_info.getActualPaletteByteLen();
857	887
858		// TODO: Make this configurable (command line option)
859		const max_missing_bytes = 4096;
860		if (num_padding_bytes > max_missing_bytes) {
861		var numbers_as_bytes: [16]u8 = undefined;
862		std.mem.writeInt(u64, numbers_as_bytes[0..8], num_padding_bytes, native_endian);
863		std.mem.writeInt(u64, numbers_as_bytes[8..16], max_missing_bytes, native_endian);
864		const values_string_index = try self.diagnostics.putString(&numbers_as_bytes);
865		try self.addErrorDetails(.{
866		.err = .bmp_too_many_missing_palette_bytes,
867		.token = filename_token,
868		.extra = .{ .number = values_string_index },
869		});
870		return self.addErrorDetailsAndFail(.{
871		.err = .bmp_too_many_missing_palette_bytes,
872		.type = .note,
873		.print_source_line = false,
874		.token = filename_token,
875		});
876		}
877
878	888	var number_as_bytes: [8]u8 = undefined;
879	889	std.mem.writeInt(u64, &number_as_bytes, num_padding_bytes, native_endian);
880	890	const value_string_index = try self.diagnostics.putString(&number_as_bytes);
881	891	try self.addErrorDetails(.{
882	892	.err = .bmp_missing_palette_bytes,
883		.type = .warning,
	893	.type = .err,
884	894	.token = filename_token,
885	895	.extra = .{ .number = value_string_index },
886	896	});
887	897	const pixel_data_len = bitmap_info.getPixelDataLen(file_size);
	898	// TODO: This is a hack, but we know we have already added
	899	// at least one entry to the diagnostics strings, so we can
	900	// get away with using 0 to mean 'no string' here.
	901	var miscompiled_bytes_string_index: u32 = 0;
888	902	if (pixel_data_len > 0) {
889	903	const miscompiled_bytes = @min(pixel_data_len, num_padding_bytes);
890	904	std.mem.writeInt(u64, &number_as_bytes, miscompiled_bytes, native_endian);
891		const miscompiled_bytes_string_index = try self.diagnostics.putString(&number_as_bytes);
892		try self.addErrorDetails(.{
893		.err = .rc_would_miscompile_bmp_palette_padding,
894		.type = .warning,
895		.token = filename_token,
896		.extra = .{ .number = miscompiled_bytes_string_index },
897		});
	905	miscompiled_bytes_string_index = try self.diagnostics.putString(&number_as_bytes);
898	906	}
	907	return self.addErrorDetailsAndFail(.{
	908	.err = .rc_would_miscompile_bmp_palette_padding,
	909	.type = .note,
	910	.print_source_line = false,
	911	.token = filename_token,
	912	.extra = .{ .number = miscompiled_bytes_string_index },
	913	});
899	914	}
900	915
901	916	// TODO: It might be possible that the calculation done in this function
...	...	@@ -905,7 +920,7 @@ pub const Compiler = struct {
905	920	const bmp_bytes_to_write: u32 = @intCast(bitmap_info.getExpectedByteLen(file_size));
906	921
907	922	header.data_size = bmp_bytes_to_write;
908		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	923	try header.write(writer, self.errContext(node.id));
909	924	try file.seekTo(bmp.file_header_len);
910	925	const file_reader = file.reader();
911	926	try writeResourceDataNoPadding(writer, file_reader, bitmap_info.dib_header_size);
...	...	@@ -914,12 +929,6 @@ pub const Compiler = struct {
914	929	}
915	930	if (bitmap_info.getExpectedPaletteByteLen() > 0) {
916	931	try writeResourceDataNoPadding(writer, file_reader, @intCast(bitmap_info.getActualPaletteByteLen()));
917		// We know that the number of missing palette bytes is <= 4096
918		// (see `bmp_too_many_missing_palette_bytes` error case above)
919		const padding_bytes: usize = @intCast(bitmap_info.getMissingPaletteByteLen());
920		if (padding_bytes > 0) {
921		try writer.writeByteNTimes(0, padding_bytes);
922		}
923	932	}
924	933	try file.seekTo(bitmap_info.pixel_data_offset);
925	934	const pixel_bytes: u32 = @intCast(file_size - bitmap_info.pixel_data_offset);
...	...	@@ -932,13 +941,13 @@ pub const Compiler = struct {
932	941	// Add warning and skip this resource
933	942	// Note: The Win32 compiler prints this as an error but it doesn't fail the compilation
934	943	// and the duplicate resource is skipped.
935		try self.addErrorDetails(ErrorDetails{
	944	try self.addErrorDetails(.{
936	945	.err = .font_id_already_defined,
937	946	.token = node.id,
938	947	.type = .warning,
939	948	.extra = .{ .number = header.name_value.ordinal },
940	949	});
941		try self.addErrorDetails(ErrorDetails{
	950	try self.addErrorDetails(.{
942	951	.err = .font_id_already_defined,
943	952	.token = self.state.font_dir.ids.get(header.name_value.ordinal).?,
944	953	.type = .note,
...	...	@@ -957,7 +966,7 @@ pub const Compiler = struct {
957	966
958	967	// We now know that the data size will fit in a u32
959	968	header.data_size = @intCast(file_size);
960		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	969	try header.write(writer, self.errContext(node.id));
961	970
962	971	var header_slurping_reader = headerSlurpingReader(148, file.reader());
963	972	try writeResourceData(writer, header_slurping_reader.reader(), header.data_size);
...	...	@@ -968,19 +977,13 @@ pub const Compiler = struct {
968	977	}, node.id);
969	978	return;
970	979	},
971		.ACCELERATOR,
972		.ANICURSOR,
973		.ANIICON,
974		.CURSOR,
975		.DIALOG,
976		.DLGINCLUDE,
977		.FONTDIR,
978		.ICON,
979		.MENU,
980		.STRING,
981		.TOOLBAR,
982		.VERSION,
983		.VXD,
	980	.ACCELERATOR, // Cannot use an external file, enforced by the parser
	981	.DIALOG, // Cannot use an external file, enforced by the parser
	982	.DLGINCLUDE, // Handled specially above
	983	.MENU, // Cannot use an external file, enforced by the parser
	984	.STRING, // Parser error if this resource is specified as a number
	985	.TOOLBAR, // Cannot use an external file, enforced by the parser
	986	.VERSION, // Cannot use an external file, enforced by the parser
984	987	=> unreachable,
985	988	_ => unreachable,
986	989	}
...	...	@@ -998,7 +1001,7 @@ pub const Compiler = struct {
998	1001	}
999	1002	// We now know that the data size will fit in a u32
1000	1003	header.data_size = @intCast(data_size);
1001		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	1004	try header.write(writer, self.errContext(node.id));
1002	1005	try writeResourceData(writer, file.reader(), header.data_size);
1003	1006	}
1004	1007
...	...	@@ -1188,7 +1191,7 @@ pub const Compiler = struct {
1188	1191	};
1189	1192	const parsed = try literals.parseQuotedAsciiString(self.allocator, bytes, .{
1190	1193	.start_column = column,
1191		.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token },
	1194	.diagnostics = self.errContext(literal_node.token),
1192	1195	.output_code_page = self.output_code_pages.getForToken(literal_node.token),
1193	1196	});
1194	1197	errdefer self.allocator.free(parsed);
...	...	@@ -1202,7 +1205,8 @@ pub const Compiler = struct {
1202	1205	};
1203	1206	const parsed_string = try literals.parseQuotedWideString(self.allocator, bytes, .{
1204	1207	.start_column = column,
1205		.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal_node.token },
	1208	.diagnostics = self.errContext(literal_node.token),
	1209	.output_code_page = self.output_code_pages.getForToken(literal_node.token),
1206	1210	});
1207	1211	errdefer self.allocator.free(parsed_string);
1208	1212	return .{ .wide_string = parsed_string };
...	...	@@ -1259,7 +1263,7 @@ pub const Compiler = struct {
1259	1263
1260	1264	header.applyMemoryFlags(common_resource_attributes, self.source);
1261	1265
1262		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = id_token });
	1266	try header.write(writer, self.errContext(id_token));
1263	1267	}
1264	1268
1265	1269	pub fn writeResourceDataNoPadding(writer: anytype, data_reader: anytype, data_size: u32) !void {
...	...	@@ -1297,7 +1301,8 @@ pub const Compiler = struct {
1297	1301	const column = literal.token.calculateColumn(self.source, 8, null);
1298	1302	return res.parseAcceleratorKeyString(bytes, is_virt, .{
1299	1303	.start_column = column,
1300		.diagnostics = .{ .diagnostics = self.diagnostics, .token = literal.token },
	1304	.diagnostics = self.errContext(literal.token),
	1305	.output_code_page = self.output_code_pages.getForToken(literal.token),
1301	1306	});
1302	1307	}
1303	1308	}
...	...	@@ -1332,7 +1337,7 @@ pub const Compiler = struct {
1332	1337	header.applyMemoryFlags(node.common_resource_attributes, self.source);
1333	1338	header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
1334	1339
1335		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	1340	try header.write(writer, self.errContext(node.id));
1336	1341
1337	1342	var data_fbs = std.io.fixedBufferStream(data_buffer.items);
1338	1343	try writeResourceData(writer, data_fbs.reader(), data_size);
...	...	@@ -1348,6 +1353,16 @@ pub const Compiler = struct {
1348	1353	const modifier = rc.AcceleratorTypeAndOptions.map.get(type_or_option.slice(self.source)).?;
1349	1354	modifiers.apply(modifier);
1350	1355	}
	1356	if ((modifiers.isSet(.control) or modifiers.isSet(.shift)) and !modifiers.isSet(.virtkey)) {
	1357	try self.addErrorDetails(.{
	1358	.err = .accelerator_shift_or_control_without_virtkey,
	1359	.type = .warning,
	1360	// We know that one of SHIFT or CONTROL was specified, so there's at least one item
	1361	// in this list.
	1362	.token = accelerator.type_and_options[0],
	1363	.token_span_end = accelerator.type_and_options[accelerator.type_and_options.len - 1],
	1364	});
	1365	}
1351	1366	if (accelerator.event.isNumberExpression() and !modifiers.explicit_ascii_or_virtkey) {
1352	1367	return self.addErrorDetailsAndFail(.{
1353	1368	.err = .accelerator_type_required,
...	...	@@ -1399,7 +1414,7 @@ pub const Compiler = struct {
1399	1414	var limited_writer = limitedWriter(data_buffer.writer(), std.math.maxInt(u32));
1400	1415	const data_writer = limited_writer.writer();
1401	1416
1402		const resource = Resource.fromString(.{
	1417	const resource = ResourceType.fromString(.{
1403	1418	.slice = node.type.slice(self.source),
1404	1419	.code_page = self.input_code_pages.getForToken(node.type),
1405	1420	});
...	...	@@ -1414,8 +1429,6 @@ pub const Compiler = struct {
1414	1429	menu.deinit(self.allocator);
1415	1430	}
1416	1431	}
1417		var skipped_menu_or_classes = std.ArrayList(*Node.SimpleStatement).init(self.allocator);
1418		defer skipped_menu_or_classes.deinit();
1419	1432	var last_menu: *Node.SimpleStatement = undefined;
1420	1433	var last_class: *Node.SimpleStatement = undefined;
1421	1434	var last_menu_would_be_forced_ordinal = false;
...	...	@@ -1445,9 +1458,6 @@ pub const Compiler = struct {
1445	1458	},
1446	1459	.class => {
1447	1460	const is_duplicate = optional_statement_values.class != null;
1448		if (is_duplicate) {
1449		try skipped_menu_or_classes.append(last_class);
1450		}
1451	1461	const forced_ordinal = is_duplicate and optional_statement_values.class.? == .ordinal;
1452	1462	// In the Win32 RC compiler, if any CLASS values that are interpreted as
1453	1463	// an ordinal exist, it affects all future CLASS statements and forces
...	...	@@ -1475,9 +1485,6 @@ pub const Compiler = struct {
1475	1485	},
1476	1486	.menu => {
1477	1487	const is_duplicate = optional_statement_values.menu != null;
1478		if (is_duplicate) {
1479		try skipped_menu_or_classes.append(last_menu);
1480		}
1481	1488	const forced_ordinal = is_duplicate and optional_statement_values.menu.? == .ordinal;
1482	1489	// In the Win32 RC compiler, if any MENU values that are interpreted as
1483	1490	// an ordinal exist, it affects all future MENU statements and forces
...	...	@@ -1561,22 +1568,6 @@ pub const Compiler = struct {
1561	1568	}
1562	1569	}
1563	1570
1564		for (skipped_menu_or_classes.items) \|simple_statement\| {
1565		const statement_identifier = simple_statement.identifier;
1566		const statement_type = rc.OptionalStatements.dialog_map.get(statement_identifier.slice(self.source)) orelse continue;
1567		try self.addErrorDetails(.{
1568		.err = .duplicate_menu_or_class_skipped,
1569		.type = .warning,
1570		.token = simple_statement.identifier,
1571		.token_span_start = simple_statement.base.getFirstToken(),
1572		.token_span_end = simple_statement.base.getLastToken(),
1573		.extra = .{ .menu_or_class = switch (statement_type) {
1574		.menu => .menu,
1575		.class => .class,
1576		else => unreachable,
1577		} },
1578		});
1579		}
1580	1571	// The Win32 RC compiler miscompiles the value in the following scenario:
1581	1572	// Multiple CLASS parameters are specified and any of them are treated as a number, then
1582	1573	// the last CLASS is always treated as a number no matter what
...	...	@@ -1739,7 +1730,7 @@ pub const Compiler = struct {
1739	1730	header.applyMemoryFlags(node.common_resource_attributes, self.source);
1740	1731	header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
1741	1732
1742		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	1733	try header.write(writer, self.errContext(node.id));
1743	1734
1744	1735	var data_fbs = std.io.fixedBufferStream(data_buffer.items);
1745	1736	try writeResourceData(writer, data_fbs.reader(), data_size);
...	...	@@ -1749,7 +1740,7 @@ pub const Compiler = struct {
1749	1740	self: *Compiler,
1750	1741	node: *Node.Dialog,
1751	1742	data_writer: anytype,
1752		resource: Resource,
	1743	resource: ResourceType,
1753	1744	optional_statement_values: *const DialogOptionalStatementValues,
1754	1745	x: Number,
1755	1746	y: Number,
...	...	@@ -1809,7 +1800,7 @@ pub const Compiler = struct {
1809	1800	self: *Compiler,
1810	1801	control: *Node.ControlStatement,
1811	1802	data_writer: anytype,
1812		resource: Resource,
	1803	resource: ResourceType,
1813	1804	bytes_written_so_far: u32,
1814	1805	controls_by_id: std.AutoHashMap(u32, const Node.ControlStatement),
1815	1806	) !void {
...	...	@@ -2053,7 +2044,7 @@ pub const Compiler = struct {
2053	2044
2054	2045	header.applyMemoryFlags(node.common_resource_attributes, self.source);
2055	2046
2056		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	2047	try header.write(writer, self.errContext(node.id));
2057	2048
2058	2049	var data_fbs = std.io.fixedBufferStream(data_buffer.items);
2059	2050	try writeResourceData(writer, data_fbs.reader(), data_size);
...	...	@@ -2067,7 +2058,7 @@ pub const Compiler = struct {
2067	2058	node: *Node.FontStatement,
2068	2059	};
2069	2060
2070		pub fn writeDialogFont(self: *Compiler, resource: Resource, values: FontStatementValues, writer: anytype) !void {
	2061	pub fn writeDialogFont(self: *Compiler, resource: ResourceType, values: FontStatementValues, writer: anytype) !void {
2071	2062	const node = values.node;
2072	2063	const point_size = evaluateNumberExpression(node.point_size, self.source, self.input_code_pages);
2073	2064	try writer.writeInt(u16, point_size.asWord(), .little);
...	...	@@ -2104,7 +2095,7 @@ pub const Compiler = struct {
2104	2095	.slice = node.type.slice(self.source),
2105	2096	.code_page = self.input_code_pages.getForToken(node.type),
2106	2097	};
2107		const resource = Resource.fromString(type_bytes);
	2098	const resource = ResourceType.fromString(type_bytes);
2108	2099	std.debug.assert(resource == .menu or resource == .menuex);
2109	2100
2110	2101	self.writeMenuData(node, data_writer, resource) catch \|err\| switch (err) {
...	...	@@ -2128,7 +2119,7 @@ pub const Compiler = struct {
2128	2119	header.applyMemoryFlags(node.common_resource_attributes, self.source);
2129	2120	header.applyOptionalStatements(node.optional_statements, self.source, self.input_code_pages);
2130	2121
2131		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	2122	try header.write(writer, self.errContext(node.id));
2132	2123
2133	2124	var data_fbs = std.io.fixedBufferStream(data_buffer.items);
2134	2125	try writeResourceData(writer, data_fbs.reader(), data_size);
...	...	@@ -2136,7 +2127,7 @@ pub const Compiler = struct {
2136	2127
2137	2128	/// Expects `data_writer` to be a LimitedWriter limited to u32, meaning all writes to
2138	2129	/// the writer within this function could return error.NoSpaceLeft
2139		pub fn writeMenuData(self: Compiler, node: Node.Menu, data_writer: anytype, resource: Resource) !void {
	2130	pub fn writeMenuData(self: Compiler, node: Node.Menu, data_writer: anytype, resource: ResourceType) !void {
2140	2131	// menu header
2141	2132	const version: u16 = if (resource == .menu) 0 else 1;
2142	2133	try data_writer.writeInt(u16, version, .little);
...	...	@@ -2393,7 +2384,7 @@ pub const Compiler = struct {
2393	2384
2394	2385	header.applyMemoryFlags(node.common_resource_attributes, self.source);
2395	2386
2396		try header.write(writer, .{ .diagnostics = self.diagnostics, .token = node.id });
	2387	try header.write(writer, self.errContext(node.id));
2397	2388
2398	2389	var data_fbs = std.io.fixedBufferStream(data_buffer.items);
2399	2390	try writeResourceData(writer, data_fbs.reader(), data_size);
...	...	@@ -2525,14 +2516,14 @@ pub const Compiler = struct {
2525	2516	// It might be nice to have these errors point to the ids rather than the
2526	2517	// string tokens, but that would mean storing the id token of each string
2527	2518	// which doesn't seem worth it just for slightly better error messages.
2528		try self.addErrorDetails(ErrorDetails{
	2519	try self.addErrorDetails(.{
2529	2520	.err = .string_already_defined,
2530	2521	.token = string.string,
2531	2522	.extra = .{ .string_and_language = .{ .id = string_id, .language = language } },
2532	2523	});
2533	2524	const existing_def_table = self.state.string_tables.tables.getPtr(language).?;
2534	2525	const existing_definition = existing_def_table.get(string_id).?;
2535		return self.addErrorDetailsAndFail(ErrorDetails{
	2526	return self.addErrorDetailsAndFail(.{
2536	2527	.err = .string_already_defined,
2537	2528	.type = .note,
2538	2529	.token = existing_definition,
...	...	@@ -2628,7 +2619,7 @@ pub const Compiler = struct {
2628	2619
2629	2620	pub fn init(allocator: Allocator, id_bytes: SourceBytes, type_bytes: SourceBytes, data_size: DWORD, language: res.Language, version: DWORD, characteristics: DWORD) InitError!ResourceHeader {
2630	2621	const type_value = type: {
2631		const resource_type = Resource.fromString(type_bytes);
	2622	const resource_type = ResourceType.fromString(type_bytes);
2632	2623	if (res.RT.fromResource(resource_type)) \|rt_constant\| {
2633	2624	break :type NameOrOrdinal{ .ordinal = @intFromEnum(rt_constant) };
2634	2625	} else {
...	...	@@ -2673,7 +2664,7 @@ pub const Compiler = struct {
2673	2664	padding_after_name: u2,
2674	2665	};
2675	2666
2676		fn calcSize(self: ResourceHeader) error{Overflow}!SizeInfo {
	2667	pub fn calcSize(self: ResourceHeader) error{Overflow}!SizeInfo {
2677	2668	var header_size: u32 = 8;
2678	2669	header_size = try std.math.add(
2679	2670	u32,
...	...	@@ -2699,6 +2690,7 @@ pub const Compiler = struct {
2699	2690	const size_info = self.calcSize() catch {
2700	2691	try err_ctx.diagnostics.append(.{
2701	2692	.err = .resource_data_size_exceeds_max,
	2693	.code_page = err_ctx.code_page,
2702	2694	.token = err_ctx.token,
2703	2695	});
2704	2696	return error.CompileError;
...	...	@@ -2706,7 +2698,7 @@ pub const Compiler = struct {
2706	2698	return self.writeSizeInfo(writer, size_info);
2707	2699	}
2708	2700
2709		fn writeSizeInfo(self: ResourceHeader, writer: anytype, size_info: SizeInfo) !void {
	2701	pub fn writeSizeInfo(self: ResourceHeader, writer: anytype, size_info: SizeInfo) !void {
2710	2702	try writer.writeInt(DWORD, self.data_size, .little); // DataSize
2711	2703	try writer.writeInt(DWORD, size_info.bytes, .little); // HeaderSize
2712	2704	try self.type_value.write(writer); // TYPE
...	...	@@ -2863,19 +2855,44 @@ pub const Compiler = struct {
2863	2855	self.sourceBytesForToken(token),
2864	2856	.{
2865	2857	.start_column = token.calculateColumn(self.source, 8, null),
2866		.diagnostics = .{ .diagnostics = self.diagnostics, .token = token },
	2858	.diagnostics = self.errContext(token),
	2859	.output_code_page = self.output_code_pages.getForToken(token),
2867	2860	},
2868	2861	);
2869	2862	}
2870	2863
2871		fn addErrorDetails(self: *Compiler, details: ErrorDetails) Allocator.Error!void {
	2864	fn addErrorDetailsWithCodePage(self: *Compiler, details: ErrorDetails) Allocator.Error!void {
2872	2865	try self.diagnostics.append(details);
2873	2866	}
2874	2867
2875		fn addErrorDetailsAndFail(self: *Compiler, details: ErrorDetails) error{ CompileError, OutOfMemory } {
2876		try self.addErrorDetails(details);
	2868	/// Code page is looked up in input_code_pages using the token
	2869	fn addErrorDetails(self: *Compiler, details_without_code_page: errors.ErrorDetailsWithoutCodePage) Allocator.Error!void {
	2870	const details = ErrorDetails{
	2871	.err = details_without_code_page.err,
	2872	.code_page = self.input_code_pages.getForToken(details_without_code_page.token),
	2873	.token = details_without_code_page.token,
	2874	.token_span_start = details_without_code_page.token_span_start,
	2875	.token_span_end = details_without_code_page.token_span_end,
	2876	.type = details_without_code_page.type,
	2877	.print_source_line = details_without_code_page.print_source_line,
	2878	.extra = details_without_code_page.extra,
	2879	};
	2880	try self.addErrorDetailsWithCodePage(details);
	2881	}
	2882
	2883	/// Code page is looked up in input_code_pages using the token
	2884	fn addErrorDetailsAndFail(self: *Compiler, details_without_code_page: errors.ErrorDetailsWithoutCodePage) error{ CompileError, OutOfMemory } {
	2885	try self.addErrorDetails(details_without_code_page);
2877	2886	return error.CompileError;
2878	2887	}
	2888
	2889	fn errContext(self: *Compiler, token: Token) errors.DiagnosticsContext {
	2890	return .{
	2891	.diagnostics = self.diagnostics,
	2892	.token = token,
	2893	.code_page = self.input_code_pages.getForToken(token),
	2894	};
	2895	}
2879	2896	};
2880	2897
2881	2898	pub const OpenSearchPathError = std.fs.Dir.OpenError;
...	...	@@ -3247,7 +3264,8 @@ pub const StringTable = struct {
3247	3264	const bytes = SourceBytes{ .slice = slice, .code_page = code_page };
3248	3265	const utf16_string = try literals.parseQuotedStringAsWideString(compiler.allocator, bytes, .{
3249	3266	.start_column = column,
3250		.diagnostics = .{ .diagnostics = compiler.diagnostics, .token = string_token },
	3267	.diagnostics = compiler.errContext(string_token),
	3268	.output_code_page = compiler.output_code_pages.getForToken(string_token),
3251	3269	});
3252	3270	defer compiler.allocator.free(utf16_string);
3253	3271

lib/compiler/resinator/disjoint_code_page.zig created+99

...	...	@@ -0,0 +1,99 @@
	1	const std = @import("std");
	2	const lex = @import("lex.zig");
	3	const SourceMappings = @import("source_mapping.zig").SourceMappings;
	4	const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
	5
	6	pub fn hasDisjointCodePage(source: []const u8, source_mappings: ?*const SourceMappings, default_code_page: SupportedCodePage) bool {
	7	var line_handler = lex.LineHandler{ .buffer = source };
	8	var i: usize = 0;
	9	while (i < source.len) {
	10	const codepoint = default_code_page.codepointAt(i, source) orelse break;
	11	const c = codepoint.value;
	12	switch (c) {
	13	'\r', '\n' => {
	14	_ = line_handler.incrementLineNumber(i);
	15	// Any lines that are not from the root file interrupt the disjoint code page
	16	if (source_mappings != null and !source_mappings.?.isRootFile(line_handler.line_number)) return false;
	17	},
	18	// whitespace is ignored
	19	' ',
	20	'\t',
	21	// NBSP, this should technically be in the TODO below, but it is treated as whitespace
	22	// due to a (misguided) special casing in the lexer, see the TODO in lex.zig
	23	'\u{A0}',
	24	=> {},
	25
	26	// TODO: All of the below are treated as whitespace by the Win32 RC preprocessor, which also
	27	// means they are trimmed from the file during preprocessing. This means that these characters
	28	// should be treated like ' ', '\t' above, but since the resinator preprocessor does not treat
	29	// them as whitespace or trim whitespace, files with these characters are likely going to
	30	// error. So, in the future some sort of emulation of/rejection of the Win32 behavior might
	31	// make handling these codepoints specially make sense, but for now it doesn't really matter
	32	// so they are not handled specially for simplicity's sake.
	33	//'\u{1680}',
	34	//'\u{180E}',
	35	//'\u{2001}',
	36	//'\u{2002}',
	37	//'\u{2003}',
	38	//'\u{2004}',
	39	//'\u{2005}',
	40	//'\u{2006}',
	41	//'\u{2007}',
	42	//'\u{2008}',
	43	//'\u{2009}',
	44	//'\u{200A}',
	45	//'\u{2028}',
	46	//'\u{2029}',
	47	//'\u{202F}',
	48	//'\u{205F}',
	49	//'\u{3000}',
	50
	51	'#' => {
	52	if (source_mappings != null and !source_mappings.?.isRootFile(line_handler.line_number)) {
	53	return false;
	54	}
	55	const start_i = i;
	56	while (i < source.len and source[i] != '\r' and source[i] != '\n') : (i += 1) {}
	57	const line = source[start_i..i];
	58	_ = (lex.parsePragmaCodePage(line) catch \|err\| switch (err) {
	59	error.NotPragma => return false,
	60	error.NotCodePagePragma => continue,
	61	error.CodePagePragmaUnsupportedCodePage => continue,
	62	else => continue,
	63	}) orelse return false; // DEFAULT interrupts disjoint code page
	64
	65	// If we got a code page, then it is a disjoint code page pragma
	66	return true;
	67	},
	68	else => {
	69	// Any other character interrupts the disjoint code page
	70	return false;
	71	},
	72	}
	73
	74	i += codepoint.byte_len;
	75	}
	76	return false;
	77	}
	78
	79	test hasDisjointCodePage {
	80	try std.testing.expect(hasDisjointCodePage("#pragma code_page(65001)\n", null, .windows1252));
	81	// NBSP is a special case
	82	try std.testing.expect(hasDisjointCodePage("\xA0\n#pragma code_page(65001)\n", null, .windows1252));
	83	try std.testing.expect(hasDisjointCodePage("\u{A0}\n#pragma code_page(1252)\n", null, .utf8));
	84	// other preprocessor commands don't interrupt
	85	try std.testing.expect(hasDisjointCodePage("#pragma foo\n#pragma code_page(65001)\n", null, .windows1252));
	86	// invalid code page doesn't interrupt
	87	try std.testing.expect(hasDisjointCodePage("#pragma code_page(1234567)\n#pragma code_page(65001)\n", null, .windows1252));
	88
	89	try std.testing.expect(!hasDisjointCodePage("#if 1\n#endif\n#pragma code_page(65001)", null, .windows1252));
	90	try std.testing.expect(!hasDisjointCodePage("// comment\n#pragma code_page(65001)", null, .windows1252));
	91	try std.testing.expect(!hasDisjointCodePage("/* comment */\n#pragma code_page(65001)", null, .windows1252));
	92	}
	93
	94	test "multiline comment edge case" {
	95	// TODO
	96	if (true) return error.SkipZigTest;
	97
	98	try std.testing.expect(hasDisjointCodePage("/* comment */#pragma code_page(65001)", null, .windows1252));
	99	}

lib/compiler/resinator/errors.zig+391-204

...	...	@@ -8,7 +8,8 @@ const ico = @import("ico.zig");
8	8	const bmp = @import("bmp.zig");
9	9	const parse = @import("parse.zig");
10	10	const lang = @import("lang.zig");
11		const CodePage = @import("code_pages.zig").CodePage;
	11	const code_pages = @import("code_pages.zig");
	12	const SupportedCodePage = code_pages.SupportedCodePage;
12	13	const builtin = @import("builtin");
13	14	const native_endian = builtin.cpu.arch.endian();
14	15
...	...	@@ -64,7 +65,7 @@ pub const Diagnostics = struct {
64	65	defer std.debug.unlockStdErr();
65	66	const stderr = std.io.getStdErr().writer();
66	67	for (self.errors.items) \|err_details\| {
67		renderErrorMessage(self.allocator, stderr, tty_config, cwd, err_details, source, self.strings.items, source_mappings) catch return;
	68	renderErrorMessage(stderr, tty_config, cwd, err_details, source, self.strings.items, source_mappings) catch return;
68	69	}
69	70	}
70	71
...	...	@@ -94,32 +95,22 @@ pub const Diagnostics = struct {
94	95	pub const DiagnosticsContext = struct {
95	96	diagnostics: *Diagnostics,
96	97	token: Token,
	98	/// Code page of the source file at the token location
	99	code_page: SupportedCodePage,
97	100	};
98	101
99	102	pub const ErrorDetails = struct {
100	103	err: Error,
101	104	token: Token,
	105	/// Code page of the source file at the token location
	106	code_page: SupportedCodePage,
102	107	/// If non-null, should be before `token`. If null, `token` is assumed to be the start.
103	108	token_span_start: ?Token = null,
104	109	/// If non-null, should be after `token`. If null, `token` is assumed to be the end.
105	110	token_span_end: ?Token = null,
106	111	type: Type = .err,
107	112	print_source_line: bool = true,
108		extra: union {
109		none: void,
110		expected: Token.Id,
111		number: u32,
112		expected_types: ExpectedTypes,
113		resource: rc.Resource,
114		string_and_language: StringAndLanguage,
115		file_open_error: FileOpenError,
116		icon_read_error: IconReadError,
117		icon_dir: IconDirContext,
118		bmp_read_error: BitmapReadError,
119		accelerator_error: AcceleratorError,
120		statement_with_u16_param: StatementWithU16Param,
121		menu_or_class: enum { class, menu },
122		} = .{ .none = {} },
	113	extra: Extra = .{ .none = {} },
123	114
124	115	pub const Type = enum {
125	116	/// Fatal error, stops compilation
...	...	@@ -137,9 +128,25 @@ pub const ErrorDetails = struct {
137	128	hint,
138	129	};
139	130
	131	pub const Extra = union {
	132	none: void,
	133	expected: Token.Id,
	134	number: u32,
	135	expected_types: ExpectedTypes,
	136	resource: rc.ResourceType,
	137	string_and_language: StringAndLanguage,
	138	file_open_error: FileOpenError,
	139	icon_read_error: IconReadError,
	140	icon_dir: IconDirContext,
	141	bmp_read_error: BitmapReadError,
	142	accelerator_error: AcceleratorError,
	143	statement_with_u16_param: StatementWithU16Param,
	144	menu_or_class: enum { class, menu },
	145	};
	146
140	147	comptime {
141	148	// all fields in the extra union should be 32 bits or less
142		for (std.meta.fields(std.meta.fieldInfo(ErrorDetails, .extra).type)) \|field\| {
	149	for (std.meta.fields(Extra)) \|field\| {
143	150	std.debug.assert(@bitSizeOf(field.type) <= 32);
144	151	}
145	152	}
...	...	@@ -321,6 +328,8 @@ pub const ErrorDetails = struct {
321	328	close_paren_expression,
322	329	unary_plus_expression,
323	330	rc_could_miscompile_control_params,
	331	dangling_literal_at_eof,
	332	disjoint_code_page,
324	333
325	334	// Compiler
326	335	/// `string_and_language` is populated
...	...	@@ -331,6 +340,7 @@ pub const ErrorDetails = struct {
331	340	/// `accelerator_error` is populated
332	341	invalid_accelerator_key,
333	342	accelerator_type_required,
	343	accelerator_shift_or_control_without_virtkey,
334	344	rc_would_miscompile_control_padding,
335	345	rc_would_miscompile_control_class_ordinal,
336	346	/// `icon_dir` is populated
...	...	@@ -356,11 +366,6 @@ pub const ErrorDetails = struct {
356	366	/// `number` is populated and contains a string index for which the string contains
357	367	/// the bytes of a `u64` (native endian). The `u64` contains the number of miscompiled bytes.
358	368	rc_would_miscompile_bmp_palette_padding,
359		/// `number` is populated and contains a string index for which the string contains
360		/// the bytes of two `u64`s (native endian). The first contains the number of missing
361		/// palette bytes and the second contains the max number of missing palette bytes.
362		/// If type is `.note`, then `extra` is `none`.
363		bmp_too_many_missing_palette_bytes,
364	369	resource_header_size_exceeds_max,
365	370	resource_data_size_exceeds_max,
366	371	control_extra_data_size_exceeds_max,
...	...	@@ -383,15 +388,16 @@ pub const ErrorDetails = struct {
383	388	rc_would_miscompile_dialog_menu_or_class_id_forced_ordinal,
384	389	rc_would_miscompile_dialog_menu_id_starts_with_digit,
385	390	dialog_menu_id_was_uppercased,
386		/// `menu_or_class` is populated and contains the type of the parameter statement
387		duplicate_menu_or_class_skipped,
	391	duplicate_optional_statement_skipped,
388	392	invalid_digit_character_in_ordinal,
389	393
390	394	// Literals
391	395	/// `number` is populated
392		rc_would_miscompile_codepoint_byte_swap,
	396	rc_would_miscompile_codepoint_whitespace,
393	397	/// `number` is populated
394	398	rc_would_miscompile_codepoint_skip,
	399	/// `number` is populated
	400	rc_would_miscompile_codepoint_bom,
395	401	tab_converted_to_spaces,
396	402
397	403	// General (used in various places)
...	...	@@ -403,10 +409,50 @@ pub const ErrorDetails = struct {
403	409	failed_to_open_cwd,
404	410	};
405	411
	412	fn formatToken(
	413	ctx: TokenFormatContext,
	414	comptime fmt: []const u8,
	415	options: std.fmt.FormatOptions,
	416	writer: anytype,
	417	) !void {
	418	_ = fmt;
	419	_ = options;
	420
	421	switch (ctx.token.id) {
	422	.eof => return writer.writeAll(ctx.token.id.nameForErrorDisplay()),
	423	else => {},
	424	}
	425
	426	const slice = ctx.token.slice(ctx.source);
	427	var src_i: usize = 0;
	428	while (src_i < slice.len) {
	429	const codepoint = ctx.code_page.codepointAt(src_i, slice) orelse break;
	430	defer src_i += codepoint.byte_len;
	431	const display_codepoint = codepointForDisplay(codepoint) orelse continue;
	432	var buf: [4]u8 = undefined;
	433	const utf8_len = std.unicode.utf8Encode(display_codepoint, &buf) catch unreachable;
	434	try writer.writeAll(buf[0..utf8_len]);
	435	}
	436	}
	437
	438	const TokenFormatContext = struct {
	439	token: Token,
	440	source: []const u8,
	441	code_page: SupportedCodePage,
	442	};
	443
	444	fn fmtToken(self: ErrorDetails, source: []const u8) std.fmt.Formatter(formatToken) {
	445	return .{ .data = .{
	446	.token = self.token,
	447	.code_page = self.code_page,
	448	.source = source,
	449	} };
	450	}
	451
406	452	pub fn render(self: ErrorDetails, writer: anytype, source: []const u8, strings: []const []const u8) !void {
407	453	switch (self.err) {
408	454	.unfinished_string_literal => {
409		return writer.print("unfinished string literal at '{s}', expected closing '\"'", .{self.token.nameForErrorDisplay(source)});
	455	return writer.print("unfinished string literal at '{s}', expected closing '\"'", .{self.fmtToken(source)});
410	456	},
411	457	.string_literal_too_long => {
412	458	return writer.print("string literal too long (max is currently {} characters)", .{self.extra.number});
...	...	@@ -474,33 +520,33 @@ pub const ErrorDetails = struct {
474	520	number_slice.len += 1;
475	521	}
476	522	const number = std.fmt.parseUnsigned(u16, number_slice, 10) catch unreachable;
477		const code_page = CodePage.getByIdentifier(number) catch unreachable;
	523	const code_page = code_pages.getByIdentifier(number) catch unreachable;
478	524	// TODO: Improve or maybe add a note making it more clear that the code page
479	525	// is valid and that the code page is unsupported purely due to a limitation
480	526	// in this compiler.
481	527	return writer.print("unsupported code page '{s} (id={})' in #pragma code_page", .{ @tagName(code_page), number });
482	528	},
483	529	.unfinished_raw_data_block => {
484		return writer.print("unfinished raw data block at '{s}', expected closing '}}' or 'END'", .{self.token.nameForErrorDisplay(source)});
	530	return writer.print("unfinished raw data block at '{s}', expected closing '}}' or 'END'", .{self.fmtToken(source)});
485	531	},
486	532	.unfinished_string_table_block => {
487		return writer.print("unfinished STRINGTABLE block at '{s}', expected closing '}}' or 'END'", .{self.token.nameForErrorDisplay(source)});
	533	return writer.print("unfinished STRINGTABLE block at '{s}', expected closing '}}' or 'END'", .{self.fmtToken(source)});
488	534	},
489	535	.expected_token => {
490		return writer.print("expected '{s}', got '{s}'", .{ self.extra.expected.nameForErrorDisplay(), self.token.nameForErrorDisplay(source) });
	536	return writer.print("expected '{s}', got '{s}'", .{ self.extra.expected.nameForErrorDisplay(), self.fmtToken(source) });
491	537	},
492	538	.expected_something_else => {
493	539	try writer.writeAll("expected ");
494	540	try self.extra.expected_types.writeCommaSeparated(writer);
495		return writer.print("; got '{s}'", .{self.token.nameForErrorDisplay(source)});
	541	return writer.print("; got '{s}'", .{self.fmtToken(source)});
496	542	},
497	543	.resource_type_cant_use_raw_data => switch (self.type) {
498		.err, .warning => try writer.print("expected '<filename>', found '{s}' (resource type '{s}' can't use raw data)", .{ self.token.nameForErrorDisplay(source), self.extra.resource.nameForErrorDisplay() }),
499		.note => try writer.print("if '{s}' is intended to be a filename, it must be specified as a quoted string literal", .{self.token.nameForErrorDisplay(source)}),
	544	.err, .warning => try writer.print("expected '<filename>', found '{s}' (resource type '{s}' can't use raw data)", .{ self.fmtToken(source), self.extra.resource.nameForErrorDisplay() }),
	545	.note => try writer.print("if '{s}' is intended to be a filename, it must be specified as a quoted string literal", .{self.fmtToken(source)}),
500	546	.hint => return,
501	547	},
502	548	.id_must_be_ordinal => {
503		try writer.print("id of resource type '{s}' must be an ordinal (u16), got '{s}'", .{ self.extra.resource.nameForErrorDisplay(), self.token.nameForErrorDisplay(source) });
	549	try writer.print("id of resource type '{s}' must be an ordinal (u16), got '{s}'", .{ self.extra.resource.nameForErrorDisplay(), self.fmtToken(source) });
504	550	},
505	551	.name_or_id_not_allowed => {
506	552	try writer.print("name or id is not allowed for resource type '{s}'", .{self.extra.resource.nameForErrorDisplay()});
...	...	@@ -516,7 +562,7 @@ pub const ErrorDetails = struct {
516	562	try writer.writeAll("ASCII character not equivalent to virtual key code");
517	563	},
518	564	.empty_menu_not_allowed => {
519		try writer.print("empty menu of type '{s}' not allowed", .{self.token.nameForErrorDisplay(source)});
	565	try writer.print("empty menu of type '{s}' not allowed", .{self.fmtToken(source)});
520	566	},
521	567	.rc_would_miscompile_version_value_padding => switch (self.type) {
522	568	.err, .warning => return writer.print("the padding before this quoted string value would be miscompiled by the Win32 RC compiler", .{}),
...	...	@@ -570,19 +616,18 @@ pub const ErrorDetails = struct {
570	616	.note => return writer.print("to avoid the potential miscompilation, consider adding a comma after the style parameter", .{}),
571	617	.hint => return,
572	618	},
	619	.dangling_literal_at_eof => {
	620	try writer.writeAll("dangling literal at end-of-file; this is not a problem, but it is likely a mistake");
	621	},
	622	.disjoint_code_page => switch (self.type) {
	623	.err, .warning => return writer.print("#pragma code_page as the first thing in the .rc script can cause the input and output code pages to become out-of-sync", .{}),
	624	.note => return writer.print("to avoid unexpected behavior, add a comment (or anything else) above the #pragma code_page line", .{}),
	625	.hint => return,
	626	},
573	627	.string_already_defined => switch (self.type) {
574	628	.err, .warning => {
575		const language_id = self.extra.string_and_language.language.asInt();
576		const language_name = language_name: {
577		if (std.meta.intToEnum(lang.LanguageId, language_id)) \|lang_enum_val\| {
578		break :language_name @tagName(lang_enum_val);
579		} else \|_\| {}
580		if (language_id == lang.LOCALE_CUSTOM_UNSPECIFIED) {
581		break :language_name "LOCALE_CUSTOM_UNSPECIFIED";
582		}
583		break :language_name "<UNKNOWN>";
584		};
585		return writer.print("string with id {d} (0x{X}) already defined for language {s} (0x{X})", .{ self.extra.string_and_language.id, self.extra.string_and_language.id, language_name, language_id });
	629	const language = self.extra.string_and_language.language;
	630	return writer.print("string with id {d} (0x{X}) already defined for language {}", .{ self.extra.string_and_language.id, self.extra.string_and_language.id, language });
586	631	},
587	632	.note => return writer.print("previous definition of string with id {d} (0x{X}) here", .{ self.extra.string_and_language.id, self.extra.string_and_language.id }),
588	633	.hint => return,
...	...	@@ -597,14 +642,17 @@ pub const ErrorDetails = struct {
597	642	try writer.print("unable to open file '{s}': {s}", .{ strings[self.extra.file_open_error.filename_string_index], @tagName(self.extra.file_open_error.err) });
598	643	},
599	644	.invalid_accelerator_key => {
600		try writer.print("invalid accelerator key '{s}': {s}", .{ self.token.nameForErrorDisplay(source), @tagName(self.extra.accelerator_error.err) });
	645	try writer.print("invalid accelerator key '{s}': {s}", .{ self.fmtToken(source), @tagName(self.extra.accelerator_error.err) });
601	646	},
602	647	.accelerator_type_required => {
603		try writer.print("accelerator type [ASCII or VIRTKEY] required when key is an integer", .{});
	648	try writer.writeAll("accelerator type [ASCII or VIRTKEY] required when key is an integer");
	649	},
	650	.accelerator_shift_or_control_without_virtkey => {
	651	try writer.writeAll("SHIFT or CONTROL used without VIRTKEY");
604	652	},
605	653	.rc_would_miscompile_control_padding => switch (self.type) {
606	654	.err, .warning => return writer.print("the padding before this control would be miscompiled by the Win32 RC compiler (it would insert 2 extra bytes of padding)", .{}),
607		.note => return writer.print("to avoid the potential miscompilation, consider removing any 'control data' blocks from the controls in this dialog", .{}),
	655	.note => return writer.print("to avoid the potential miscompilation, consider adding one more byte to the control data of the control preceding this one", .{}),
608	656	.hint => return,
609	657	},
610	658	.rc_would_miscompile_control_class_ordinal => switch (self.type) {
...	...	@@ -625,7 +673,7 @@ pub const ErrorDetails = struct {
625	673	try writer.print("resource with format '{s}' (at index {}) is not allowed in {s} resource groups", .{ @tagName(self.extra.icon_dir.icon_format), self.extra.icon_dir.index, @tagName(self.extra.icon_dir.icon_type) });
626	674	},
627	675	.icon_dir_and_resource_type_mismatch => {
628		const unexpected_type: rc.Resource = if (self.extra.resource == .icon) .cursor else .icon;
	676	const unexpected_type: rc.ResourceType = if (self.extra.resource == .icon) .cursor else .icon;
629	677	// TODO: Better wording
630	678	try writer.print("resource type '{s}' does not match type '{s}' specified in the file", .{ self.extra.resource.nameForErrorDisplay(), unexpected_type.nameForErrorDisplay() });
631	679	},
...	...	@@ -663,23 +711,15 @@ pub const ErrorDetails = struct {
663	711	.bmp_missing_palette_bytes => {
664	712	const bytes = strings[self.extra.number];
665	713	const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
666		try writer.print("bitmap has {d} missing color palette bytes which will be padded with zeroes", .{missing_bytes});
	714	try writer.print("bitmap has {d} missing color palette bytes", .{missing_bytes});
667	715	},
668	716	.rc_would_miscompile_bmp_palette_padding => {
669		const bytes = strings[self.extra.number];
670		const miscompiled_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
671		try writer.print("the missing color palette bytes would be miscompiled by the Win32 RC compiler (the added padding bytes would include {d} bytes of the pixel data)", .{miscompiled_bytes});
672		},
673		.bmp_too_many_missing_palette_bytes => switch (self.type) {
674		.err, .warning => {
	717	try writer.writeAll("the Win32 RC compiler would erroneously pad out the missing bytes");
	718	if (self.extra.number != 0) {
675	719	const bytes = strings[self.extra.number];
676		const missing_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
677		const max_missing_bytes = std.mem.readInt(u64, bytes[8..16], native_endian);
678		try writer.print("bitmap has {} missing color palette bytes which exceeds the maximum of {}", .{ missing_bytes, max_missing_bytes });
679		},
680		// TODO: command line option
681		.note => try writer.writeAll("the maximum number of missing color palette bytes is configurable via <<TODO command line option>>"),
682		.hint => return,
	720	const miscompiled_bytes = std.mem.readInt(u64, bytes[0..8], native_endian);
	721	try writer.print(" (and the added padding bytes would include {d} bytes of the pixel data)", .{miscompiled_bytes});
	722	}
683	723	},
684	724	.resource_header_size_exceeds_max => {
685	725	try writer.print("resource's header length exceeds maximum of {} bytes", .{std.math.maxInt(u32)});
...	...	@@ -749,23 +789,22 @@ pub const ErrorDetails = struct {
749	789	.hint => return,
750	790	},
751	791	.dialog_menu_id_was_uppercased => return,
752		.duplicate_menu_or_class_skipped => {
753		return writer.print("this {s} was ignored; when multiple {s} statements are specified, only the last takes precedence", .{
754		@tagName(self.extra.menu_or_class),
755		@tagName(self.extra.menu_or_class),
756		});
	792	.duplicate_optional_statement_skipped => {
	793	return writer.writeAll("this statement was ignored; when multiple statements of the same type are specified, only the last takes precedence");
757	794	},
758	795	.invalid_digit_character_in_ordinal => {
759	796	return writer.writeAll("non-ASCII digit characters are not allowed in ordinal (number) values");
760	797	},
761		.rc_would_miscompile_codepoint_byte_swap => switch (self.type) {
762		.err, .warning => return writer.print("codepoint U+{X} within a string literal would be miscompiled by the Win32 RC compiler (the bytes of the UTF-16 code unit would be swapped)", .{self.extra.number}),
763		.note => return writer.print("to avoid the potential miscompilation, an integer escape sequence in a wide string literal could be used instead: L\"\\x{X}\"", .{self.extra.number}),
764		.hint => return,
	798	.rc_would_miscompile_codepoint_whitespace => {
	799	const treated_as = self.extra.number >> 8;
	800	return writer.print("codepoint U+{X:0>4} within a string literal would be miscompiled by the Win32 RC compiler (it would get treated as U+{X:0>4})", .{ self.extra.number, treated_as });
765	801	},
766		.rc_would_miscompile_codepoint_skip => switch (self.type) {
767		.err, .warning => return writer.print("codepoint U+{X} within a string literal would be miscompiled by the Win32 RC compiler (the codepoint would be missing from the compiled resource)", .{self.extra.number}),
768		.note => return writer.print("to avoid the potential miscompilation, an integer escape sequence in a wide string literal could be used instead: L\"\\x{X}\"", .{self.extra.number}),
	802	.rc_would_miscompile_codepoint_skip => {
	803	return writer.print("codepoint U+{X:0>4} within a string literal would be miscompiled by the Win32 RC compiler (the codepoint would be missing from the compiled resource)", .{self.extra.number});
	804	},
	805	.rc_would_miscompile_codepoint_bom => switch (self.type) {
	806	.err, .warning => return writer.print("codepoint U+{X:0>4} within a string literal would cause the entire file to be miscompiled by the Win32 RC compiler", .{self.extra.number}),
	807	.note => return writer.writeAll("the presence of this codepoint causes all non-ASCII codepoints to be byteswapped by the Win32 RC preprocessor"),
769	808	.hint => return,
770	809	},
771	810	.tab_converted_to_spaces => switch (self.type) {
...	...	@@ -790,14 +829,7 @@ pub const ErrorDetails = struct {
790	829	after_len: usize,
791	830	};
792	831
793		pub fn visualTokenInfo(self: ErrorDetails, source_line_start: usize, source_line_end: usize) VisualTokenInfo {
794		// Note: A perfect solution here would involve full grapheme cluster
795		// awareness, but oh well. This will give incorrect offsets
796		// if there are any multibyte codepoints within the relevant span,
797		// and even more inflated for grapheme clusters.
798		//
799		// We mitigate this slightly when we know we'll be pointing at
800		// something that displays as 1 character.
	832	pub fn visualTokenInfo(self: ErrorDetails, source_line_start: usize, source_line_end: usize, source: []const u8) VisualTokenInfo {
801	833	return switch (self.err) {
802	834	// These can technically be more than 1 byte depending on encoding,
803	835	// but they always refer to one visual character/grapheme.
...	...	@@ -808,27 +840,65 @@ pub const ErrorDetails = struct {
808	840	.illegal_private_use_character,
809	841	=> .{
810	842	.before_len = 0,
811		.point_offset = self.token.start - source_line_start,
	843	.point_offset = cellCount(self.code_page, source, source_line_start, self.token.start),
812	844	.after_len = 0,
813	845	},
814	846	else => .{
815	847	.before_len = before: {
816	848	const start = @max(source_line_start, if (self.token_span_start) \|span_start\| span_start.start else self.token.start);
817		break :before self.token.start - start;
	849	break :before cellCount(self.code_page, source, start, self.token.start);
818	850	},
819		.point_offset = self.token.start - source_line_start,
	851	.point_offset = cellCount(self.code_page, source, source_line_start, self.token.start),
820	852	.after_len = after: {
821	853	const end = @min(source_line_end, if (self.token_span_end) \|span_end\| span_end.end else self.token.end);
822	854	// end may be less than start when pointing to EOF
823	855	if (end <= self.token.start) break :after 0;
824		break :after end - self.token.start - 1;
	856	break :after cellCount(self.code_page, source, self.token.start, end) - 1;
825	857	},
826	858	},
827	859	};
828	860	}
829	861	};
830	862
831		pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_config: std.io.tty.Config, cwd: std.fs.Dir, err_details: ErrorDetails, source: []const u8, strings: []const []const u8, source_mappings: ?SourceMappings) !void {
	863	/// Convenience struct only useful when the code page can be inferred from the token
	864	pub const ErrorDetailsWithoutCodePage = blk: {
	865	const details_info = @typeInfo(ErrorDetails);
	866	const fields = details_info.@"struct".fields;
	867	var fields_without_codepage: [fields.len - 1]std.builtin.Type.StructField = undefined;
	868	var i: usize = 0;
	869	for (fields) \|field\| {
	870	if (std.mem.eql(u8, field.name, "code_page")) continue;
	871	fields_without_codepage[i] = field;
	872	i += 1;
	873	}
	874	std.debug.assert(i == fields_without_codepage.len);
	875	break :blk @Type(.{ .@"struct" = .{
	876	.layout = .auto,
	877	.fields = &fields_without_codepage,
	878	.decls = &.{},
	879	.is_tuple = false,
	880	} });
	881	};
	882
	883	fn cellCount(code_page: SupportedCodePage, source: []const u8, start_index: usize, end_index: usize) usize {
	884	// Note: This is an imperfect solution. A proper implementation here would
	885	// involve full grapheme cluster awareness + grapheme width data, but oh well.
	886	var codepoint_count: usize = 0;
	887	var index: usize = start_index;
	888	while (index < end_index) {
	889	const codepoint = code_page.codepointAt(index, source) orelse break;
	890	defer index += codepoint.byte_len;
	891	_ = codepointForDisplay(codepoint) orelse continue;
	892	codepoint_count += 1;
	893	// no need to count more than we will display
	894	if (codepoint_count >= max_source_line_codepoints + truncated_str.len) break;
	895	}
	896	return codepoint_count;
	897	}
	898
	899	const truncated_str = "<...truncated...>";
	900
	901	pub fn renderErrorMessage(writer: anytype, tty_config: std.io.tty.Config, cwd: std.fs.Dir, err_details: ErrorDetails, source: []const u8, strings: []const []const u8, source_mappings: ?SourceMappings) !void {
832	902	if (err_details.type == .hint) return;
833	903
834	904	const source_line_start = err_details.token.getLineStartForErrorDisplay(source);
...	...	@@ -884,45 +954,61 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con
884	954	}
885	955
886	956	const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);
887		const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len);
	957	const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len, source);
	958	const truncated_visual_info = ErrorDetails.VisualTokenInfo{
	959	.before_len = if (visual_info.point_offset > max_source_line_codepoints and visual_info.before_len > 0)
	960	(visual_info.before_len + 1) -\| (visual_info.point_offset - max_source_line_codepoints)
	961	else
	962	visual_info.before_len,
	963	.point_offset = @min(max_source_line_codepoints + 1, visual_info.point_offset),
	964	.after_len = if (visual_info.point_offset > max_source_line_codepoints)
	965	@min(truncated_str.len - 3, visual_info.after_len)
	966	else
	967	@min(max_source_line_codepoints - visual_info.point_offset + (truncated_str.len - 2), visual_info.after_len),
	968	};
888	969
889	970	// Need this to determine if the 'line originated from' note is worth printing
890		var source_line_for_display_buf = try std.ArrayList(u8).initCapacity(allocator, source_line.len);
891		defer source_line_for_display_buf.deinit();
892		try writeSourceSlice(source_line_for_display_buf.writer(), source_line);
893
894		// TODO: General handling of long lines, not tied to this specific error
895		if (err_details.err == .string_literal_too_long) {
896		const before_slice = source_line[0..@min(source_line.len, visual_info.point_offset + 16)];
897		try writeSourceSlice(writer, before_slice);
	971	var source_line_for_display_buf: [max_source_line_bytes]u8 = undefined;
	972	const source_line_for_display = writeSourceSlice(&source_line_for_display_buf, source_line, err_details.code_page);
	973
	974	try writer.writeAll(source_line_for_display.line);
	975	if (source_line_for_display.truncated) {
898	976	try tty_config.setColor(writer, .dim);
899		try writer.writeAll("<...truncated...>");
	977	try writer.writeAll(truncated_str);
900	978	try tty_config.setColor(writer, .reset);
901		} else {
902		try writer.writeAll(source_line_for_display_buf.items);
903	979	}
904	980	try writer.writeByte('\n');
905	981
906	982	try tty_config.setColor(writer, .green);
907		const num_spaces = visual_info.point_offset - visual_info.before_len;
	983	const num_spaces = truncated_visual_info.point_offset - truncated_visual_info.before_len;
908	984	try writer.writeByteNTimes(' ', num_spaces);
909		try writer.writeByteNTimes('~', visual_info.before_len);
	985	try writer.writeByteNTimes('~', truncated_visual_info.before_len);
910	986	try writer.writeByte('^');
911		if (visual_info.after_len > 0) {
912		var num_squiggles = visual_info.after_len;
913		if (err_details.err == .string_literal_too_long) {
914		num_squiggles = @min(num_squiggles, 15);
915		}
916		try writer.writeByteNTimes('~', num_squiggles);
917		}
	987	try writer.writeByteNTimes('~', truncated_visual_info.after_len);
918	988	try writer.writeByte('\n');
919	989	try tty_config.setColor(writer, .reset);
920	990
921	991	if (corresponding_span != null and corresponding_file != null) {
922		var corresponding_lines = try CorrespondingLines.init(allocator, cwd, err_details, source_line_for_display_buf.items, corresponding_span.?, corresponding_file.?);
923		defer corresponding_lines.deinit(allocator);
924
925		if (!corresponding_lines.worth_printing_note) return;
	992	var worth_printing_lines: bool = true;
	993	var initial_lines_err: ?anyerror = null;
	994	var corresponding_lines: ?CorrespondingLines = CorrespondingLines.init(
	995	cwd,
	996	err_details,
	997	source_line_for_display.line,
	998	corresponding_span.?,
	999	corresponding_file.?,
	1000	) catch \|err\| switch (err) {
	1001	error.NotWorthPrintingLines => blk: {
	1002	worth_printing_lines = false;
	1003	break :blk null;
	1004	},
	1005	error.NotWorthPrintingNote => return,
	1006	else => \|e\| blk: {
	1007	initial_lines_err = e;
	1008	break :blk null;
	1009	},
	1010	};
	1011	defer if (corresponding_lines) \|*cl\| cl.deinit();
926	1012
927	1013	try tty_config.setColor(writer, .bold);
928	1014	if (corresponding_file) \|file\| {
...	...	@@ -947,85 +1033,222 @@ pub fn renderErrorMessage(allocator: std.mem.Allocator, writer: anytype, tty_con
947	1033	try writer.print(" of file '{s}'\n", .{corresponding_file.?});
948	1034	try tty_config.setColor(writer, .reset);
949	1035
950		if (!corresponding_lines.worth_printing_lines) return;
951
952		if (corresponding_lines.lines_is_error_message) {
	1036	if (!worth_printing_lines) return;
	1037
	1038	const write_lines_err: ?anyerror = write_lines: {
	1039	if (initial_lines_err) \|err\| break :write_lines err;
	1040	while (corresponding_lines.?.next() catch \|err\| {
	1041	break :write_lines err;
	1042	}) \|display_line\| {
	1043	try writer.writeAll(display_line.line);
	1044	if (display_line.truncated) {
	1045	try tty_config.setColor(writer, .dim);
	1046	try writer.writeAll(truncated_str);
	1047	try tty_config.setColor(writer, .reset);
	1048	}
	1049	try writer.writeByte('\n');
	1050	}
	1051	break :write_lines null;
	1052	};
	1053	if (write_lines_err) \|err\| {
953	1054	try tty_config.setColor(writer, .red);
954	1055	try writer.writeAll(" \| ");
955	1056	try tty_config.setColor(writer, .reset);
956	1057	try tty_config.setColor(writer, .dim);
957		try writer.writeAll(corresponding_lines.lines.items);
	1058	try writer.print("unable to print line(s) from file: {s}\n", .{@errorName(err)});
958	1059	try tty_config.setColor(writer, .reset);
959		try writer.writeAll("\n\n");
960		return;
961	1060	}
962
963		try writer.writeAll(corresponding_lines.lines.items);
964		try writer.writeAll("\n\n");
	1061	try writer.writeByte('\n');
965	1062	}
966	1063	}
967	1064
968		const CorrespondingLines = struct {
969		worth_printing_note: bool = true,
970		worth_printing_lines: bool = true,
971		lines: std.ArrayListUnmanaged(u8) = .empty,
972		lines_is_error_message: bool = false,
973
974		pub fn init(allocator: std.mem.Allocator, cwd: std.fs.Dir, err_details: ErrorDetails, lines_for_comparison: []const u8, corresponding_span: SourceMappings.CorrespondingSpan, corresponding_file: []const u8) !CorrespondingLines {
975		var corresponding_lines = CorrespondingLines{};
	1065	const VisualLine = struct {
	1066	line: []u8,
	1067	truncated: bool,
	1068	};
976	1069
	1070	const CorrespondingLines = struct {
	1071	// enough room for one more codepoint, just so that we don't have to keep
	1072	// track of this being truncated, since the extra codepoint will ensure
	1073	// the visual line will need to truncate in that case.
	1074	line_buf: [max_source_line_bytes + 4]u8 = undefined,
	1075	line_len: usize = 0,
	1076	visual_line_buf: [max_source_line_bytes]u8 = undefined,
	1077	visual_line_len: usize = 0,
	1078	truncated: bool = false,
	1079	line_num: usize = 1,
	1080	initial_line: bool = true,
	1081	last_byte: u8 = 0,
	1082	at_eof: bool = false,
	1083	span: SourceMappings.CorrespondingSpan,
	1084	file: std.fs.File,
	1085	buffered_reader: BufferedReaderType,
	1086	code_page: SupportedCodePage,
	1087
	1088	const BufferedReaderType = std.io.BufferedReader(512, std.fs.File.Reader);
	1089
	1090	pub fn init(cwd: std.fs.Dir, err_details: ErrorDetails, line_for_comparison: []const u8, corresponding_span: SourceMappings.CorrespondingSpan, corresponding_file: []const u8) !CorrespondingLines {
977	1091	// We don't do line comparison for this error, so don't print the note if the line
978	1092	// number is different
979		if (err_details.err == .string_literal_too_long and err_details.token.line_number == corresponding_span.start_line) {
980		corresponding_lines.worth_printing_note = false;
981		return corresponding_lines;
	1093	if (err_details.err == .string_literal_too_long and err_details.token.line_number != corresponding_span.start_line) {
	1094	return error.NotWorthPrintingNote;
982	1095	}
983	1096
984	1097	// Don't print the originating line for this error, we know it's really long
985	1098	if (err_details.err == .string_literal_too_long) {
986		corresponding_lines.worth_printing_lines = false;
987		return corresponding_lines;
	1099	return error.NotWorthPrintingLines;
988	1100	}
989	1101
990		var writer = corresponding_lines.lines.writer(allocator);
991		if (utils.openFileNotDir(cwd, corresponding_file, .{})) \|file\| {
992		defer file.close();
993		var buffered_reader = std.io.bufferedReader(file.reader());
994		writeLinesFromStream(writer, buffered_reader.reader(), corresponding_span.start_line, corresponding_span.end_line) catch \|err\| switch (err) {
995		error.LinesNotFound => {
996		corresponding_lines.lines.clearRetainingCapacity();
997		try writer.print("unable to print line(s) from file: {s}", .{@errorName(err)});
998		corresponding_lines.lines_is_error_message = true;
999		return corresponding_lines;
1000		},
1001		else => \|e\| return e,
1002		};
1003		} else \|err\| {
1004		corresponding_lines.lines.clearRetainingCapacity();
1005		try writer.print("unable to print line(s) from file: {s}", .{@errorName(err)});
1006		corresponding_lines.lines_is_error_message = true;
1007		return corresponding_lines;
1008		}
	1102	var corresponding_lines = CorrespondingLines{
	1103	.span = corresponding_span,
	1104	.file = try utils.openFileNotDir(cwd, corresponding_file, .{}),
	1105	.buffered_reader = undefined,
	1106	.code_page = err_details.code_page,
	1107	};
	1108	corresponding_lines.buffered_reader = BufferedReaderType{
	1109	.unbuffered_reader = corresponding_lines.file.reader(),
	1110	};
	1111	errdefer corresponding_lines.deinit();
	1112
	1113	var fbs = std.io.fixedBufferStream(&corresponding_lines.line_buf);
	1114	const writer = fbs.writer();
	1115
	1116	try corresponding_lines.writeLineFromStreamVerbatim(
	1117	writer,
	1118	corresponding_lines.buffered_reader.reader(),
	1119	corresponding_span.start_line,
	1120	);
	1121
	1122	const visual_line = writeSourceSlice(
	1123	&corresponding_lines.visual_line_buf,
	1124	corresponding_lines.line_buf[0..corresponding_lines.line_len],
	1125	err_details.code_page,
	1126	);
	1127	corresponding_lines.visual_line_len = visual_line.line.len;
	1128	corresponding_lines.truncated = visual_line.truncated;
1009	1129
1010	1130	// If the lines are the same as they were before preprocessing, skip printing the note entirely
1011		if (std.mem.eql(u8, lines_for_comparison, corresponding_lines.lines.items)) {
1012		corresponding_lines.worth_printing_note = false;
	1131	if (corresponding_span.start_line == corresponding_span.end_line and std.mem.eql(
	1132	u8,
	1133	line_for_comparison,
	1134	corresponding_lines.visual_line_buf[0..corresponding_lines.visual_line_len],
	1135	)) {
	1136	return error.NotWorthPrintingNote;
1013	1137	}
	1138
1014	1139	return corresponding_lines;
1015	1140	}
1016	1141
1017		pub fn deinit(self: *CorrespondingLines, allocator: std.mem.Allocator) void {
1018		self.lines.deinit(allocator);
	1142	pub fn next(self: *CorrespondingLines) !?VisualLine {
	1143	if (self.initial_line) {
	1144	self.initial_line = false;
	1145	return .{
	1146	.line = self.visual_line_buf[0..self.visual_line_len],
	1147	.truncated = self.truncated,
	1148	};
	1149	}
	1150	if (self.line_num > self.span.end_line) return null;
	1151	if (self.at_eof) return error.LinesNotFound;
	1152
	1153	self.line_len = 0;
	1154	self.visual_line_len = 0;
	1155
	1156	var fbs = std.io.fixedBufferStream(&self.line_buf);
	1157	const writer = fbs.writer();
	1158
	1159	try self.writeLineFromStreamVerbatim(
	1160	writer,
	1161	self.buffered_reader.reader(),
	1162	self.line_num,
	1163	);
	1164
	1165	const visual_line = writeSourceSlice(
	1166	&self.visual_line_buf,
	1167	self.line_buf[0..self.line_len],
	1168	self.code_page,
	1169	);
	1170	self.visual_line_len = visual_line.line.len;
	1171
	1172	return visual_line;
	1173	}
	1174
	1175	fn writeLineFromStreamVerbatim(self: *CorrespondingLines, writer: anytype, input: anytype, line_num: usize) !void {
	1176	while (try readByteOrEof(input)) \|byte\| {
	1177	switch (byte) {
	1178	'\n', '\r' => {
	1179	if (!utils.isLineEndingPair(self.last_byte, byte)) {
	1180	const line_complete = self.line_num == line_num;
	1181	self.line_num += 1;
	1182	if (line_complete) {
	1183	self.last_byte = byte;
	1184	return;
	1185	}
	1186	} else {
	1187	// reset last_byte to a non-line ending so that
	1188	// consecutive CRLF pairs don't get treated as one
	1189	// long line ending 'pair'
	1190	self.last_byte = 0;
	1191	continue;
	1192	}
	1193	},
	1194	else => {
	1195	if (self.line_num == line_num) {
	1196	if (writer.writeByte(byte)) {
	1197	self.line_len += 1;
	1198	} else \|err\| switch (err) {
	1199	error.NoSpaceLeft => {},
	1200	else => \|e\| return e,
	1201	}
	1202	}
	1203	},
	1204	}
	1205	self.last_byte = byte;
	1206	}
	1207	self.at_eof = true;
	1208	// hacky way to get next to return null
	1209	self.line_num += 1;
	1210	}
	1211
	1212	fn readByteOrEof(reader: anytype) !?u8 {
	1213	return reader.readByte() catch \|err\| switch (err) {
	1214	error.EndOfStream => return null,
	1215	else => \|e\| return e,
	1216	};
	1217	}
	1218
	1219	pub fn deinit(self: *CorrespondingLines) void {
	1220	self.file.close();
1019	1221	}
1020	1222	};
1021	1223
1022		fn writeSourceSlice(writer: anytype, slice: []const u8) !void {
1023		for (slice) \|c\| try writeSourceByte(writer, c);
	1224	const max_source_line_codepoints = 120;
	1225	const max_source_line_bytes = max_source_line_codepoints * 4;
	1226
	1227	fn writeSourceSlice(buf: []u8, slice: []const u8, code_page: SupportedCodePage) VisualLine {
	1228	var src_i: usize = 0;
	1229	var dest_i: usize = 0;
	1230	var codepoint_count: usize = 0;
	1231	while (src_i < slice.len) {
	1232	const codepoint = code_page.codepointAt(src_i, slice) orelse break;
	1233	defer src_i += codepoint.byte_len;
	1234	const display_codepoint = codepointForDisplay(codepoint) orelse continue;
	1235	codepoint_count += 1;
	1236	if (codepoint_count > max_source_line_codepoints) {
	1237	return .{ .line = buf[0..dest_i], .truncated = true };
	1238	}
	1239	const utf8_len = std.unicode.utf8Encode(display_codepoint, buf[dest_i..]) catch unreachable;
	1240	dest_i += utf8_len;
	1241	}
	1242	return .{ .line = buf[0..dest_i], .truncated = false };
1024	1243	}
1025	1244
1026		inline fn writeSourceByte(writer: anytype, byte: u8) !void {
1027		switch (byte) {
1028		'\x00'...'\x08', '\x0E'...'\x1F', '\x7F' => try writer.writeAll("�"),
	1245	fn codepointForDisplay(codepoint: code_pages.Codepoint) ?u21 {
	1246	return switch (codepoint.value) {
	1247	'\x00'...'\x08',
	1248	'\x0E'...'\x1F',
	1249	'\x7F',
	1250	code_pages.Codepoint.invalid,
	1251	=> '�',
1029	1252	// \r is seemingly ignored by the RC compiler so skipping it when printing source lines
1030	1253	// could help avoid confusing output (e.g. RC\rDATA if printed verbatim would show up
1031	1254	// in the console as DATA but the compiler reads it as RCDATA)
...	...	@@ -1033,44 +1256,8 @@ inline fn writeSourceByte(writer: anytype, byte: u8) !void {
1033	1256	// NOTE: This is irrelevant when using the clang preprocessor, because unpaired \r
1034	1257	// characters get converted to \n, but may become relevant if another
1035	1258	// preprocessor is used instead.
1036		'\r' => {},
1037		'\t', '\x0B', '\x0C' => try writer.writeByte(' '),
1038		else => try writer.writeByte(byte),
1039		}
1040		}
1041
1042		pub fn writeLinesFromStream(writer: anytype, input: anytype, start_line: usize, end_line: usize) !void {
1043		var line_num: usize = 1;
1044		var last_byte: u8 = 0;
1045		while (try readByteOrEof(input)) \|byte\| {
1046		switch (byte) {
1047		'\n', '\r' => {
1048		if (!utils.isLineEndingPair(last_byte, byte)) {
1049		if (line_num == end_line) return;
1050		if (line_num >= start_line) try writeSourceByte(writer, byte);
1051		line_num += 1;
1052		} else {
1053		// reset last_byte to a non-line ending so that
1054		// consecutive CRLF pairs don't get treated as one
1055		// long line ending 'pair'
1056		last_byte = 0;
1057		continue;
1058		}
1059		},
1060		else => {
1061		if (line_num >= start_line) try writeSourceByte(writer, byte);
1062		},
1063		}
1064		last_byte = byte;
1065		}
1066		if (line_num != end_line) {
1067		return error.LinesNotFound;
1068		}
1069		}
1070
1071		pub fn readByteOrEof(reader: anytype) !?u8 {
1072		return reader.readByte() catch \|err\| switch (err) {
1073		error.EndOfStream => return null,
1074		else => \|e\| return e,
	1259	'\r' => null,
	1260	'\t', '\x0B', '\x0C' => ' ',
	1261	else => \|v\| v,
1075	1262	};
1076	1263	}

lib/compiler/resinator/lang.zig+1-1

...	...	@@ -119,7 +119,7 @@ test tagToId {
119	119	}
120	120
121	121	test "exhaustive tagToId" {
122		inline for (@typeInfo(LanguageId).Enum.fields) \|field\| {
	122	inline for (@typeInfo(LanguageId).@"enum".fields) \|field\| {
123	123	const id = tagToId(field.name) catch \|err\| {
124	124	std.debug.print("tag: {s}\n", .{field.name});
125	125	return err;

lib/compiler/resinator/lex.zig+134-113

...	...	@@ -8,7 +8,7 @@ const std = @import("std");
8	8	const ErrorDetails = @import("errors.zig").ErrorDetails;
9	9	const columnWidth = @import("literals.zig").columnWidth;
10	10	const code_pages = @import("code_pages.zig");
11		const CodePage = code_pages.CodePage;
	11	const SupportedCodePage = code_pages.SupportedCodePage;
12	12	const SourceMappings = @import("source_mapping.zig").SourceMappings;
13	13	const isNonAsciiDigit = @import("utils.zig").isNonAsciiDigit;
14	14
...	...	@@ -62,13 +62,6 @@ pub const Token = struct {
62	62	return buffer[self.start..self.end];
63	63	}
64	64
65		pub fn nameForErrorDisplay(self: Token, buffer: []const u8) []const u8 {
66		return switch (self.id) {
67		.eof => self.id.nameForErrorDisplay(),
68		else => self.slice(buffer),
69		};
70		}
71
72	65	/// Returns 0-based column
73	66	pub fn calculateColumn(token: Token, source: []const u8, tab_columns: usize, maybe_line_start: ?usize) usize {
74	67	const line_start = maybe_line_start orelse token.getLineStartForColumnCalc(source);
...	...	@@ -214,18 +207,19 @@ pub const Lexer = struct {
214	207	line_handler: LineHandler,
215	208	at_start_of_line: bool = true,
216	209	error_context_token: ?Token = null,
217		current_code_page: CodePage,
218		default_code_page: CodePage,
	210	current_code_page: SupportedCodePage,
	211	default_code_page: SupportedCodePage,
219	212	source_mappings: ?*SourceMappings,
220	213	max_string_literal_codepoints: u15,
221	214	/// Needed to determine whether or not the output code page should
222	215	/// be set in the parser.
223	216	seen_pragma_code_pages: u2 = 0,
	217	last_pragma_code_page_token: ?Token = null,
224	218
225	219	pub const Error = LexError;
226	220
227	221	pub const LexerOptions = struct {
228		default_code_page: CodePage = .windows1252,
	222	default_code_page: SupportedCodePage = .windows1252,
229	223	source_mappings: ?*SourceMappings = null,
230	224	max_string_literal_codepoints: u15 = default_max_string_literal_codepoints,
231	225	};
...	...	@@ -291,6 +285,8 @@ pub const Lexer = struct {
291	285	},
292	286	// NBSP only counts as whitespace at the start of a line (but
293	287	// can be intermixed with other whitespace). Who knows why.
	288	// TODO: This should either be removed, or it should also include
	289	// the codepoints listed in disjoint_code_page.zig
294	290	'\xA0' => if (self.at_start_of_line) {
295	291	result.start = self.index + codepoint.byte_len;
296	292	} else {
...	...	@@ -305,12 +301,8 @@ pub const Lexer = struct {
305	301	}
306	302	self.at_start_of_line = false;
307	303	},
308		// Semi-colon acts as a line-terminator, but in this lexing mode
309		// that's only true if it's at the start of a line.
310	304	';' => {
311		if (self.at_start_of_line) {
312		state = .semicolon;
313		}
	305	state = .semicolon;
314	306	self.at_start_of_line = false;
315	307	},
316	308	else => {
...	...	@@ -345,7 +337,11 @@ pub const Lexer = struct {
345	337	}
346	338	} else { // got EOF
347	339	switch (state) {
348		.start, .semicolon => {},
	340	.start => {},
	341	.semicolon => {
	342	// Skip past everything up to the EOF
	343	result.start = self.index;
	344	},
349	345	.literal => {
350	346	result.id = .literal;
351	347	},
...	...	@@ -357,6 +353,10 @@ pub const Lexer = struct {
357	353	}
358	354
359	355	result.end = self.index;
	356
	357	// EOF tokens must have their start index match the end index
	358	std.debug.assert(result.id != .eof or result.start == result.end);
	359
360	360	return result;
361	361	}
362	362
...	...	@@ -796,7 +796,11 @@ pub const Lexer = struct {
796	796	}
797	797	} else { // got EOF
798	798	switch (state) {
799		.start, .semicolon => {},
	799	.start => {},
	800	.semicolon => {
	801	// Skip past everything up to the EOF
	802	result.start = self.index;
	803	},
800	804	.literal_or_quoted_wide_string, .literal, .e, .en, .b, .be, .beg, .begi => {
801	805	result.id = .literal;
802	806	},
...	...	@@ -835,6 +839,9 @@ pub const Lexer = struct {
835	839	}
836	840	}
837	841
	842	// EOF tokens must have their start index match the end index
	843	std.debug.assert(result.id != .eof or result.start == result.end);
	844
838	845	return result;
839	846	}
840	847
...	...	@@ -878,7 +885,7 @@ pub const Lexer = struct {
878	885	// and miscompilations when used within string literals. We avoid the miscompilation
879	886	// within string literals and emit a warning, but outside of string literals it makes
880	887	// more sense to just disallow these codepoints.
881		0x900, 0xA00, 0xA0D, 0x2000, 0xFFFE, 0xD00 => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return,
	888	0x900, 0xA00, 0xA0D, 0x2000, 0xD00, 0xFFFE, 0xFFFF => if (!in_string_literal) error.IllegalCodepointOutsideStringLiterals else return,
882	889	else => return,
883	890	};
884	891	self.error_context_token = .{
...	...	@@ -899,90 +906,11 @@ pub const Lexer = struct {
899	906	};
900	907	errdefer self.error_context_token = token;
901	908	const full_command = self.buffer[start..end];
902		var command = full_command;
903
904		// Anything besides exactly this is ignored by the Windows RC implementation
905		const expected_directive = "#pragma";
906		if (!std.mem.startsWith(u8, command, expected_directive)) return;
907		command = command[expected_directive.len..];
908
909		if (command.len == 0 or !std.ascii.isWhitespace(command[0])) return;
910		while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
911		command = command[1..];
912		}
913
914		// Note: CoDe_PaGeZ is also treated as "code_page" by the Windows RC implementation,
915		// and it will error with 'Missing left parenthesis in code_page #pragma'
916		const expected_extension = "code_page";
917		if (!std.ascii.startsWithIgnoreCase(command, expected_extension)) return;
918		command = command[expected_extension.len..];
919
920		while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
921		command = command[1..];
922		}
923
924		if (command.len == 0 or command[0] != '(') {
925		return error.CodePagePragmaMissingLeftParen;
926		}
927		command = command[1..];
928	909
929		while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
930		command = command[1..];
931		}
932
933		var num_str: []u8 = command[0..0];
934		while (command.len > 0 and (command[0] != ')' and !std.ascii.isWhitespace(command[0]))) {
935		command = command[1..];
936		num_str.len += 1;
937		}
938
939		if (num_str.len == 0) {
940		return error.CodePagePragmaNotInteger;
941		}
942
943		while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
944		command = command[1..];
945		}
946
947		if (command.len == 0 or command[0] != ')') {
948		return error.CodePagePragmaMissingRightParen;
949		}
950
951		const code_page = code_page: {
952		if (std.ascii.eqlIgnoreCase("DEFAULT", num_str)) {
953		break :code_page self.default_code_page;
954		}
955
956		// The Win32 compiler behaves fairly strangely around maxInt(u32):
957		// - If the overflowed u32 wraps and becomes a known code page ID, then
958		// it will error/warn with "Codepage not valid: ignored" (depending on /w)
959		// - If the overflowed u32 wraps and does not become a known code page ID,
960		// then it will error with 'constant too big' and 'Codepage not integer'
961		//
962		// Instead of that, we just have a separate error specifically for overflow.
963		const num = parseCodePageNum(num_str) catch \|err\| switch (err) {
964		error.InvalidCharacter => return error.CodePagePragmaNotInteger,
965		error.Overflow => return error.CodePagePragmaOverflow,
966		};
967
968		// Anything that starts with 0 but does not resolve to 0 is treated as invalid, e.g. 01252
969		if (num_str[0] == '0' and num != 0) {
970		return error.CodePagePragmaInvalidCodePage;
971		}
972		// Anything that resolves to 0 is treated as 'not an integer' by the Win32 implementation.
973		else if (num == 0) {
974		return error.CodePagePragmaNotInteger;
975		}
976		// Anything above u16 max is not going to be found since our CodePage enum is backed by a u16.
977		if (num > std.math.maxInt(u16)) {
978		return error.CodePagePragmaInvalidCodePage;
979		}
980
981		break :code_page code_pages.CodePage.getByIdentifierEnsureSupported(@intCast(num)) catch \|err\| switch (err) {
982		error.InvalidCodePage => return error.CodePagePragmaInvalidCodePage,
983		error.UnsupportedCodePage => return error.CodePagePragmaUnsupportedCodePage,
984		};
985		};
	910	const code_page = (parsePragmaCodePage(full_command) catch \|err\| switch (err) {
	911	error.NotPragma, error.NotCodePagePragma => return,
	912	else => \|e\| return e,
	913	}) orelse self.default_code_page;
986	914
987	915	// https://learn.microsoft.com/en-us/windows/win32/menurc/pragma-directives
988	916	// > This pragma is not supported in an included resource file (.rc)
...	...	@@ -998,24 +926,16 @@ pub const Lexer = struct {
998	926	}
999	927
1000	928	self.seen_pragma_code_pages +\|= 1;
	929	self.last_pragma_code_page_token = token;
1001	930	self.current_code_page = code_page;
1002	931	}
1003	932
1004		fn parseCodePageNum(str: []const u8) !u32 {
1005		var x: u32 = 0;
1006		for (str) \|c\| {
1007		const digit = try std.fmt.charToDigit(c, 10);
1008		if (x != 0) x = try std.math.mul(u32, x, 10);
1009		x = try std.math.add(u32, x, digit);
1010		}
1011		return x;
1012		}
1013
1014	933	pub fn getErrorDetails(self: Self, lex_err: LexError) ErrorDetails {
1015	934	const err = switch (lex_err) {
1016	935	error.UnfinishedStringLiteral => ErrorDetails.Error.unfinished_string_literal,
1017	936	error.StringLiteralTooLong => return .{
1018	937	.err = .string_literal_too_long,
	938	.code_page = self.current_code_page,
1019	939	.token = self.error_context_token.?,
1020	940	.extra = .{ .number = self.max_string_literal_codepoints },
1021	941	},
...	...	@@ -1037,11 +957,112 @@ pub const Lexer = struct {
1037	957	};
1038	958	return .{
1039	959	.err = err,
	960	.code_page = self.current_code_page,
1040	961	.token = self.error_context_token.?,
1041	962	};
1042	963	}
1043	964	};
1044	965
	966	fn parseCodePageNum(str: []const u8) !u32 {
	967	var x: u32 = 0;
	968	for (str) \|c\| {
	969	const digit = try std.fmt.charToDigit(c, 10);
	970	if (x != 0) x = try std.math.mul(u32, x, 10);
	971	x = try std.math.add(u32, x, digit);
	972	}
	973	return x;
	974	}
	975
	976	/// Returns `null` when the code_page is set to DEFAULT
	977	pub fn parsePragmaCodePage(full_command: []const u8) !?SupportedCodePage {
	978	var command = full_command;
	979
	980	// Anything besides exactly this is ignored by the Windows RC implementation
	981	const expected_directive = "#pragma";
	982	if (!std.mem.startsWith(u8, command, expected_directive)) return error.NotPragma;
	983	command = command[expected_directive.len..];
	984
	985	if (command.len == 0 or !std.ascii.isWhitespace(command[0])) return error.NotCodePagePragma;
	986	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
	987	command = command[1..];
	988	}
	989
	990	// Note: CoDe_PaGeZ is also treated as "code_page" by the Windows RC implementation,
	991	// and it will error with 'Missing left parenthesis in code_page #pragma'
	992	const expected_extension = "code_page";
	993	if (!std.ascii.startsWithIgnoreCase(command, expected_extension)) return error.NotCodePagePragma;
	994	command = command[expected_extension.len..];
	995
	996	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
	997	command = command[1..];
	998	}
	999
	1000	if (command.len == 0 or command[0] != '(') {
	1001	return error.CodePagePragmaMissingLeftParen;
	1002	}
	1003	command = command[1..];
	1004
	1005	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
	1006	command = command[1..];
	1007	}
	1008
	1009	var num_str: []u8 = command[0..0];
	1010	while (command.len > 0 and (command[0] != ')' and !std.ascii.isWhitespace(command[0]))) {
	1011	command = command[1..];
	1012	num_str.len += 1;
	1013	}
	1014
	1015	if (num_str.len == 0) {
	1016	return error.CodePagePragmaNotInteger;
	1017	}
	1018
	1019	while (command.len > 0 and std.ascii.isWhitespace(command[0])) {
	1020	command = command[1..];
	1021	}
	1022
	1023	if (command.len == 0 or command[0] != ')') {
	1024	return error.CodePagePragmaMissingRightParen;
	1025	}
	1026
	1027	const code_page: ?SupportedCodePage = code_page: {
	1028	if (std.ascii.eqlIgnoreCase("DEFAULT", num_str)) {
	1029	break :code_page null;
	1030	}
	1031
	1032	// The Win32 compiler behaves fairly strangely around maxInt(u32):
	1033	// - If the overflowed u32 wraps and becomes a known code page ID, then
	1034	// it will error/warn with "Codepage not valid: ignored" (depending on /w)
	1035	// - If the overflowed u32 wraps and does not become a known code page ID,
	1036	// then it will error with 'constant too big' and 'Codepage not integer'
	1037	//
	1038	// Instead of that, we just have a separate error specifically for overflow.
	1039	const num = parseCodePageNum(num_str) catch \|err\| switch (err) {
	1040	error.InvalidCharacter => return error.CodePagePragmaNotInteger,
	1041	error.Overflow => return error.CodePagePragmaOverflow,
	1042	};
	1043
	1044	// Anything that starts with 0 but does not resolve to 0 is treated as invalid, e.g. 01252
	1045	if (num_str[0] == '0' and num != 0) {
	1046	return error.CodePagePragmaInvalidCodePage;
	1047	}
	1048	// Anything that resolves to 0 is treated as 'not an integer' by the Win32 implementation.
	1049	else if (num == 0) {
	1050	return error.CodePagePragmaNotInteger;
	1051	}
	1052	// Anything above u16 max is not going to be found since our CodePage enum is backed by a u16.
	1053	if (num > std.math.maxInt(u16)) {
	1054	return error.CodePagePragmaInvalidCodePage;
	1055	}
	1056
	1057	break :code_page code_pages.getByIdentifierEnsureSupported(@intCast(num)) catch \|err\| switch (err) {
	1058	error.InvalidCodePage => return error.CodePagePragmaInvalidCodePage,
	1059	error.UnsupportedCodePage => return error.CodePagePragmaUnsupportedCodePage,
	1060	};
	1061	};
	1062
	1063	return code_page;
	1064	}
	1065
1045	1066	fn testLexNormal(source: []const u8, expected_tokens: []const Token.Id) !void {
1046	1067	var lexer = Lexer.init(source, .{});
1047	1068	if (dumpTokensDuringTests) std.debug.print("\n----------------------\n{s}\n----------------------\n", .{lexer.buffer});
...	...	@@ -1074,7 +1095,7 @@ test "normal: string literals" {
1074	1095
1075	1096	test "superscript chars and code pages" {
1076	1097	const firstToken = struct {
1077		pub fn firstToken(source: []const u8, default_code_page: CodePage, comptime lex_method: Lexer.LexMethod) LexError!Token {
	1098	pub fn firstToken(source: []const u8, default_code_page: SupportedCodePage, comptime lex_method: Lexer.LexMethod) LexError!Token {
1078	1099	var lexer = Lexer.init(source, .{ .default_code_page = default_code_page });
1079	1100	return lexer.next(lex_method);
1080	1101	}

lib/compiler/resinator/literals.zig+280-93

...	...	@@ -1,6 +1,6 @@
1	1	const std = @import("std");
2	2	const code_pages = @import("code_pages.zig");
3		const CodePage = code_pages.CodePage;
	3	const SupportedCodePage = code_pages.SupportedCodePage;
4	4	const windows1252 = @import("windows1252.zig");
5	5	const ErrorDetails = @import("errors.zig").ErrorDetails;
6	6	const DiagnosticsContext = @import("errors.zig").DiagnosticsContext;
...	...	@@ -18,7 +18,7 @@ pub fn isValidNumberDataLiteral(str: []const u8) bool {
18	18
19	19	pub const SourceBytes = struct {
20	20	slice: []const u8,
21		code_page: CodePage,
	21	code_page: SupportedCodePage,
22	22	};
23	23
24	24	pub const StringType = enum { ascii, wide };
...	...	@@ -53,7 +53,7 @@ pub const StringType = enum { ascii, wide };
53	53	/// branches should never actually be hit during this function.
54	54	pub const IterativeStringParser = struct {
55	55	source: []const u8,
56		code_page: CodePage,
	56	code_page: SupportedCodePage,
57	57	/// The type of the string inferred by the prefix (L"" or "")
58	58	/// This is what matters for things like the maximum digits in an
59	59	/// escape sequence, whether or not invalid escape sequences are skipped, etc.
...	...	@@ -98,32 +98,55 @@ pub const IterativeStringParser = struct {
98	98
99	99	pub const ParsedCodepoint = struct {
100	100	codepoint: u21,
101		/// Note: If this is true, `codepoint` will be a value with a max of maxInt(u16).
102		/// This is enforced by using saturating arithmetic, so in e.g. a wide string literal the
103		/// octal escape sequence \7777777 (2,097,151) will be parsed into the value 0xFFFF (65,535).
104		/// If the value needs to be truncated to a smaller integer (for ASCII string literals), then that
105		/// must be done by the caller.
	101	/// Note: If this is true, `codepoint` will have an effective maximum value
	102	/// of 0xFFFF, as `codepoint` is calculated using wrapping arithmetic on a u16.
	103	/// If the value needs to be truncated to a smaller integer (e.g. for ASCII string
	104	/// literals), then that must be done by the caller.
106	105	from_escaped_integer: bool = false,
	106	/// Denotes that the codepoint is:
	107	/// - Escaped (has a \ in front of it), and
	108	/// - Has a value >= U+10000, meaning it would be encoded as a surrogate
	109	/// pair in UTF-16, and
	110	/// - Is part of a wide string literal
	111	///
	112	/// Normally in wide string literals, invalid escapes are omitted
	113	/// during parsing (the codepoints are not returned at all during
	114	/// the `next` call), but this is a special case in which the
	115	/// escape only applies to the high surrogate pair of the codepoint.
	116	///
	117	/// TODO: Maybe just return the low surrogate codepoint by itself in this case.
	118	escaped_surrogate_pair: bool = false,
107	119	};
108	120
109	121	pub fn next(self: *IterativeStringParser) std.mem.Allocator.Error!?ParsedCodepoint {
110	122	const result = try self.nextUnchecked();
111	123	if (self.diagnostics != null and result != null and !result.?.from_escaped_integer) {
112	124	switch (result.?.codepoint) {
113		0x900, 0xA00, 0xA0D, 0x2000, 0xFFFE, 0xD00 => {
	125	0x0900, 0x0A00, 0x0A0D, 0x2000, 0x0D00 => {
114	126	const err: ErrorDetails.Error = if (result.?.codepoint == 0xD00)
115	127	.rc_would_miscompile_codepoint_skip
116	128	else
117		.rc_would_miscompile_codepoint_byte_swap;
	129	.rc_would_miscompile_codepoint_whitespace;
118	130	try self.diagnostics.?.diagnostics.append(ErrorDetails{
119	131	.err = err,
120	132	.type = .warning,
	133	.code_page = self.code_page,
121	134	.token = self.diagnostics.?.token,
122	135	.extra = .{ .number = result.?.codepoint },
123	136	});
	137	},
	138	0xFFFE, 0xFFFF => {
124	139	try self.diagnostics.?.diagnostics.append(ErrorDetails{
125		.err = err,
	140	.err = .rc_would_miscompile_codepoint_bom,
	141	.type = .warning,
	142	.code_page = self.code_page,
	143	.token = self.diagnostics.?.token,
	144	.extra = .{ .number = result.?.codepoint },
	145	});
	146	try self.diagnostics.?.diagnostics.append(ErrorDetails{
	147	.err = .rc_would_miscompile_codepoint_bom,
126	148	.type = .note,
	149	.code_page = self.code_page,
127	150	.token = self.diagnostics.?.token,
128	151	.print_source_line = false,
129	152	.extra = .{ .number = result.?.codepoint },
...	...	@@ -188,11 +211,13 @@ pub const IterativeStringParser = struct {
188	211	try self.diagnostics.?.diagnostics.append(ErrorDetails{
189	212	.err = .tab_converted_to_spaces,
190	213	.type = .warning,
	214	.code_page = self.code_page,
191	215	.token = self.diagnostics.?.token,
192	216	});
193	217	try self.diagnostics.?.diagnostics.append(ErrorDetails{
194	218	.err = .tab_converted_to_spaces,
195	219	.type = .note,
	220	.code_page = self.code_page,
196	221	.token = self.diagnostics.?.token,
197	222	.print_source_line = false,
198	223	});
...	...	@@ -246,8 +271,9 @@ pub const IterativeStringParser = struct {
246	271	switch (c) {
247	272	'a', 'A' => {
248	273	self.index += codepoint.byte_len;
	274	// might be a bug in RC, but matches its behavior
249	275	return .{ .codepoint = '\x08' };
250		}, // might be a bug in RC, but matches its behavior
	276	},
251	277	'n' => {
252	278	self.index += codepoint.byte_len;
253	279	return .{ .codepoint = '\n' };
...	...	@@ -269,7 +295,65 @@ pub const IterativeStringParser = struct {
269	295	backtrack = true;
270	296	},
271	297	else => switch (self.declared_string_type) {
272		.wide => {}, // invalid escape sequences are skipped in wide strings
	298	.wide => {
	299	// All invalid escape sequences are skipped in wide strings,
	300	// but there is a special case around \<tab> where the \
	301	// is skipped but the tab character is processed.
	302	// It's actually a bit weirder than that, though, since
	303	// the preprocessor is the one that does the <tab> -> spaces
	304	// conversion, so it goes something like this:
	305	//
	306	// Before preprocessing: L"\<tab>"
	307	// After preprocessing: L"\ "
	308	//
	309	// So the parser only sees an escaped space character followed
	310	// by some other number of spaces >= 0.
	311	//
	312	// However, our preprocessor keeps tab characters intact, so we emulate
	313	// the above behavior by skipping the \ and then outputting one less
	314	// space than normal for the <tab> character.
	315	if (c == '\t') {
	316	// Only warn about a tab getting converted to spaces once per string
	317	if (self.diagnostics != null and !self.seen_tab) {
	318	try self.diagnostics.?.diagnostics.append(ErrorDetails{
	319	.err = .tab_converted_to_spaces,
	320	.type = .warning,
	321	.code_page = self.code_page,
	322	.token = self.diagnostics.?.token,
	323	});
	324	try self.diagnostics.?.diagnostics.append(ErrorDetails{
	325	.err = .tab_converted_to_spaces,
	326	.type = .note,
	327	.code_page = self.code_page,
	328	.token = self.diagnostics.?.token,
	329	.print_source_line = false,
	330	});
	331	self.seen_tab = true;
	332	}
	333
	334	const cols = columnsUntilTabStop(self.column, 8);
	335	// If the tab character would only be converted to a single space,
	336	// then we can just skip both the \ and the <tab> and move on.
	337	if (cols > 1) {
	338	self.num_pending_spaces = @intCast(cols - 2);
	339	self.index += codepoint.byte_len;
	340	return .{ .codepoint = ' ' };
	341	}
	342	}
	343	// There's a second special case when the codepoint would be encoded
	344	// as a surrogate pair in UTF-16, as the escape 'applies' to the
	345	// high surrogate pair only in this instance. This is a side-effect
	346	// of the Win32 RC compiler preprocessor outputting UTF-16 and the
	347	// compiler itself seemingly working on code units instead of code points
	348	// in this particular instance.
	349	//
	350	// We emulate this behavior by emitting the codepoint, but with a marker
	351	// that indicates that it needs to be handled specially.
	352	if (c >= 0x10000 and c != code_pages.Codepoint.invalid) {
	353	self.index += codepoint.byte_len;
	354	return .{ .codepoint = c, .escaped_surrogate_pair = true };
	355	}
	356	},
273	357	.ascii => {
274	358	// we intentionally avoid incrementing self.index
275	359	// to handle the current char in the next call,
...	...	@@ -303,6 +387,9 @@ pub const IterativeStringParser = struct {
303	387	},
304	388	.escaped_octal => switch (c) {
305	389	'0'...'7' => {
	390	// Note: We use wrapping arithmetic on a u16 here since there's been no observed
	391	// string parsing scenario where an escaped integer with a value >= the u16
	392	// max is interpreted as anything but the truncated u16 value.
306	393	string_escape_n *%= 8;
307	394	string_escape_n +%= std.fmt.charToDigit(@intCast(c), 8) catch unreachable;
308	395	string_escape_i += 1;
...	...	@@ -367,7 +454,7 @@ pub const IterativeStringParser = struct {
367	454	pub const StringParseOptions = struct {
368	455	start_column: usize = 0,
369	456	diagnostics: ?DiagnosticsContext = null,
370		output_code_page: CodePage = .windows1252,
	457	output_code_page: SupportedCodePage,
371	458	};
372	459
373	460	pub fn parseQuotedString(
...	...	@@ -389,46 +476,52 @@ pub fn parseQuotedString(
389	476
390	477	while (try iterative_parser.next()) \|parsed\| {
391	478	const c = parsed.codepoint;
392		if (parsed.from_escaped_integer) {
393		// We truncate here to get the correct behavior for ascii strings
394		try buf.append(std.mem.nativeToLittle(T, @truncate(c)));
395		} else {
396		switch (literal_type) {
397		.ascii => switch (options.output_code_page) {
398		.windows1252 => {
399		if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {
400		try buf.append(best_fit);
401		} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
402		try buf.append('?');
403		} else {
404		try buf.appendSlice("??");
405		}
406		},
407		.utf8 => {
408		var codepoint_to_encode = c;
409		if (c == code_pages.Codepoint.invalid) {
410		codepoint_to_encode = '�';
411		}
412		var utf8_buf: [4]u8 = undefined;
413		const utf8_len = std.unicode.utf8Encode(codepoint_to_encode, &utf8_buf) catch unreachable;
414		try buf.appendSlice(utf8_buf[0..utf8_len]);
415		},
416		else => unreachable, // Unsupported code page
417		},
418		.wide => {
419		if (c == code_pages.Codepoint.invalid) {
420		try buf.append(std.mem.nativeToLittle(u16, '�'));
421		} else if (c < 0x10000) {
422		const short: u16 = @intCast(c);
423		try buf.append(std.mem.nativeToLittle(u16, short));
	479	switch (literal_type) {
	480	.ascii => switch (options.output_code_page) {
	481	.windows1252 => {
	482	if (parsed.from_escaped_integer) {
	483	try buf.append(@truncate(c));
	484	} else if (windows1252.bestFitFromCodepoint(c)) \|best_fit\| {
	485	try buf.append(best_fit);
	486	} else if (c < 0x10000 or c == code_pages.Codepoint.invalid) {
	487	try buf.append('?');
424	488	} else {
	489	try buf.appendSlice("??");
	490	}
	491	},
	492	.utf8 => {
	493	var codepoint_to_encode = c;
	494	if (parsed.from_escaped_integer) {
	495	codepoint_to_encode = @as(T, @truncate(c));
	496	}
	497	const escaped_integer_outside_ascii_range = parsed.from_escaped_integer and codepoint_to_encode > 0x7F;
	498	if (escaped_integer_outside_ascii_range or c == code_pages.Codepoint.invalid) {
	499	codepoint_to_encode = '�';
	500	}
	501	var utf8_buf: [4]u8 = undefined;
	502	const utf8_len = std.unicode.utf8Encode(codepoint_to_encode, &utf8_buf) catch unreachable;
	503	try buf.appendSlice(utf8_buf[0..utf8_len]);
	504	},
	505	},
	506	.wide => {
	507	// Parsing any string type as a wide string is handled separately, see parseQuotedStringAsWideString
	508	std.debug.assert(iterative_parser.declared_string_type == .wide);
	509	if (parsed.from_escaped_integer) {
	510	try buf.append(std.mem.nativeToLittle(u16, @truncate(c)));
	511	} else if (c == code_pages.Codepoint.invalid) {
	512	try buf.append(std.mem.nativeToLittle(u16, '�'));
	513	} else if (c < 0x10000) {
	514	const short: u16 = @intCast(c);
	515	try buf.append(std.mem.nativeToLittle(u16, short));
	516	} else {
	517	if (!parsed.escaped_surrogate_pair) {
425	518	const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;
426	519	try buf.append(std.mem.nativeToLittle(u16, high));
427		const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
428		try buf.append(std.mem.nativeToLittle(u16, low));
429	520	}
430		},
431		}
	521	const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
	522	try buf.append(std.mem.nativeToLittle(u16, low));
	523	}
	524	},
432	525	}
433	526	}
434	527
...	...	@@ -449,9 +542,59 @@ pub fn parseQuotedWideString(allocator: std.mem.Allocator, bytes: SourceBytes, o
449	542	return parseQuotedString(.wide, allocator, bytes, options);
450	543	}
451	544
	545	/// Parses any string type into a wide string.
	546	/// If the string is declared as a wide string (L""), then it is handled normally.
	547	/// Otherwise, things are fairly normal with the exception of escaped integers.
	548	/// Escaped integers are handled by:
	549	/// - Truncating the escape to a u8
	550	/// - Reinterpeting the u8 as a byte from the output code page
	551	/// - Outputting the codepoint that corresponds to the interpreted byte, or � if no such
	552	/// interpretation is possible
	553	/// For example, if the code page is UTF-8, then while \x80 is a valid start byte, it's
	554	/// interpreted as a single byte, so it ends up being seen as invalid and � is outputted.
	555	/// If the code page is Windows-1252, then \x80 is interpreted to be € which has the
	556	/// codepoint U+20AC, so the UTF-16 encoding of U+20AC is outputted.
452	557	pub fn parseQuotedStringAsWideString(allocator: std.mem.Allocator, bytes: SourceBytes, options: StringParseOptions) ![:0]u16 {
453	558	std.debug.assert(bytes.slice.len >= 2); // ""
454		return parseQuotedString(.wide, allocator, bytes, options);
	559
	560	if (bytes.slice[0] == 'l' or bytes.slice[0] == 'L') {
	561	return parseQuotedWideString(allocator, bytes, options);
	562	}
	563
	564	// Note: We're only handling the case of parsing an ASCII string into a wide string from here on out.
	565	// TODO: The logic below is similar to that in AcceleratorKeyCodepointTranslator, might be worth merging the two
	566
	567	var buf = try std.ArrayList(u16).initCapacity(allocator, bytes.slice.len);
	568	errdefer buf.deinit();
	569
	570	var iterative_parser = IterativeStringParser.init(bytes, options);
	571
	572	while (try iterative_parser.next()) \|parsed\| {
	573	const c = parsed.codepoint;
	574	if (parsed.from_escaped_integer) {
	575	std.debug.assert(c != code_pages.Codepoint.invalid);
	576	const byte_to_interpret: u8 = @truncate(c);
	577	const code_unit_to_encode: u16 = switch (options.output_code_page) {
	578	.windows1252 => windows1252.toCodepoint(byte_to_interpret),
	579	.utf8 => if (byte_to_interpret > 0x7F) '�' else byte_to_interpret,
	580	};
	581	try buf.append(std.mem.nativeToLittle(u16, code_unit_to_encode));
	582	} else if (c == code_pages.Codepoint.invalid) {
	583	try buf.append(std.mem.nativeToLittle(u16, '�'));
	584	} else if (c < 0x10000) {
	585	const short: u16 = @intCast(c);
	586	try buf.append(std.mem.nativeToLittle(u16, short));
	587	} else {
	588	if (!parsed.escaped_surrogate_pair) {
	589	const high = @as(u16, @intCast((c - 0x10000) >> 10)) + 0xD800;
	590	try buf.append(std.mem.nativeToLittle(u16, high));
	591	}
	592	const low = @as(u16, @intCast(c & 0x3FF)) + 0xDC00;
	593	try buf.append(std.mem.nativeToLittle(u16, low));
	594	}
	595	}
	596
	597	return buf.toOwnedSliceSentinel(0);
455	598	}
456	599
457	600	test "parse quoted ascii string" {
...	...	@@ -464,133 +607,155 @@ test "parse quoted ascii string" {
464	607	\\"hello"
465	608	,
466	609	.code_page = .windows1252,
467		}, .{}));
	610	}, .{
	611	.output_code_page = .windows1252,
	612	}));
468	613	// hex with 0 digits
469	614	try std.testing.expectEqualSlices(u8, "\x00", try parseQuotedAsciiString(arena, .{
470	615	.slice =
471	616	\\"\x"
472	617	,
473	618	.code_page = .windows1252,
474		}, .{}));
	619	}, .{
	620	.output_code_page = .windows1252,
	621	}));
475	622	// hex max of 2 digits
476	623	try std.testing.expectEqualSlices(u8, "\xFFf", try parseQuotedAsciiString(arena, .{
477	624	.slice =
478	625	\\"\XfFf"
479	626	,
480	627	.code_page = .windows1252,
481		}, .{}));
	628	}, .{
	629	.output_code_page = .windows1252,
	630	}));
482	631	// octal with invalid octal digit
483	632	try std.testing.expectEqualSlices(u8, "\x019", try parseQuotedAsciiString(arena, .{
484	633	.slice =
485	634	\\"\19"
486	635	,
487	636	.code_page = .windows1252,
488		}, .{}));
	637	}, .{
	638	.output_code_page = .windows1252,
	639	}));
489	640	// escaped quotes
490	641	try std.testing.expectEqualSlices(u8, " \" ", try parseQuotedAsciiString(arena, .{
491	642	.slice =
492	643	\\" "" "
493	644	,
494	645	.code_page = .windows1252,
495		}, .{}));
	646	}, .{
	647	.output_code_page = .windows1252,
	648	}));
496	649	// backslash right before escaped quotes
497	650	try std.testing.expectEqualSlices(u8, "\"", try parseQuotedAsciiString(arena, .{
498	651	.slice =
499	652	\\"\"""
500	653	,
501	654	.code_page = .windows1252,
502		}, .{}));
	655	}, .{
	656	.output_code_page = .windows1252,
	657	}));
503	658	// octal overflow
504	659	try std.testing.expectEqualSlices(u8, "\x01", try parseQuotedAsciiString(arena, .{
505	660	.slice =
506	661	\\"\401"
507	662	,
508	663	.code_page = .windows1252,
509		}, .{}));
	664	}, .{
	665	.output_code_page = .windows1252,
	666	}));
510	667	// escapes
511	668	try std.testing.expectEqualSlices(u8, "\x08\n\r\t\\", try parseQuotedAsciiString(arena, .{
512	669	.slice =
513	670	\\"\a\n\r\t\\"
514	671	,
515	672	.code_page = .windows1252,
516		}, .{}));
	673	}, .{
	674	.output_code_page = .windows1252,
	675	}));
517	676	// uppercase escapes
518	677	try std.testing.expectEqualSlices(u8, "\x08\\N\\R\t\\", try parseQuotedAsciiString(arena, .{
519	678	.slice =
520	679	\\"\A\N\R\T\\"
521	680	,
522	681	.code_page = .windows1252,
523		}, .{}));
	682	}, .{
	683	.output_code_page = .windows1252,
	684	}));
524	685	// backslash on its own
525	686	try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(arena, .{
526	687	.slice =
527	688	\\"\"
528	689	,
529	690	.code_page = .windows1252,
530		}, .{}));
	691	}, .{
	692	.output_code_page = .windows1252,
	693	}));
531	694	// unrecognized escapes
532	695	try std.testing.expectEqualSlices(u8, "\\b", try parseQuotedAsciiString(arena, .{
533	696	.slice =
534	697	\\"\b"
535	698	,
536	699	.code_page = .windows1252,
537		}, .{}));
	700	}, .{
	701	.output_code_page = .windows1252,
	702	}));
538	703	// escaped carriage returns
539	704	try std.testing.expectEqualSlices(u8, "\\", try parseQuotedAsciiString(
540	705	arena,
541	706	.{ .slice = "\"\\\r\r\r\r\r\"", .code_page = .windows1252 },
542		.{},
	707	.{ .output_code_page = .windows1252 },
543	708	));
544	709	// escaped newlines
545	710	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
546	711	arena,
547	712	.{ .slice = "\"\\\n\n\n\n\n\"", .code_page = .windows1252 },
548		.{},
	713	.{ .output_code_page = .windows1252 },
549	714	));
550	715	// escaped CRLF pairs
551	716	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
552	717	arena,
553	718	.{ .slice = "\"\\\r\n\r\n\r\n\r\n\r\n\"", .code_page = .windows1252 },
554		.{},
	719	.{ .output_code_page = .windows1252 },
555	720	));
556	721	// escaped newlines with other whitespace
557	722	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
558	723	arena,
559	724	.{ .slice = "\"\\\n \t\r\n \r\t\n \t\"", .code_page = .windows1252 },
560		.{},
	725	.{ .output_code_page = .windows1252 },
561	726	));
562	727	// literal tab characters get converted to spaces (dependent on source file columns)
563	728	try std.testing.expectEqualSlices(u8, " ", try parseQuotedAsciiString(
564	729	arena,
565	730	.{ .slice = "\"\t\"", .code_page = .windows1252 },
566		.{},
	731	.{ .output_code_page = .windows1252 },
567	732	));
568	733	try std.testing.expectEqualSlices(u8, "abc ", try parseQuotedAsciiString(
569	734	arena,
570	735	.{ .slice = "\"abc\t\"", .code_page = .windows1252 },
571		.{},
	736	.{ .output_code_page = .windows1252 },
572	737	));
573	738	try std.testing.expectEqualSlices(u8, "abcdefg ", try parseQuotedAsciiString(
574	739	arena,
575	740	.{ .slice = "\"abcdefg\t\"", .code_page = .windows1252 },
576		.{},
	741	.{ .output_code_page = .windows1252 },
577	742	));
578	743	try std.testing.expectEqualSlices(u8, "\\ ", try parseQuotedAsciiString(
579	744	arena,
580	745	.{ .slice = "\"\\\t\"", .code_page = .windows1252 },
581		.{},
	746	.{ .output_code_page = .windows1252 },
582	747	));
583	748	// literal CR's get dropped
584	749	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
585	750	arena,
586	751	.{ .slice = "\"\r\r\r\r\r\"", .code_page = .windows1252 },
587		.{},
	752	.{ .output_code_page = .windows1252 },
588	753	));
589	754	// contiguous newlines and whitespace get collapsed to <space><newline>
590	755	try std.testing.expectEqualSlices(u8, " \n", try parseQuotedAsciiString(
591	756	arena,
592	757	.{ .slice = "\"\n\r\r \r\n \t \"", .code_page = .windows1252 },
593		.{},
	758	.{ .output_code_page = .windows1252 },
594	759	));
595	760	}
596	761
...	...	@@ -602,32 +767,32 @@ test "parse quoted ascii string with utf8 code page" {
602	767	try std.testing.expectEqualSlices(u8, "", try parseQuotedAsciiString(
603	768	arena,
604	769	.{ .slice = "\"\"", .code_page = .utf8 },
605		.{},
	770	.{ .output_code_page = .windows1252 },
606	771	));
607	772	// Codepoints that don't have a Windows-1252 representation get converted to ?
608	773	try std.testing.expectEqualSlices(u8, "?????????", try parseQuotedAsciiString(
609	774	arena,
610	775	.{ .slice = "\"кириллица\"", .code_page = .utf8 },
611		.{},
	776	.{ .output_code_page = .windows1252 },
612	777	));
613	778	// Codepoints that have a best fit mapping get converted accordingly,
614	779	// these are box drawing codepoints
615	780	try std.testing.expectEqualSlices(u8, "\x2b\x2d\x2b", try parseQuotedAsciiString(
616	781	arena,
617	782	.{ .slice = "\"┌─┐\"", .code_page = .utf8 },
618		.{},
	783	.{ .output_code_page = .windows1252 },
619	784	));
620	785	// Invalid UTF-8 gets converted to ? depending on well-formedness
621	786	try std.testing.expectEqualSlices(u8, "????", try parseQuotedAsciiString(
622	787	arena,
623	788	.{ .slice = "\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },
624		.{},
	789	.{ .output_code_page = .windows1252 },
625	790	));
626	791	// Codepoints that would require a UTF-16 surrogate pair get converted to ??
627	792	try std.testing.expectEqualSlices(u8, "??", try parseQuotedAsciiString(
628	793	arena,
629	794	.{ .slice = "\"\xF2\xAF\xBA\xB4\"", .code_page = .utf8 },
630		.{},
	795	.{ .output_code_page = .windows1252 },
631	796	));
632	797
633	798	// Output code page changes how invalid UTF-8 gets converted, since it
...	...	@@ -652,6 +817,18 @@ test "parse quoted ascii string with utf8 code page" {
652	817	));
653	818	}
654	819
	820	test "parse quoted string with different input/output code pages" {
	821	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	822	defer arena_allocator.deinit();
	823	const arena = arena_allocator.allocator();
	824
	825	try std.testing.expectEqualSlices(u8, "€��\x60\x7F", try parseQuotedAsciiString(
	826	arena,
	827	.{ .slice = "\"\x80\\x8a\\600\\612\\540\\577\"", .code_page = .windows1252 },
	828	.{ .output_code_page = .utf8 },
	829	));
	830	}
	831
655	832	test "parse quoted wide string" {
656	833	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
657	834	defer arena_allocator.deinit();
...	...	@@ -662,52 +839,62 @@ test "parse quoted wide string" {
662	839	\\L"hello"
663	840	,
664	841	.code_page = .windows1252,
665		}, .{}));
	842	}, .{
	843	.output_code_page = .windows1252,
	844	}));
666	845	// hex with 0 digits
667	846	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{0x0}, try parseQuotedWideString(arena, .{
668	847	.slice =
669	848	\\L"\x"
670	849	,
671	850	.code_page = .windows1252,
672		}, .{}));
	851	}, .{
	852	.output_code_page = .windows1252,
	853	}));
673	854	// hex max of 4 digits
674	855	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0xFFFF), std.mem.nativeToLittle(u16, 'f') }, try parseQuotedWideString(arena, .{
675	856	.slice =
676	857	\\L"\XfFfFf"
677	858	,
678	859	.code_page = .windows1252,
679		}, .{}));
	860	}, .{
	861	.output_code_page = .windows1252,
	862	}));
680	863	// octal max of 7 digits
681	864	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x9493), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '3') }, try parseQuotedWideString(arena, .{
682	865	.slice =
683	866	\\L"\111222333"
684	867	,
685	868	.code_page = .windows1252,
686		}, .{}));
	869	}, .{
	870	.output_code_page = .windows1252,
	871	}));
687	872	// octal overflow
688	873	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0xFF01)}, try parseQuotedWideString(arena, .{
689	874	.slice =
690	875	\\L"\777401"
691	876	,
692	877	.code_page = .windows1252,
693		}, .{}));
	878	}, .{
	879	.output_code_page = .windows1252,
	880	}));
694	881	// literal tab characters get converted to spaces (dependent on source file columns)
695	882	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("abcdefg "), try parseQuotedWideString(
696	883	arena,
697	884	.{ .slice = "L\"abcdefg\t\"", .code_page = .windows1252 },
698		.{},
	885	.{ .output_code_page = .windows1252 },
699	886	));
700	887	// Windows-1252 conversion
701	888	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("ðð€€€"), try parseQuotedWideString(
702	889	arena,
703	890	.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .windows1252 },
704		.{},
	891	.{ .output_code_page = .windows1252 },
705	892	));
706	893	// Invalid escape sequences are skipped
707	894	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedWideString(
708	895	arena,
709	896	.{ .slice = "L\"\\H\"", .code_page = .windows1252 },
710		.{},
	897	.{ .output_code_page = .windows1252 },
711	898	));
712	899	}
713	900
...	...	@@ -719,18 +906,18 @@ test "parse quoted wide string with utf8 code page" {
719	906	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{}, try parseQuotedWideString(
720	907	arena,
721	908	.{ .slice = "L\"\"", .code_page = .utf8 },
722		.{},
	909	.{ .output_code_page = .windows1252 },
723	910	));
724	911	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedWideString(
725	912	arena,
726	913	.{ .slice = "L\"кириллица\"", .code_page = .utf8 },
727		.{},
	914	.{ .output_code_page = .windows1252 },
728	915	));
729	916	// Invalid UTF-8 gets converted to � depending on well-formedness
730	917	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("��"), try parseQuotedWideString(
731	918	arena,
732	919	.{ .slice = "L\"\xf0\xf0\x80\x80\x80\"", .code_page = .utf8 },
733		.{},
	920	.{ .output_code_page = .windows1252 },
734	921	));
735	922	}
736	923
...	...	@@ -742,29 +929,29 @@ test "parse quoted ascii string as wide string" {
742	929	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("кириллица"), try parseQuotedStringAsWideString(
743	930	arena,
744	931	.{ .slice = "\"кириллица\"", .code_page = .utf8 },
745		.{},
	932	.{ .output_code_page = .windows1252 },
746	933	));
747	934	// Whether or not invalid escapes are skipped is still determined by the L prefix
748	935	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral("\\H"), try parseQuotedStringAsWideString(
749	936	arena,
750	937	.{ .slice = "\"\\H\"", .code_page = .windows1252 },
751		.{},
	938	.{ .output_code_page = .windows1252 },
752	939	));
753	940	try std.testing.expectEqualSentinel(u16, 0, std.unicode.utf8ToUtf16LeStringLiteral(""), try parseQuotedStringAsWideString(
754	941	arena,
755	942	.{ .slice = "L\"\\H\"", .code_page = .windows1252 },
756		.{},
	943	.{ .output_code_page = .windows1252 },
757	944	));
758	945	// Maximum escape sequence value is also determined by the L prefix
759	946	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{ std.mem.nativeToLittle(u16, 0x12), std.mem.nativeToLittle(u16, '3'), std.mem.nativeToLittle(u16, '4') }, try parseQuotedStringAsWideString(
760	947	arena,
761	948	.{ .slice = "\"\\x1234\"", .code_page = .windows1252 },
762		.{},
	949	.{ .output_code_page = .windows1252 },
763	950	));
764	951	try std.testing.expectEqualSentinel(u16, 0, &[_:0]u16{std.mem.nativeToLittle(u16, 0x1234)}, try parseQuotedStringAsWideString(
765	952	arena,
766	953	.{ .slice = "L\"\\x1234\"", .code_page = .windows1252 },
767		.{},
	954	.{ .output_code_page = .windows1252 },
768	955	));
769	956	}
770	957

lib/compiler/resinator/main.zig+26-10

...	...	@@ -7,6 +7,7 @@ const Diagnostics = @import("errors.zig").Diagnostics;
7	7	const cli = @import("cli.zig");
8	8	const preprocess = @import("preprocess.zig");
9	9	const renderErrorMessage = @import("utils.zig").renderErrorMessage;
	10	const hasDisjointCodePage = @import("disjoint_code_page.zig").hasDisjointCodePage;
10	11	const aro = @import("aro");
11	12
12	13	pub fn main() !void {
...	...	@@ -64,7 +65,7 @@ pub fn main() !void {
64	65
65	66	if (!zig_integration) {
66	67	// print any warnings/notes
67		cli_diagnostics.renderToStdErr(args, stderr_config);
	68	cli_diagnostics.renderToStdErr(cli_args, stderr_config);
68	69	// If there was something printed, then add an extra newline separator
69	70	// so that there is a clear separation between the cli diagnostics and whatever
70	71	// gets printed after
...	...	@@ -179,16 +180,30 @@ pub fn main() !void {
179	180	// Note: We still want to run this when no-preprocess is set because:
180	181	// 1. We want to print accurate line numbers after removing multiline comments
181	182	// 2. We want to be able to handle an already-preprocessed input with #line commands in it
182		var mapping_results = try parseAndRemoveLineCommands(allocator, full_input, full_input, .{ .initial_filename = options.input_filename });
183		defer mapping_results.mappings.deinit(allocator);
184
185		const final_input = removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings) catch \|err\| switch (err) {
186		error.InvalidSourceMappingCollapse => {
187		try error_handler.emitMessage(allocator, .err, "failed during comment removal; this is a known bug", .{});
	183	var mapping_results = parseAndRemoveLineCommands(allocator, full_input, full_input, .{ .initial_filename = options.input_filename }) catch \|err\| switch (err) {
	184	error.InvalidLineCommand => {
	185	// TODO: Maybe output the invalid line command
	186	try renderErrorMessage(stderr.writer(), stderr_config, .err, "invalid line command in the preprocessed source", .{});
	187	if (options.preprocess == .no) {
	188	try renderErrorMessage(stderr.writer(), stderr_config, .note, "line commands must be of the format: #line <num> \"<path>\"", .{});
	189	} else {
	190	try renderErrorMessage(stderr.writer(), stderr_config, .note, "this is likely to be a bug, please report it", .{});
	191	}
188	192	std.process.exit(1);
189	193	},
190		else => \|e\| return e,
	194	error.LineNumberOverflow => {
	195	// TODO: Better error message
	196	try renderErrorMessage(stderr.writer(), stderr_config, .err, "line number count exceeded maximum of {}", .{std.math.maxInt(usize)});
	197	std.process.exit(1);
	198	},
	199	error.OutOfMemory => \|e\| return e,
191	200	};
	201	defer mapping_results.mappings.deinit(allocator);
	202
	203	const default_code_page = options.default_code_page orelse .windows1252;
	204	const has_disjoint_code_page = hasDisjointCodePage(mapping_results.result, &mapping_results.mappings, default_code_page);
	205
	206	const final_input = try removeComments(mapping_results.result, mapping_results.result, &mapping_results.mappings);
192	207
193	208	var output_file = std.fs.cwd().createFile(options.output_filename, .{}) catch \|err\| {
194	209	try error_handler.emitMessage(allocator, .err, "unable to create output file '{s}': {s}", .{ options.output_filename, @errorName(err) });
...	...	@@ -211,7 +226,8 @@ pub fn main() !void {
211	226	.extra_include_paths = options.extra_include_paths.items,
212	227	.system_include_paths = include_paths,
213	228	.default_language_id = options.default_language_id,
214		.default_code_page = options.default_code_page orelse .windows1252,
	229	.default_code_page = default_code_page,
	230	.disjoint_code_page = has_disjoint_code_page,
215	231	.verbose = options.verbose,
216	232	.null_terminate_string_table_strings = options.null_terminate_string_table_strings,
217	233	.max_string_literal_codepoints = options.max_string_literal_codepoints,
...	...	@@ -513,7 +529,7 @@ fn diagnosticsToErrorBundle(
513	529	};
514	530	if (err_details.print_source_line) {
515	531	const source_line = err_details.token.getLineForErrorDisplay(source, source_line_start);
516		const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len);
	532	const visual_info = err_details.visualTokenInfo(source_line_start, source_line_start + source_line.len, source);
517	533	src_loc.span_start = @intCast(visual_info.point_offset - visual_info.before_len);
518	534	src_loc.span_main = @intCast(visual_info.point_offset);
519	535	src_loc.span_end = @intCast(visual_info.point_offset + 1 + visual_info.after_len);

lib/compiler/resinator/parse.zig+174-69

...	...	@@ -4,9 +4,10 @@ const Token = @import("lex.zig").Token;
4	4	const Node = @import("ast.zig").Node;
5	5	const Tree = @import("ast.zig").Tree;
6	6	const CodePageLookup = @import("ast.zig").CodePageLookup;
7		const Resource = @import("rc.zig").Resource;
	7	const ResourceType = @import("rc.zig").ResourceType;
8	8	const Allocator = std.mem.Allocator;
9	9	const ErrorDetails = @import("errors.zig").ErrorDetails;
	10	const ErrorDetailsWithoutCodePage = @import("errors.zig").ErrorDetailsWithoutCodePage;
10	11	const Diagnostics = @import("errors.zig").Diagnostics;
11	12	const SourceBytes = @import("literals.zig").SourceBytes;
12	13	const Compiler = @import("compile.zig").Compiler;
...	...	@@ -30,6 +31,7 @@ pub const Parser = struct {
30	31
31	32	pub const Options = struct {
32	33	warn_instead_of_error_on_invalid_code_page: bool = false,
	34	disjoint_code_page: bool = false,
33	35	};
34	36
35	37	pub fn init(lexer: *Lexer, options: Options) Parser {
...	...	@@ -47,6 +49,7 @@ pub const Parser = struct {
47	49	diagnostics: *Diagnostics,
48	50	input_code_page_lookup: CodePageLookup,
49	51	output_code_page_lookup: CodePageLookup,
	52	warned_about_disjoint_code_page: bool,
50	53	};
51	54
52	55	pub fn parse(self: Self, allocator: Allocator, diagnostics: Diagnostics) Error!*Tree {
...	...	@@ -61,6 +64,7 @@ pub const Parser = struct {
61	64	.diagnostics = diagnostics,
62	65	.input_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page),
63	66	.output_code_page_lookup = CodePageLookup.init(arena.allocator(), self.lexer.default_code_page),
	67	.warned_about_disjoint_code_page = false,
64	68	};
65	69
66	70	const parsed_root = try self.parseRoot();
...	...	@@ -116,7 +120,7 @@ pub const Parser = struct {
116	120	const maybe_common_resource_attribute = try self.lookaheadToken(.normal);
117	121	if (maybe_common_resource_attribute.id == .literal and rc.CommonResourceAttributes.map.has(maybe_common_resource_attribute.slice(self.lexer.buffer))) {
118	122	try common_resource_attributes.append(self.state.arena, maybe_common_resource_attribute);
119		self.nextToken(.normal) catch unreachable;
	123	try self.nextToken(.normal);
120	124	} else {
121	125	break;
122	126	}
...	...	@@ -130,8 +134,13 @@ pub const Parser = struct {
130	134	/// optional statements (if any). If there are no optional statements, the
131	135	/// current token is unchanged.
132	136	/// The returned slice is allocated by the parser's arena
133		fn parseOptionalStatements(self: Self, resource: Resource) ![]Node {
	137	fn parseOptionalStatements(self: Self, resource: ResourceType) ![]Node {
134	138	var optional_statements: std.ArrayListUnmanaged(*Node) = .empty;
	139
	140	const num_statement_types = @typeInfo(rc.OptionalStatements).@"enum".fields.len;
	141	var statement_type_has_duplicates = [_]bool{false} ** num_statement_types;
	142	var last_statement_per_type = [_]?Node{null} * num_statement_types;
	143
135	144	while (true) {
136	145	const lookahead_token = try self.lookaheadToken(.normal);
137	146	if (lookahead_token.id != .literal) break;
...	...	@@ -140,7 +149,13 @@ pub const Parser = struct {
140	149	.dialog, .dialogex => rc.OptionalStatements.dialog_map.get(slice) orelse break,
141	150	else => break,
142	151	};
143		self.nextToken(.normal) catch unreachable;
	152	try self.nextToken(.normal);
	153
	154	const type_i = @intFromEnum(optional_statement_type);
	155	if (last_statement_per_type[type_i] != null) {
	156	statement_type_has_duplicates[type_i] = true;
	157	}
	158
144	159	switch (optional_statement_type) {
145	160	.language => {
146	161	const language = try self.parseLanguageStatement();
...	...	@@ -166,7 +181,7 @@ pub const Parser = struct {
166	181	try self.nextToken(.normal);
167	182	const value = self.state.token;
168	183	if (!value.isStringLiteral()) {
169		return self.addErrorDetailsAndFail(ErrorDetails{
	184	return self.addErrorDetailsAndFail(.{
170	185	.err = .expected_something_else,
171	186	.token = value,
172	187	.extra = .{ .expected_types = .{
...	...	@@ -223,7 +238,7 @@ pub const Parser = struct {
223	238	try self.nextToken(.normal);
224	239	const typeface = self.state.token;
225	240	if (!typeface.isStringLiteral()) {
226		return self.addErrorDetailsAndFail(ErrorDetails{
	241	return self.addErrorDetailsAndFail(.{
227	242	.err = .expected_something_else,
228	243	.token = typeface,
229	244	.extra = .{ .expected_types = .{
...	...	@@ -272,7 +287,42 @@ pub const Parser = struct {
272	287	try optional_statements.append(self.state.arena, &node.base);
273	288	},
274	289	}
	290
	291	last_statement_per_type[type_i] = optional_statements.items[optional_statements.items.len - 1];
275	292	}
	293
	294	for (optional_statements.items) \|optional_statement\| {
	295	const type_i = type_i: {
	296	switch (optional_statement.id) {
	297	.simple_statement => {
	298	const simple_statement: *Node.SimpleStatement = @alignCast(@fieldParentPtr("base", optional_statement));
	299	const statement_identifier = simple_statement.identifier;
	300	const slice = statement_identifier.slice(self.lexer.buffer);
	301	const optional_statement_type = rc.OptionalStatements.map.get(slice) orelse
	302	rc.OptionalStatements.dialog_map.get(slice).?;
	303	break :type_i @intFromEnum(optional_statement_type);
	304	},
	305	.font_statement => {
	306	break :type_i @intFromEnum(rc.OptionalStatements.font);
	307	},
	308	.language_statement => {
	309	break :type_i @intFromEnum(rc.OptionalStatements.language);
	310	},
	311	else => unreachable,
	312	}
	313	};
	314	if (!statement_type_has_duplicates[type_i]) continue;
	315	if (optional_statement == last_statement_per_type[type_i].?) continue;
	316
	317	try self.addErrorDetails(.{
	318	.err = .duplicate_optional_statement_skipped,
	319	.type = .warning,
	320	.token = optional_statement.getFirstToken(),
	321	.token_span_start = optional_statement.getFirstToken(),
	322	.token_span_end = optional_statement.getLastToken(),
	323	});
	324	}
	325
276	326	return optional_statements.toOwnedSlice(self.state.arena);
277	327	}
278	328
...	...	@@ -311,12 +361,13 @@ pub const Parser = struct {
311	361	const maybe_end_token = try self.lookaheadToken(.normal);
312	362	switch (maybe_end_token.id) {
313	363	.end => {
314		self.nextToken(.normal) catch unreachable;
	364	try self.nextToken(.normal);
315	365	break;
316	366	},
317	367	.eof => {
318		return self.addErrorDetailsAndFail(ErrorDetails{
	368	return self.addErrorDetailsWithCodePageAndFail(.{
319	369	.err = .unfinished_string_table_block,
	370	.code_page = self.lexer.current_code_page,
320	371	.token = maybe_end_token,
321	372	});
322	373	},
...	...	@@ -328,7 +379,7 @@ pub const Parser = struct {
328	379
329	380	try self.nextToken(.normal);
330	381	if (self.state.token.id != .quoted_ascii_string and self.state.token.id != .quoted_wide_string) {
331		return self.addErrorDetailsAndFail(ErrorDetails{
	382	return self.addErrorDetailsAndFail(.{
332	383	.err = .expected_something_else,
333	384	.token = self.state.token,
334	385	.extra = .{ .expected_types = .{ .string_literal = true } },
...	...	@@ -345,7 +396,7 @@ pub const Parser = struct {
345	396	}
346	397
347	398	if (strings.items.len == 0) {
348		return self.addErrorDetailsAndFail(ErrorDetails{
	399	return self.addErrorDetailsAndFail(.{
349	400	.err = .expected_token, // TODO: probably a more specific error message
350	401	.token = self.state.token,
351	402	.extra = .{ .expected = .number },
...	...	@@ -374,7 +425,12 @@ pub const Parser = struct {
374	425	// of projects. So, we have special compatibility for this particular case.
375	426	const maybe_eof = try self.lookaheadToken(.whitespace_delimiter_only);
376	427	if (maybe_eof.id == .eof) {
377		// TODO: emit warning
	428	try self.addErrorDetails(.{
	429	.err = .dangling_literal_at_eof,
	430	.type = .warning,
	431	.token = first_token,
	432	});
	433
378	434	var context = try self.state.arena.alloc(Token, 2);
379	435	context[0] = first_token;
380	436	context[1] = maybe_eof;
...	...	@@ -413,12 +469,12 @@ pub const Parser = struct {
413	469	if (maybe_ordinal == null) {
414	470	const would_be_win32_rc_ordinal = res.NameOrOrdinal.maybeNonAsciiOrdinalFromString(id_bytes);
415	471	if (would_be_win32_rc_ordinal) \|win32_rc_ordinal\| {
416		try self.addErrorDetails(ErrorDetails{
	472	try self.addErrorDetails(.{
417	473	.err = .id_must_be_ordinal,
418	474	.token = id_token,
419	475	.extra = .{ .resource = resource },
420	476	});
421		return self.addErrorDetailsAndFail(ErrorDetails{
	477	return self.addErrorDetailsAndFail(.{
422	478	.err = .win32_non_ascii_ordinal,
423	479	.token = id_token,
424	480	.type = .note,
...	...	@@ -426,7 +482,7 @@ pub const Parser = struct {
426	482	.extra = .{ .number = win32_rc_ordinal.ordinal },
427	483	});
428	484	} else {
429		return self.addErrorDetailsAndFail(ErrorDetails{
	485	return self.addErrorDetailsAndFail(.{
430	486	.err = .id_must_be_ordinal,
431	487	.token = id_token,
432	488	.extra = .{ .resource = resource },
...	...	@@ -451,7 +507,7 @@ pub const Parser = struct {
451	507	const lookahead = try self.lookaheadToken(.normal);
452	508	switch (lookahead.id) {
453	509	.end, .eof => {
454		self.nextToken(.normal) catch unreachable;
	510	try self.nextToken(.normal);
455	511	break;
456	512	},
457	513	else => {},
...	...	@@ -739,19 +795,19 @@ pub const Parser = struct {
739	795
740	796	const maybe_begin = try self.lookaheadToken(.normal);
741	797	if (maybe_begin.id == .begin) {
742		self.nextToken(.normal) catch unreachable;
	798	try self.nextToken(.normal);
743	799
744	800	if (!resource.canUseRawData()) {
745		try self.addErrorDetails(ErrorDetails{
	801	try self.addErrorDetails(.{
746	802	.err = .resource_type_cant_use_raw_data,
747		.token = maybe_begin,
	803	.token = self.state.token,
748	804	.extra = .{ .resource = resource },
749	805	});
750		return self.addErrorDetailsAndFail(ErrorDetails{
	806	return self.addErrorDetailsAndFail(.{
751	807	.err = .resource_type_cant_use_raw_data,
752	808	.type = .note,
753	809	.print_source_line = false,
754		.token = maybe_begin,
	810	.token = self.state.token,
755	811	});
756	812	}
757	813
...	...	@@ -802,11 +858,12 @@ pub const Parser = struct {
802	858	const maybe_end_token = try self.lookaheadToken(.normal);
803	859	switch (maybe_end_token.id) {
804	860	.comma => {
	861	try self.nextToken(.normal);
805	862	// comma as the first token in a raw data block is an error
806	863	if (raw_data.items.len == 0) {
807		return self.addErrorDetailsAndFail(ErrorDetails{
	864	return self.addErrorDetailsAndFail(.{
808	865	.err = .expected_something_else,
809		.token = maybe_end_token,
	866	.token = self.state.token,
810	867	.extra = .{ .expected_types = .{
811	868	.number = true,
812	869	.number_expression = true,
...	...	@@ -815,16 +872,16 @@ pub const Parser = struct {
815	872	});
816	873	}
817	874	// otherwise just skip over commas
818		self.nextToken(.normal) catch unreachable;
819	875	continue;
820	876	},
821	877	.end => {
822		self.nextToken(.normal) catch unreachable;
	878	try self.nextToken(.normal);
823	879	break;
824	880	},
825	881	.eof => {
826		return self.addErrorDetailsAndFail(ErrorDetails{
	882	return self.addErrorDetailsWithCodePageAndFail(.{
827	883	.err = .unfinished_raw_data_block,
	884	.code_page = self.lexer.current_code_page,
828	885	.token = maybe_end_token,
829	886	});
830	887	},
...	...	@@ -836,10 +893,12 @@ pub const Parser = struct {
836	893	if (expression.isNumberExpression()) {
837	894	const maybe_close_paren = try self.lookaheadToken(.normal);
838	895	if (maybe_close_paren.id == .close_paren) {
	896	// advance to ensure that the code page lookup is populated for this token
	897	try self.nextToken(.normal);
839	898	// <number expression>) is an error
840		return self.addErrorDetailsAndFail(ErrorDetails{
	899	return self.addErrorDetailsAndFail(.{
841	900	.err = .expected_token,
842		.token = maybe_close_paren,
	901	.token = self.state.token,
843	902	.extra = .{ .expected = .operator },
844	903	});
845	904	}
...	...	@@ -852,10 +911,10 @@ pub const Parser = struct {
852	911	/// begin on the next token.
853	912	/// After return, the current token will be the token immediately before the end of the
854	913	/// control statement (or unchanged if the function returns null).
855		fn parseControlStatement(self: Self, resource: Resource) Error!?Node {
	914	fn parseControlStatement(self: Self, resource: ResourceType) Error!?Node {
856	915	const control_token = try self.lookaheadToken(.normal);
857	916	const control = rc.Control.map.get(control_token.slice(self.lexer.buffer)) orelse return null;
858		self.nextToken(.normal) catch unreachable;
	917	try self.nextToken(.normal);
859	918
860	919	try self.skipAnyCommas();
861	920
...	...	@@ -867,7 +926,7 @@ pub const Parser = struct {
867	926	text = self.state.token;
868	927	},
869	928	else => {
870		return self.addErrorDetailsAndFail(ErrorDetails{
	929	return self.addErrorDetailsAndFail(.{
871	930	.err = .expected_something_else,
872	931	.token = self.state.token,
873	932	.extra = .{ .expected_types = .{
...	...	@@ -920,14 +979,16 @@ pub const Parser = struct {
920	979	// the style parameter.
921	980	const lookahead_token = try self.lookaheadToken(.normal);
922	981	if (lookahead_token.id != .comma and lookahead_token.id != .eof) {
923		try self.addErrorDetails(.{
	982	try self.addErrorDetailsWithCodePage(.{
924	983	.err = .rc_could_miscompile_control_params,
925	984	.type = .warning,
	985	.code_page = self.lexer.current_code_page,
926	986	.token = lookahead_token,
927	987	});
928		try self.addErrorDetails(.{
	988	try self.addErrorDetailsWithCodePage(.{
929	989	.err = .rc_could_miscompile_control_params,
930	990	.type = .note,
	991	.code_page = self.lexer.current_code_page,
931	992	.token = style.?.getFirstToken(),
932	993	.token_span_end = style.?.getLastToken(),
933	994	});
...	...	@@ -987,7 +1048,7 @@ pub const Parser = struct {
987	1048	fn parseToolbarButtonStatement(self: Self) Error!?Node {
988	1049	const keyword_token = try self.lookaheadToken(.normal);
989	1050	const button_type = rc.ToolbarButton.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null;
990		self.nextToken(.normal) catch unreachable;
	1051	try self.nextToken(.normal);
991	1052
992	1053	switch (button_type) {
993	1054	.separator => {
...	...	@@ -1014,10 +1075,10 @@ pub const Parser = struct {
1014	1075	/// begin on the next token.
1015	1076	/// After return, the current token will be the token immediately before the end of the
1016	1077	/// menuitem statement (or unchanged if the function returns null).
1017		fn parseMenuItemStatement(self: Self, resource: Resource, top_level_menu_id_token: Token, nesting_level: u32) Error!?Node {
	1078	fn parseMenuItemStatement(self: Self, resource: ResourceType, top_level_menu_id_token: Token, nesting_level: u32) Error!?Node {
1018	1079	const menuitem_token = try self.lookaheadToken(.normal);
1019	1080	const menuitem = rc.MenuItem.map.get(menuitem_token.slice(self.lexer.buffer)) orelse return null;
1020		self.nextToken(.normal) catch unreachable;
	1081	try self.nextToken(.normal);
1021	1082
1022	1083	if (nesting_level > max_nested_menu_level) {
1023	1084	try self.addErrorDetails(.{
...	...	@@ -1050,7 +1111,7 @@ pub const Parser = struct {
1050	1111	} else {
1051	1112	const text = self.state.token;
1052	1113	if (!text.isStringLiteral()) {
1053		return self.addErrorDetailsAndFail(ErrorDetails{
	1114	return self.addErrorDetailsAndFail(.{
1054	1115	.err = .expected_something_else,
1055	1116	.token = text,
1056	1117	.extra = .{ .expected_types = .{
...	...	@@ -1070,7 +1131,7 @@ pub const Parser = struct {
1070	1131	if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) {
1071	1132	break;
1072	1133	}
1073		self.nextToken(.normal) catch unreachable;
	1134	try self.nextToken(.normal);
1074	1135	try options.append(self.state.arena, option_token);
1075	1136	try self.skipAnyCommas();
1076	1137	}
...	...	@@ -1089,7 +1150,7 @@ pub const Parser = struct {
1089	1150	try self.nextToken(.normal);
1090	1151	const text = self.state.token;
1091	1152	if (!text.isStringLiteral()) {
1092		return self.addErrorDetailsAndFail(ErrorDetails{
	1153	return self.addErrorDetailsAndFail(.{
1093	1154	.err = .expected_something_else,
1094	1155	.token = text,
1095	1156	.extra = .{ .expected_types = .{
...	...	@@ -1105,7 +1166,7 @@ pub const Parser = struct {
1105	1166	if (!rc.MenuItem.Option.map.has(option_token.slice(self.lexer.buffer))) {
1106	1167	break;
1107	1168	}
1108		self.nextToken(.normal) catch unreachable;
	1169	try self.nextToken(.normal);
1109	1170	try options.append(self.state.arena, option_token);
1110	1171	try self.skipAnyCommas();
1111	1172	}
...	...	@@ -1146,7 +1207,7 @@ pub const Parser = struct {
1146	1207	try self.nextToken(.normal);
1147	1208	const text = self.state.token;
1148	1209	if (!text.isStringLiteral()) {
1149		return self.addErrorDetailsAndFail(ErrorDetails{
	1210	return self.addErrorDetailsAndFail(.{
1150	1211	.err = .expected_something_else,
1151	1212	.token = text,
1152	1213	.extra = .{ .expected_types = .{
...	...	@@ -1257,7 +1318,7 @@ pub const Parser = struct {
1257	1318	fn parseVersionStatement(self: Self) Error!?Node {
1258	1319	const type_token = try self.lookaheadToken(.normal);
1259	1320	const statement_type = rc.VersionInfo.map.get(type_token.slice(self.lexer.buffer)) orelse return null;
1260		self.nextToken(.normal) catch unreachable;
	1321	try self.nextToken(.normal);
1261	1322	switch (statement_type) {
1262	1323	.file_version, .product_version => {
1263	1324	var parts_buffer: [4]*Node = undefined;
...	...	@@ -1301,7 +1362,7 @@ pub const Parser = struct {
1301	1362	fn parseVersionBlockOrValue(self: Self, top_level_version_id_token: Token, nesting_level: u32) Error!?Node {
1302	1363	const keyword_token = try self.lookaheadToken(.normal);
1303	1364	const keyword = rc.VersionBlock.map.get(keyword_token.slice(self.lexer.buffer)) orelse return null;
1304		self.nextToken(.normal) catch unreachable;
	1365	try self.nextToken(.normal);
1305	1366
1306	1367	if (nesting_level > max_nested_version_level) {
1307	1368	try self.addErrorDetails(.{
...	...	@@ -1541,7 +1602,7 @@ pub const Parser = struct {
1541	1602	}
1542	1603	};
1543	1604
1544		pub fn toErrorDetails(options: ParseExpressionOptions, token: Token) ErrorDetails {
	1605	pub fn toErrorDetails(options: ParseExpressionOptions, token: Token) ErrorDetailsWithoutCodePage {
1545	1606	// TODO: expected_types_override interaction with is_known_to_be_number_expression?
1546	1607	const expected_types = options.expected_types_override orelse ErrorDetails.ExpectedTypes{
1547	1608	.number = options.allowed_types.number,
...	...	@@ -1549,7 +1610,7 @@ pub const Parser = struct {
1549	1610	.string_literal = options.allowed_types.string and !options.is_known_to_be_number_expression,
1550	1611	.literal = options.allowed_types.literal and !options.is_known_to_be_number_expression,
1551	1612	};
1552		return ErrorDetails{
	1613	return .{
1553	1614	.err = .expected_something_else,
1554	1615	.token = token,
1555	1616	.extra = .{ .expected_types = expected_types },
...	...	@@ -1690,7 +1751,7 @@ pub const Parser = struct {
1690	1751
1691	1752	try self.addErrorDetails(options.toErrorDetails(self.state.token));
1692	1753	if (is_close_paren_expression) {
1693		try self.addErrorDetails(ErrorDetails{
	1754	try self.addErrorDetails(.{
1694	1755	.err = .close_paren_expression,
1695	1756	.type = .note,
1696	1757	.token = self.state.token,
...	...	@@ -1698,7 +1759,7 @@ pub const Parser = struct {
1698	1759	});
1699	1760	}
1700	1761	if (is_unary_plus_expression) {
1701		try self.addErrorDetails(ErrorDetails{
	1762	try self.addErrorDetails(.{
1702	1763	.err = .unary_plus_expression,
1703	1764	.type = .note,
1704	1765	.token = self.state.token,
...	...	@@ -1739,7 +1800,7 @@ pub const Parser = struct {
1739	1800	});
1740	1801
1741	1802	if (!rhs_node.isNumberExpression()) {
1742		return self.addErrorDetailsAndFail(ErrorDetails{
	1803	return self.addErrorDetailsAndFail(.{
1743	1804	.err = .expected_something_else,
1744	1805	.token = rhs_node.getFirstToken(),
1745	1806	.token_span_end = rhs_node.getLastToken(),
...	...	@@ -1781,16 +1842,39 @@ pub const Parser = struct {
1781	1842	fn parseOptionalTokenAdvanced(self: *Self, id: Token.Id, comptime method: Lexer.LexMethod) Error!bool {
1782	1843	const maybe_token = try self.lookaheadToken(method);
1783	1844	if (maybe_token.id != id) return false;
1784		self.nextToken(method) catch unreachable;
	1845	try self.nextToken(method);
1785	1846	return true;
1786	1847	}
1787	1848
1788		fn addErrorDetails(self: *Self, details: ErrorDetails) Allocator.Error!void {
	1849	fn addErrorDetailsWithCodePage(self: *Self, details: ErrorDetails) Allocator.Error!void {
1789	1850	try self.state.diagnostics.append(details);
1790	1851	}
1791	1852
1792		fn addErrorDetailsAndFail(self: *Self, details: ErrorDetails) Error {
1793		try self.addErrorDetails(details);
	1853	fn addErrorDetailsWithCodePageAndFail(self: *Self, details: ErrorDetails) Error {
	1854	try self.addErrorDetailsWithCodePage(details);
	1855	return error.ParseError;
	1856	}
	1857
	1858	/// Code page is looked up in input_code_page_lookup using the token, meaning the token
	1859	/// must come from nextToken (i.e. it can't be a lookahead token).
	1860	fn addErrorDetails(self: *Self, details_without_code_page: ErrorDetailsWithoutCodePage) Allocator.Error!void {
	1861	const details = ErrorDetails{
	1862	.err = details_without_code_page.err,
	1863	.code_page = self.state.input_code_page_lookup.getForToken(details_without_code_page.token),
	1864	.token = details_without_code_page.token,
	1865	.token_span_start = details_without_code_page.token_span_start,
	1866	.token_span_end = details_without_code_page.token_span_end,
	1867	.type = details_without_code_page.type,
	1868	.print_source_line = details_without_code_page.print_source_line,
	1869	.extra = details_without_code_page.extra,
	1870	};
	1871	try self.addErrorDetailsWithCodePage(details);
	1872	}
	1873
	1874	/// Code page is looked up in input_code_page_lookup using the token, meaning the token
	1875	/// must come from nextToken (i.e. it can't be a lookahead token).
	1876	fn addErrorDetailsAndFail(self: *Self, details_without_code_page: ErrorDetailsWithoutCodePage) Error {
	1877	try self.addErrorDetails(details_without_code_page);
1794	1878	return error.ParseError;
1795	1879	}
1796	1880
...	...	@@ -1798,35 +1882,34 @@ pub const Parser = struct {
1798	1882	self.state.token = token: while (true) {
1799	1883	const token = self.lexer.next(method) catch \|err\| switch (err) {
1800	1884	error.CodePagePragmaInIncludedFile => {
1801		// The Win32 RC compiler silently ignores such `#pragma code_point` directives,
	1885	// The Win32 RC compiler silently ignores such `#pragma code_page` directives,
1802	1886	// but we want to both ignore them and emit a warning
1803		try self.addErrorDetails(.{
1804		.err = .code_page_pragma_in_included_file,
1805		.type = .warning,
1806		.token = self.lexer.error_context_token.?,
1807		});
	1887	var details = self.lexer.getErrorDetails(err);
	1888	details.type = .warning;
	1889	try self.addErrorDetailsWithCodePage(details);
1808	1890	continue;
1809	1891	},
1810	1892	error.CodePagePragmaInvalidCodePage => {
1811	1893	var details = self.lexer.getErrorDetails(err);
1812	1894	if (!self.options.warn_instead_of_error_on_invalid_code_page) {
1813		return self.addErrorDetailsAndFail(details);
	1895	return self.addErrorDetailsWithCodePageAndFail(details);
1814	1896	}
1815	1897	details.type = .warning;
1816		try self.addErrorDetails(details);
	1898	try self.addErrorDetailsWithCodePage(details);
1817	1899	continue;
1818	1900	},
1819	1901	error.InvalidDigitCharacterInNumberLiteral => {
1820	1902	const details = self.lexer.getErrorDetails(err);
1821		try self.addErrorDetails(details);
1822		return self.addErrorDetailsAndFail(.{
	1903	try self.addErrorDetailsWithCodePage(details);
	1904	return self.addErrorDetailsWithCodePageAndFail(.{
1823	1905	.err = details.err,
1824	1906	.type = .note,
	1907	.code_page = self.lexer.current_code_page,
1825	1908	.token = details.token,
1826	1909	.print_source_line = false,
1827	1910	});
1828	1911	},
1829		else => return self.addErrorDetailsAndFail(self.lexer.getErrorDetails(err)),
	1912	else => return self.addErrorDetailsWithCodePageAndFail(self.lexer.getErrorDetails(err)),
1830	1913	};
1831	1914	break :token token;
1832	1915	};
...	...	@@ -1835,7 +1918,29 @@ pub const Parser = struct {
1835	1918	// But only set the output code page to the current code page if we are past the first code_page pragma in the file.
1836	1919	// Otherwise, we want to fill the lookup using the default code page so that lookups still work for lines that
1837	1920	// don't have an explicit output code page set.
1838		const output_code_page = if (self.lexer.seen_pragma_code_pages > 1) self.lexer.current_code_page else self.state.output_code_page_lookup.default_code_page;
	1921	const is_disjoint_code_page = self.options.disjoint_code_page and self.lexer.seen_pragma_code_pages == 1;
	1922	const output_code_page = if (is_disjoint_code_page)
	1923	self.state.output_code_page_lookup.default_code_page
	1924	else
	1925	self.lexer.current_code_page;
	1926
	1927	if (is_disjoint_code_page and !self.state.warned_about_disjoint_code_page) {
	1928	try self.addErrorDetailsWithCodePage(.{
	1929	.err = .disjoint_code_page,
	1930	.type = .warning,
	1931	.code_page = self.state.input_code_page_lookup.getForLineNum(self.lexer.last_pragma_code_page_token.?.line_number),
	1932	.token = self.lexer.last_pragma_code_page_token.?,
	1933	});
	1934	try self.addErrorDetailsWithCodePage(.{
	1935	.err = .disjoint_code_page,
	1936	.type = .note,
	1937	.code_page = self.state.input_code_page_lookup.getForLineNum(self.lexer.last_pragma_code_page_token.?.line_number),
	1938	.token = self.lexer.last_pragma_code_page_token.?,
	1939	.print_source_line = false,
	1940	});
	1941	self.state.warned_about_disjoint_code_page = true;
	1942	}
	1943
1839	1944	try self.state.output_code_page_lookup.setForToken(self.state.token, output_code_page);
1840	1945	}
1841	1946
...	...	@@ -1846,7 +1951,7 @@ pub const Parser = struct {
1846	1951	// Ignore this error and get the next valid token, we'll deal with this
1847	1952	// properly when getting the token for real
1848	1953	error.CodePagePragmaInIncludedFile => continue,
1849		else => return self.addErrorDetailsAndFail(self.state.lookahead_lexer.getErrorDetails(err)),
	1954	else => return self.addErrorDetailsWithCodePageAndFail(self.state.lookahead_lexer.getErrorDetails(err)),
1850	1955	};
1851	1956	};
1852	1957	}
...	...	@@ -1860,7 +1965,7 @@ pub const Parser = struct {
1860	1965	switch (self.state.token.id) {
1861	1966	.literal => {},
1862	1967	else => {
1863		return self.addErrorDetailsAndFail(ErrorDetails{
	1968	return self.addErrorDetailsAndFail(.{
1864	1969	.err = .expected_token,
1865	1970	.token = self.state.token,
1866	1971	.extra = .{ .expected = .literal },
...	...	@@ -1871,7 +1976,7 @@ pub const Parser = struct {
1871	1976
1872	1977	fn check(self: *Self, expected_token_id: Token.Id) !void {
1873	1978	if (self.state.token.id != expected_token_id) {
1874		return self.addErrorDetailsAndFail(ErrorDetails{
	1979	return self.addErrorDetailsAndFail(.{
1875	1980	.err = .expected_token,
1876	1981	.token = self.state.token,
1877	1982	.extra = .{ .expected = expected_token_id },
...	...	@@ -1879,14 +1984,14 @@ pub const Parser = struct {
1879	1984	}
1880	1985	}
1881	1986
1882		fn checkResource(self: *Self) !Resource {
	1987	fn checkResource(self: *Self) !ResourceType {
1883	1988	switch (self.state.token.id) {
1884		.literal => return Resource.fromString(.{
	1989	.literal => return ResourceType.fromString(.{
1885	1990	.slice = self.state.token.slice(self.lexer.buffer),
1886	1991	.code_page = self.lexer.current_code_page,
1887	1992	}),
1888	1993	else => {
1889		return self.addErrorDetailsAndFail(ErrorDetails{
	1994	return self.addErrorDetailsAndFail(.{
1890	1995	.err = .expected_token,
1891	1996	.token = self.state.token,
1892	1997	.extra = .{ .expected = .literal },

lib/compiler/resinator/preprocess.zig+1

...	...	@@ -96,6 +96,7 @@ pub fn appendAroArgs(arena: Allocator, argv: *std.ArrayList([]const u8), options
96	96	"--emulate=msvc",
97	97	"-nostdinc",
98	98	"-DRC_INVOKED",
	99	"-D_WIN32", // undocumented, but defined by default
99	100	});
100	101	for (options.extra_include_paths.items) \|extra_include_path\| {
101	102	try argv.append("-I");

lib/compiler/resinator/rc.zig+7-7

...	...	@@ -5,7 +5,7 @@ const SourceBytes = @import("literals.zig").SourceBytes;
5	5
6	6	// https://learn.microsoft.com/en-us/windows/win32/menurc/about-resource-files
7	7
8		pub const Resource = enum {
	8	pub const ResourceType = enum {
9	9	accelerators,
10	10	bitmap,
11	11	cursor,
...	...	@@ -48,7 +48,7 @@ pub const Resource = enum {
48	48	manifest_num,
49	49
50	50	const map = std.StaticStringMapWithEql(
51		Resource,
	51	ResourceType,
52	52	std.static_string_map.eqlAsciiIgnoreCase,
53	53	).initComptime(.{
54	54	.{ "ACCELERATORS", .accelerators },
...	...	@@ -72,7 +72,7 @@ pub const Resource = enum {
72	72	.{ "VXD", .vxd },
73	73	});
74	74
75		pub fn fromString(bytes: SourceBytes) Resource {
	75	pub fn fromString(bytes: SourceBytes) ResourceType {
76	76	const maybe_ordinal = res.NameOrOrdinal.maybeOrdinalFromString(bytes);
77	77	if (maybe_ordinal) \|ordinal\| {
78	78	if (ordinal.ordinal >= 256) return .user_defined;
...	...	@@ -81,8 +81,8 @@ pub const Resource = enum {
81	81	return map.get(bytes.slice) orelse .user_defined;
82	82	}
83	83
84		// TODO: Some comptime validation that RT <-> Resource conversion is synced?
85		pub fn fromRT(rt: res.RT) Resource {
	84	// TODO: Some comptime validation that RT <-> ResourceType conversion is synced?
	85	pub fn fromRT(rt: res.RT) ResourceType {
86	86	return switch (rt) {
87	87	.ACCELERATOR => .accelerators,
88	88	.ANICURSOR => .anicursor_num,
...	...	@@ -111,7 +111,7 @@ pub const Resource = enum {
111	111	};
112	112	}
113	113
114		pub fn canUseRawData(resource: Resource) bool {
	114	pub fn canUseRawData(resource: ResourceType) bool {
115	115	return switch (resource) {
116	116	.user_defined,
117	117	.html,
...	...	@@ -125,7 +125,7 @@ pub const Resource = enum {
125	125	};
126	126	}
127	127
128		pub fn nameForErrorDisplay(resource: Resource) []const u8 {
	128	pub fn nameForErrorDisplay(resource: ResourceType) []const u8 {
129	129	return switch (resource) {
130	130	// zig fmt: off
131	131	.accelerators, .bitmap, .cursor, .dialog, .dialogex, .dlginclude, .dlginit, .font,

lib/compiler/resinator/res.zig+175-38

...	...	@@ -1,10 +1,10 @@
1	1	const std = @import("std");
2	2	const rc = @import("rc.zig");
3		const Resource = rc.Resource;
	3	const ResourceType = rc.ResourceType;
4	4	const CommonResourceAttributes = rc.CommonResourceAttributes;
5	5	const Allocator = std.mem.Allocator;
6	6	const windows1252 = @import("windows1252.zig");
7		const CodePage = @import("code_pages.zig").CodePage;
	7	const SupportedCodePage = @import("code_pages.zig").SupportedCodePage;
8	8	const literals = @import("literals.zig");
9	9	const SourceBytes = literals.SourceBytes;
10	10	const Codepoint = @import("code_pages.zig").Codepoint;
...	...	@@ -40,7 +40,7 @@ pub const RT = enum(u8) {
40	40
41	41	/// Returns null if the resource type is user-defined
42	42	/// Asserts that the resource is not `stringtable`
43		pub fn fromResource(resource: Resource) ?RT {
	43	pub fn fromResource(resource: ResourceType) ?RT {
44	44	return switch (resource) {
45	45	.accelerators => .ACCELERATOR,
46	46	.bitmap => .BITMAP,
...	...	@@ -162,6 +162,27 @@ pub const Language = packed struct(u16) {
162	162	pub fn asInt(self: Language) u16 {
163	163	return @bitCast(self);
164	164	}
	165
	166	pub fn format(
	167	language: Language,
	168	comptime fmt: []const u8,
	169	options: std.fmt.FormatOptions,
	170	out_stream: anytype,
	171	) !void {
	172	_ = fmt;
	173	_ = options;
	174	const language_id = language.asInt();
	175	const language_name = language_name: {
	176	if (std.meta.intToEnum(lang.LanguageId, language_id)) \|lang_enum_val\| {
	177	break :language_name @tagName(lang_enum_val);
	178	} else \|_\| {}
	179	if (language_id == lang.LOCALE_CUSTOM_UNSPECIFIED) {
	180	break :language_name "LOCALE_CUSTOM_UNSPECIFIED";
	181	}
	182	break :language_name "<UNKNOWN>";
	183	};
	184	try out_stream.print("{s} (0x{X})", .{ language_name, language_id });
	185	}
165	186	};
166	187
167	188	/// https://learn.microsoft.com/en-us/windows/win32/api/winuser/ns-winuser-dlgitemtemplate#remarks
...	...	@@ -423,6 +444,50 @@ pub const NameOrOrdinal = union(enum) {
423	444	.name => return null,
424	445	}
425	446	}
	447
	448	pub fn format(
	449	self: NameOrOrdinal,
	450	comptime fmt: []const u8,
	451	options: std.fmt.FormatOptions,
	452	out_stream: anytype,
	453	) !void {
	454	_ = fmt;
	455	_ = options;
	456	switch (self) {
	457	.name => \|name\| {
	458	try out_stream.print("{s}", .{std.unicode.fmtUtf16Le(name)});
	459	},
	460	.ordinal => \|ordinal\| {
	461	try out_stream.print("{d}", .{ordinal});
	462	},
	463	}
	464	}
	465
	466	fn formatResourceType(
	467	self: NameOrOrdinal,
	468	comptime fmt: []const u8,
	469	options: std.fmt.FormatOptions,
	470	out_stream: anytype,
	471	) !void {
	472	_ = fmt;
	473	_ = options;
	474	switch (self) {
	475	.name => \|name\| {
	476	try out_stream.print("{s}", .{std.unicode.fmtUtf16Le(name)});
	477	},
	478	.ordinal => \|ordinal\| {
	479	if (std.enums.tagName(RT, @enumFromInt(ordinal))) \|predefined_type_name\| {
	480	try out_stream.print("{s}", .{predefined_type_name});
	481	} else {
	482	try out_stream.print("{d}", .{ordinal});
	483	}
	484	},
	485	}
	486	}
	487
	488	pub fn fmtResourceType(type_value: NameOrOrdinal) std.fmt.Formatter(formatResourceType) {
	489	return .{ .data = type_value };
	490	}
426	491	};
427	492
428	493	fn expectNameOrOrdinal(expected: NameOrOrdinal, actual: NameOrOrdinal) !void {
...	...	@@ -603,12 +668,33 @@ pub const AcceleratorModifiers = struct {
603	668
604	669	const AcceleratorKeyCodepointTranslator = struct {
605	670	string_type: literals.StringType,
	671	output_code_page: SupportedCodePage,
606	672
607	673	pub fn translate(self: @This(), maybe_parsed: ?literals.IterativeStringParser.ParsedCodepoint) ?u21 {
608	674	const parsed = maybe_parsed orelse return null;
609	675	if (parsed.codepoint == Codepoint.invalid) return 0xFFFD;
610		if (parsed.from_escaped_integer and self.string_type == .ascii) {
611		return windows1252.toCodepoint(@truncate(parsed.codepoint));
	676	if (parsed.from_escaped_integer) {
	677	switch (self.string_type) {
	678	.ascii => {
	679	const truncated: u8 = @truncate(parsed.codepoint);
	680	switch (self.output_code_page) {
	681	.utf8 => switch (truncated) {
	682	0...0x7F => return truncated,
	683	else => return 0xFFFD,
	684	},
	685	.windows1252 => return windows1252.toCodepoint(truncated),
	686	}
	687	},
	688	.wide => {
	689	const truncated: u16 = @truncate(parsed.codepoint);
	690	return truncated;
	691	},
	692	}
	693	}
	694	if (parsed.escaped_surrogate_pair) {
	695	// The codepoint of only the low surrogate
	696	const low = @as(u16, @intCast(parsed.codepoint & 0x3FF)) + 0xDC00;
	697	return low;
612	698	}
613	699	return parsed.codepoint;
614	700	}
...	...	@@ -623,14 +709,17 @@ pub fn parseAcceleratorKeyString(bytes: SourceBytes, is_virt: bool, options: lit
623	709	}
624	710
625	711	var parser = literals.IterativeStringParser.init(bytes, options);
626		var translator = AcceleratorKeyCodepointTranslator{ .string_type = parser.declared_string_type };
	712	var translator = AcceleratorKeyCodepointTranslator{
	713	.string_type = parser.declared_string_type,
	714	.output_code_page = options.output_code_page,
	715	};
627	716
628	717	const first_codepoint = translator.translate(try parser.next()) orelse return error.EmptyAccelerator;
629	718	// 0 is treated as a terminator, so this is equivalent to an empty string
630	719	if (first_codepoint == 0) return error.EmptyAccelerator;
631	720
632	721	if (first_codepoint == '^') {
633		// Note: Emitting this warning unconditonally whenever ^ is the first character
	722	// Note: Emitting this warning unconditionally whenever ^ is the first character
634	723	// matches the Win32 RC behavior, but it's questionable whether or not
635	724	// the warning should be emitted for ^^ since that results in the ASCII
636	725	// character ^ being written to the .res.
...	...	@@ -638,11 +727,18 @@ pub fn parseAcceleratorKeyString(bytes: SourceBytes, is_virt: bool, options: lit
638	727	try options.diagnostics.?.diagnostics.append(.{
639	728	.err = .ascii_character_not_equivalent_to_virtual_key_code,
640	729	.type = .warning,
	730	.code_page = bytes.code_page,
641	731	.token = options.diagnostics.?.token,
642	732	});
643	733	}
644	734
645	735	const c = translator.translate(try parser.next()) orelse return error.InvalidControlCharacter;
	736
	737	const third_codepoint = translator.translate(try parser.next());
	738	// 0 is treated as a terminator, so a 0 in the third position is fine but
	739	// anything else is too many codepoints for an accelerator
	740	if (third_codepoint != null and third_codepoint.? != 0) return error.InvalidControlCharacter;
	741
646	742	switch (c) {
647	743	'^' => return '^', // special case
648	744	'a'...'z', 'A'...'Z' => return std.ascii.toUpper(@intCast(c)) - 0x40,
...	...	@@ -699,44 +795,44 @@ test "accelerator keys" {
699	795	try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString(
700	796	.{ .slice = "\"^a\"", .code_page = .windows1252 },
701	797	false,
702		.{},
	798	.{ .output_code_page = .windows1252 },
703	799	));
704	800	try std.testing.expectEqual(@as(u16, 1), try parseAcceleratorKeyString(
705	801	.{ .slice = "\"^A\"", .code_page = .windows1252 },
706	802	false,
707		.{},
	803	.{ .output_code_page = .windows1252 },
708	804	));
709	805	try std.testing.expectEqual(@as(u16, 26), try parseAcceleratorKeyString(
710	806	.{ .slice = "\"^Z\"", .code_page = .windows1252 },
711	807	false,
712		.{},
	808	.{ .output_code_page = .windows1252 },
713	809	));
714	810	try std.testing.expectEqual(@as(u16, '^'), try parseAcceleratorKeyString(
715	811	.{ .slice = "\"^^\"", .code_page = .windows1252 },
716	812	false,
717		.{},
	813	.{ .output_code_page = .windows1252 },
718	814	));
719	815
720	816	try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString(
721	817	.{ .slice = "\"a\"", .code_page = .windows1252 },
722	818	false,
723		.{},
	819	.{ .output_code_page = .windows1252 },
724	820	));
725	821	try std.testing.expectEqual(@as(u16, 0x6162), try parseAcceleratorKeyString(
726	822	.{ .slice = "\"ab\"", .code_page = .windows1252 },
727	823	false,
728		.{},
	824	.{ .output_code_page = .windows1252 },
729	825	));
730	826
731	827	try std.testing.expectEqual(@as(u16, 'C'), try parseAcceleratorKeyString(
732	828	.{ .slice = "\"c\"", .code_page = .windows1252 },
733	829	true,
734		.{},
	830	.{ .output_code_page = .windows1252 },
735	831	));
736	832	try std.testing.expectEqual(@as(u16, 0x6363), try parseAcceleratorKeyString(
737	833	.{ .slice = "\"cc\"", .code_page = .windows1252 },
738	834	true,
739		.{},
	835	.{ .output_code_page = .windows1252 },
740	836	));
741	837
742	838	// \x00 or any escape that evaluates to zero acts as a terminator, everything past it
...	...	@@ -744,93 +840,93 @@ test "accelerator keys" {
744	840	try std.testing.expectEqual(@as(u16, 'a'), try parseAcceleratorKeyString(
745	841	.{ .slice = "\"a\\0bcdef\"", .code_page = .windows1252 },
746	842	false,
747		.{},
	843	.{ .output_code_page = .windows1252 },
748	844	));
749	845
750	846	// \x80 is € in Windows-1252, which is Unicode codepoint 20AC
751	847	try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
752	848	.{ .slice = "\"\x80\"", .code_page = .windows1252 },
753	849	false,
754		.{},
	850	.{ .output_code_page = .windows1252 },
755	851	));
756	852	// This depends on the code page, though, with codepage 65001, \x80
757	853	// on its own is invalid UTF-8 so it gets converted to the replacement character
758	854	try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
759	855	.{ .slice = "\"\x80\"", .code_page = .utf8 },
760	856	false,
761		.{},
	857	.{ .output_code_page = .windows1252 },
762	858	));
763	859	try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
764	860	.{ .slice = "\"\x80\x80\"", .code_page = .windows1252 },
765	861	false,
766		.{},
	862	.{ .output_code_page = .windows1252 },
767	863	));
768	864	// This also behaves the same with escaped characters
769	865	try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
770	866	.{ .slice = "\"\\x80\"", .code_page = .windows1252 },
771	867	false,
772		.{},
	868	.{ .output_code_page = .windows1252 },
773	869	));
774	870	// Even with utf8 code page
775	871	try std.testing.expectEqual(@as(u16, 0x20AC), try parseAcceleratorKeyString(
776	872	.{ .slice = "\"\\x80\"", .code_page = .utf8 },
777	873	false,
778		.{},
	874	.{ .output_code_page = .windows1252 },
779	875	));
780	876	try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
781	877	.{ .slice = "\"\\x80\\x80\"", .code_page = .windows1252 },
782	878	false,
783		.{},
	879	.{ .output_code_page = .windows1252 },
784	880	));
785	881	// Wide string with the actual characters behaves like the ASCII string version
786	882	try std.testing.expectEqual(@as(u16, 0xCCAC), try parseAcceleratorKeyString(
787	883	.{ .slice = "L\"\x80\x80\"", .code_page = .windows1252 },
788	884	false,
789		.{},
	885	.{ .output_code_page = .windows1252 },
790	886	));
791	887	// But wide string with escapes behaves differently
792	888	try std.testing.expectEqual(@as(u16, 0x8080), try parseAcceleratorKeyString(
793	889	.{ .slice = "L\"\\x80\\x80\"", .code_page = .windows1252 },
794	890	false,
795		.{},
	891	.{ .output_code_page = .windows1252 },
796	892	));
797	893	// and invalid escapes within wide strings get skipped
798	894	try std.testing.expectEqual(@as(u16, 'z'), try parseAcceleratorKeyString(
799	895	.{ .slice = "L\"\\Hz\"", .code_page = .windows1252 },
800	896	false,
801		.{},
	897	.{ .output_code_page = .windows1252 },
802	898	));
803	899
804	900	// any non-A-Z codepoints are illegal
805	901	try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
806	902	.{ .slice = "\"^\x83\"", .code_page = .windows1252 },
807	903	false,
808		.{},
	904	.{ .output_code_page = .windows1252 },
809	905	));
810	906	try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
811	907	.{ .slice = "\"^1\"", .code_page = .windows1252 },
812	908	false,
813		.{},
	909	.{ .output_code_page = .windows1252 },
814	910	));
815	911	try std.testing.expectError(error.InvalidControlCharacter, parseAcceleratorKeyString(
816	912	.{ .slice = "\"^\"", .code_page = .windows1252 },
817	913	false,
818		.{},
	914	.{ .output_code_page = .windows1252 },
819	915	));
820	916	try std.testing.expectError(error.EmptyAccelerator, parseAcceleratorKeyString(
821	917	.{ .slice = "\"\"", .code_page = .windows1252 },
822	918	false,
823		.{},
	919	.{ .output_code_page = .windows1252 },
824	920	));
825	921	try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
826	922	.{ .slice = "\"hello\"", .code_page = .windows1252 },
827	923	false,
828		.{},
	924	.{ .output_code_page = .windows1252 },
829	925	));
830	926	try std.testing.expectError(error.ControlCharacterOutOfRange, parseAcceleratorKeyString(
831	927	.{ .slice = "\"^\x80\"", .code_page = .windows1252 },
832	928	false,
833		.{},
	929	.{ .output_code_page = .windows1252 },
834	930	));
835	931
836	932	// Invalid UTF-8 gets converted to 0xFFFD, multiple invalids get shifted and added together
...	...	@@ -838,40 +934,81 @@ test "accelerator keys" {
838	934	try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString(
839	935	.{ .slice = "\"\x80\x80\"", .code_page = .utf8 },
840	936	false,
841		.{},
	937	.{ .output_code_page = .windows1252 },
842	938	));
843	939	try std.testing.expectEqual(@as(u16, 0xFCFD), try parseAcceleratorKeyString(
844	940	.{ .slice = "L\"\x80\x80\"", .code_page = .utf8 },
845	941	false,
846		.{},
	942	.{ .output_code_page = .windows1252 },
847	943	));
848	944
849	945	// Codepoints >= 0x10000
850	946	try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString(
851	947	.{ .slice = "\"\xF0\x90\x84\x80\"", .code_page = .utf8 },
852	948	false,
853		.{},
	949	.{ .output_code_page = .windows1252 },
854	950	));
855	951	try std.testing.expectEqual(@as(u16, 0xDD00), try parseAcceleratorKeyString(
856	952	.{ .slice = "L\"\xF0\x90\x84\x80\"", .code_page = .utf8 },
857	953	false,
858		.{},
	954	.{ .output_code_page = .windows1252 },
859	955	));
860	956	try std.testing.expectEqual(@as(u16, 0x9C01), try parseAcceleratorKeyString(
861	957	.{ .slice = "\"\xF4\x80\x80\x81\"", .code_page = .utf8 },
862	958	false,
863		.{},
	959	.{ .output_code_page = .windows1252 },
864	960	));
865	961	// anything before or after a codepoint >= 0x10000 causes an error
866	962	try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
867	963	.{ .slice = "\"a\xF0\x90\x80\x80\"", .code_page = .utf8 },
868	964	false,
869		.{},
	965	.{ .output_code_page = .windows1252 },
870	966	));
871	967	try std.testing.expectError(error.AcceleratorTooLong, parseAcceleratorKeyString(
872	968	.{ .slice = "\"\xF0\x90\x80\x80a\"", .code_page = .utf8 },
873	969	false,
874		.{},
	970	.{ .output_code_page = .windows1252 },
	971	));
	972
	973	// Misc special cases
	974	try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
	975	.{ .slice = "\"\\777\"", .code_page = .utf8 },
	976	false,
	977	.{ .output_code_page = .utf8 },
	978	));
	979	try std.testing.expectEqual(@as(u16, 0xFFFF), try parseAcceleratorKeyString(
	980	.{ .slice = "L\"\\7777777\"", .code_page = .utf8 },
	981	false,
	982	.{ .output_code_page = .utf8 },
	983	));
	984	try std.testing.expectEqual(@as(u16, 0x01), try parseAcceleratorKeyString(
	985	.{ .slice = "L\"\\200001\"", .code_page = .utf8 },
	986	false,
	987	.{ .output_code_page = .utf8 },
	988	));
	989	// Escape of a codepoint >= 0x10000 omits the high surrogate pair
	990	try std.testing.expectEqual(@as(u16, 0xDF48), try parseAcceleratorKeyString(
	991	.{ .slice = "L\"\\𐍈\"", .code_page = .utf8 },
	992	false,
	993	.{ .output_code_page = .utf8 },
	994	));
	995	// Invalid escape code is skipped, allows for 2 codepoints afterwards
	996	try std.testing.expectEqual(@as(u16, 0x7878), try parseAcceleratorKeyString(
	997	.{ .slice = "L\"\\kxx\"", .code_page = .utf8 },
	998	false,
	999	.{ .output_code_page = .utf8 },
	1000	));
	1001	// Escape of a codepoint >= 0x10000 allows for a codepoint afterwards
	1002	try std.testing.expectEqual(@as(u16, 0x4878), try parseAcceleratorKeyString(
	1003	.{ .slice = "L\"\\𐍈x\"", .code_page = .utf8 },
	1004	false,
	1005	.{ .output_code_page = .utf8 },
	1006	));
	1007	// Input code page of 1252, output code page of utf-8
	1008	try std.testing.expectEqual(@as(u16, 0xFFFD), try parseAcceleratorKeyString(
	1009	.{ .slice = "\"\\270\"", .code_page = .windows1252 },
	1010	false,
	1011	.{ .output_code_page = .utf8 },
875	1012	));
876	1013	}
877	1014

lib/compiler/resinator/source_mapping.zig+438-41

...	...	@@ -38,7 +38,7 @@ pub const ParseAndRemoveLineCommandsOptions = struct {
38	38	///
39	39	/// If `options.initial_filename` is provided, that filename is guaranteed to be
40	40	/// within the `mappings.files` table and `root_filename_offset` will be set appropriately.
41		pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) !ParseLineCommandsResult {
	41	pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf: []u8, options: ParseAndRemoveLineCommandsOptions) error{ OutOfMemory, InvalidLineCommand, LineNumberOverflow }!ParseLineCommandsResult {
42	42	var parse_result = ParseLineCommandsResult{
43	43	.result = undefined,
44	44	.mappings = .{},
...	...	@@ -53,12 +53,41 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
53	53	parse_result.mappings.root_filename_offset = try parse_result.mappings.files.put(allocator, initial_filename);
54	54	}
55	55
	56	// This implementation attempts to be comment and string aware in order
	57	// to avoid errant #line <num> "<filename>" within multiline comments
	58	// leading to problems in the source mapping after comments are removed,
	59	// but it is not a perfect implementation (intentionally).
	60	//
	61	// The current implementation does not handle cases like
	62	// /* foo */ #line ...
	63	// #line ... // foo
	64	// #line ... /* foo ...
	65	// etc
	66	//
	67	// (the first example will not be recognized as a #line command, the second
	68	// and third will error with InvalidLineCommand)
	69	//
	70	// This is fine, though, since #line commands are generated by the
	71	// preprocessor so in normal circumstances they will be well-formed and
	72	// consistent. The only realistic way the imperfect implementation could
	73	// affect a 'real' use-case would be someone taking the output of a
	74	// preprocessor, editing it manually to add comments before/after #line
	75	// commands, and then running it through resinator with /:no-preprocess.
	76
56	77	std.debug.assert(buf.len >= source.len);
57	78	var result = UncheckedSliceWriter{ .slice = buf };
58	79	const State = enum {
59	80	line_start,
60	81	preprocessor,
61	82	non_preprocessor,
	83	forward_slash,
	84	line_comment,
	85	multiline_comment,
	86	multiline_comment_end,
	87	single_quoted,
	88	single_quoted_escape,
	89	double_quoted,
	90	double_quoted_escape,
62	91	};
63	92	var state: State = .line_start;
64	93	var index: usize = 0;
...	...	@@ -66,8 +95,8 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
66	95	var preprocessor_start: usize = 0;
67	96	var line_number: usize = 1;
68	97	while (index < source.len) : (index += 1) {
69		const c = source[index];
70		switch (state) {
	98	var c = source[index];
	99	state: switch (state) {
71	100	.line_start => switch (c) {
72	101	'#' => {
73	102	preprocessor_start = index;
...	...	@@ -93,6 +122,27 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
93	122	pending_start = index;
94	123	}
95	124	},
	125	'/' => {
	126	if (!current_mapping.ignore_contents) {
	127	result.writeSlice(source[pending_start orelse index .. index + 1]);
	128	pending_start = null;
	129	}
	130	state = .forward_slash;
	131	},
	132	'\'' => {
	133	if (!current_mapping.ignore_contents) {
	134	result.writeSlice(source[pending_start orelse index .. index + 1]);
	135	pending_start = null;
	136	}
	137	state = .single_quoted;
	138	},
	139	'"' => {
	140	if (!current_mapping.ignore_contents) {
	141	result.writeSlice(source[pending_start orelse index .. index + 1]);
	142	pending_start = null;
	143	}
	144	state = .double_quoted;
	145	},
96	146	else => {
97	147	state = .non_preprocessor;
98	148	if (pending_start != null) {
...	...	@@ -107,25 +157,246 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
107	157	}
108	158	},
109	159	},
	160	.forward_slash => switch (c) {
	161	'\r', '\n' => {
	162	const is_crlf = formsLineEndingPair(source, c, index + 1);
	163	if (!current_mapping.ignore_contents) {
	164	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	165
	166	result.write(c);
	167	if (is_crlf) result.write(source[index + 1]);
	168	line_number += 1;
	169	}
	170	if (is_crlf) index += 1;
	171	state = .line_start;
	172	pending_start = null;
	173	},
	174	'/' => {
	175	if (!current_mapping.ignore_contents) {
	176	result.write(c);
	177	}
	178	state = .line_comment;
	179	},
	180	'*' => {
	181	if (!current_mapping.ignore_contents) {
	182	result.write(c);
	183	}
	184	state = .multiline_comment;
	185	},
	186	else => {
	187	if (!current_mapping.ignore_contents) {
	188	result.write(c);
	189	}
	190	state = .non_preprocessor;
	191	},
	192	},
	193	.line_comment => switch (c) {
	194	'\r', '\n' => {
	195	const is_crlf = formsLineEndingPair(source, c, index + 1);
	196	if (!current_mapping.ignore_contents) {
	197	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	198
	199	result.write(c);
	200	if (is_crlf) result.write(source[index + 1]);
	201	line_number += 1;
	202	}
	203	if (is_crlf) index += 1;
	204	state = .line_start;
	205	pending_start = null;
	206	},
	207	else => {
	208	if (!current_mapping.ignore_contents) {
	209	result.write(c);
	210	}
	211	},
	212	},
	213	.multiline_comment => switch (c) {
	214	'\r', '\n' => {
	215	const is_crlf = formsLineEndingPair(source, c, index + 1);
	216	if (!current_mapping.ignore_contents) {
	217	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	218
	219	result.write(c);
	220	if (is_crlf) result.write(source[index + 1]);
	221	line_number += 1;
	222	}
	223	if (is_crlf) index += 1;
	224	pending_start = null;
	225	},
	226	'*' => {
	227	if (!current_mapping.ignore_contents) {
	228	result.write(c);
	229	}
	230	state = .multiline_comment_end;
	231	},
	232	else => {
	233	if (!current_mapping.ignore_contents) {
	234	result.write(c);
	235	}
	236	},
	237	},
	238	.multiline_comment_end => switch (c) {
	239	'\r', '\n' => {
	240	const is_crlf = formsLineEndingPair(source, c, index + 1);
	241	if (!current_mapping.ignore_contents) {
	242	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	243
	244	result.write(c);
	245	if (is_crlf) result.write(source[index + 1]);
	246	line_number += 1;
	247	}
	248	if (is_crlf) index += 1;
	249	state = .multiline_comment;
	250	pending_start = null;
	251	},
	252	'/' => {
	253	if (!current_mapping.ignore_contents) {
	254	result.write(c);
	255	}
	256	state = .non_preprocessor;
	257	},
	258	'*' => {
	259	if (!current_mapping.ignore_contents) {
	260	result.write(c);
	261	}
	262	// stay in multiline_comment_end state
	263	},
	264	else => {
	265	if (!current_mapping.ignore_contents) {
	266	result.write(c);
	267	}
	268	state = .multiline_comment;
	269	},
	270	},
	271	.single_quoted => switch (c) {
	272	'\r', '\n' => {
	273	const is_crlf = formsLineEndingPair(source, c, index + 1);
	274	if (!current_mapping.ignore_contents) {
	275	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	276
	277	result.write(c);
	278	if (is_crlf) result.write(source[index + 1]);
	279	line_number += 1;
	280	}
	281	if (is_crlf) index += 1;
	282	state = .line_start;
	283	pending_start = null;
	284	},
	285	'\\' => {
	286	if (!current_mapping.ignore_contents) {
	287	result.write(c);
	288	}
	289	state = .single_quoted_escape;
	290	},
	291	'\'' => {
	292	if (!current_mapping.ignore_contents) {
	293	result.write(c);
	294	}
	295	state = .non_preprocessor;
	296	},
	297	else => {
	298	if (!current_mapping.ignore_contents) {
	299	result.write(c);
	300	}
	301	},
	302	},
	303	.single_quoted_escape => switch (c) {
	304	'\r', '\n' => {
	305	const is_crlf = formsLineEndingPair(source, c, index + 1);
	306	if (!current_mapping.ignore_contents) {
	307	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	308
	309	result.write(c);
	310	if (is_crlf) result.write(source[index + 1]);
	311	line_number += 1;
	312	}
	313	if (is_crlf) index += 1;
	314	state = .line_start;
	315	pending_start = null;
	316	},
	317	else => {
	318	if (!current_mapping.ignore_contents) {
	319	result.write(c);
	320	}
	321	state = .single_quoted;
	322	},
	323	},
	324	.double_quoted => switch (c) {
	325	'\r', '\n' => {
	326	const is_crlf = formsLineEndingPair(source, c, index + 1);
	327	if (!current_mapping.ignore_contents) {
	328	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	329
	330	result.write(c);
	331	if (is_crlf) result.write(source[index + 1]);
	332	line_number += 1;
	333	}
	334	if (is_crlf) index += 1;
	335	state = .line_start;
	336	pending_start = null;
	337	},
	338	'\\' => {
	339	if (!current_mapping.ignore_contents) {
	340	result.write(c);
	341	}
	342	state = .double_quoted_escape;
	343	},
	344	'"' => {
	345	if (!current_mapping.ignore_contents) {
	346	result.write(c);
	347	}
	348	state = .non_preprocessor;
	349	},
	350	else => {
	351	if (!current_mapping.ignore_contents) {
	352	result.write(c);
	353	}
	354	},
	355	},
	356	.double_quoted_escape => switch (c) {
	357	'\r', '\n' => {
	358	const is_crlf = formsLineEndingPair(source, c, index + 1);
	359	if (!current_mapping.ignore_contents) {
	360	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
	361
	362	result.write(c);
	363	if (is_crlf) result.write(source[index + 1]);
	364	line_number += 1;
	365	}
	366	if (is_crlf) index += 1;
	367	state = .line_start;
	368	pending_start = null;
	369	},
	370	else => {
	371	if (!current_mapping.ignore_contents) {
	372	result.write(c);
	373	}
	374	state = .double_quoted;
	375	},
	376	},
110	377	.preprocessor => switch (c) {
111	378	'\r', '\n' => {
112	379	// Now that we have the full line we can decide what to do with it
113	380	const preprocessor_str = source[preprocessor_start..index];
114		const is_crlf = formsLineEndingPair(source, c, index + 1);
115	381	if (std.mem.startsWith(u8, preprocessor_str, "#line")) {
116	382	try handleLineCommand(allocator, preprocessor_str, &current_mapping);
	383	const is_crlf = formsLineEndingPair(source, c, index + 1);
	384	if (is_crlf) index += 1;
	385	state = .line_start;
	386	pending_start = null;
117	387	} else {
118		if (!current_mapping.ignore_contents) {
119		try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
120
121		const line_ending_len: usize = if (is_crlf) 2 else 1;
122		result.writeSlice(source[pending_start.? .. index + line_ending_len]);
123		line_number += 1;
124		}
	388	// Backtrack and reparse the line in the non_preprocessor state,
	389	// since it's possible that this line contains a multiline comment
	390	// start, etc.
	391	state = .non_preprocessor;
	392	index = pending_start.?;
	393	pending_start = null;
	394	// TODO: This is a hacky way to implement this, c needs to be
	395	// updated since we're using continue :state here
	396	c = source[index];
	397	// continue to avoid the index += 1 of the while loop
	398	continue :state .non_preprocessor;
125	399	}
126		if (is_crlf) index += 1;
127		state = .line_start;
128		pending_start = null;
129	400	},
130	401	else => {},
131	402	},
...	...	@@ -143,6 +414,24 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
143	414	state = .line_start;
144	415	pending_start = null;
145	416	},
	417	'/' => {
	418	if (!current_mapping.ignore_contents) {
	419	result.write(c);
	420	}
	421	state = .forward_slash;
	422	},
	423	'\'' => {
	424	if (!current_mapping.ignore_contents) {
	425	result.write(c);
	426	}
	427	state = .single_quoted;
	428	},
	429	'"' => {
	430	if (!current_mapping.ignore_contents) {
	431	result.write(c);
	432	}
	433	state = .double_quoted;
	434	},
146	435	else => {
147	436	if (!current_mapping.ignore_contents) {
148	437	result.write(c);
...	...	@@ -153,7 +442,16 @@ pub fn parseAndRemoveLineCommands(allocator: Allocator, source: []const u8, buf:
153	442	} else {
154	443	switch (state) {
155	444	.line_start => {},
156		.non_preprocessor => {
	445	.forward_slash,
	446	.line_comment,
	447	.multiline_comment,
	448	.multiline_comment_end,
	449	.single_quoted,
	450	.single_quoted_escape,
	451	.double_quoted,
	452	.double_quoted_escape,
	453	.non_preprocessor,
	454	=> {
157	455	try handleLineEnd(allocator, line_number, &parse_result.mappings, &current_mapping);
158	456	},
159	457	.preprocessor => {
...	...	@@ -207,34 +505,40 @@ pub fn handleLineEnd(allocator: Allocator, post_processed_line_number: usize, ma
207	505
208	506	try mapping.set(post_processed_line_number, current_mapping.line_num, filename_offset);
209	507
210		current_mapping.line_num += 1;
	508	current_mapping.line_num = std.math.add(usize, current_mapping.line_num, 1) catch return error.LineNumberOverflow;
211	509	current_mapping.pending = false;
212	510	}
213	511
214	512	// TODO: Might want to provide diagnostics on invalid line commands instead of just returning
215		pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{OutOfMemory}!void {
	513	pub fn handleLineCommand(allocator: Allocator, line_command: []const u8, current_mapping: *CurrentMapping) error{ OutOfMemory, InvalidLineCommand }!void {
216	514	// TODO: Are there other whitespace characters that should be included?
217	515	var tokenizer = std.mem.tokenizeAny(u8, line_command, " \t");
218		const line_directive = tokenizer.next() orelse return; // #line
219		if (!std.mem.eql(u8, line_directive, "#line")) return;
220		const linenum_str = tokenizer.next() orelse return;
221		const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return;
	516	const line_directive = tokenizer.next() orelse return error.InvalidLineCommand; // #line
	517	if (!std.mem.eql(u8, line_directive, "#line")) return error.InvalidLineCommand;
	518	const linenum_str = tokenizer.next() orelse return error.InvalidLineCommand;
	519	const linenum = std.fmt.parseUnsigned(usize, linenum_str, 10) catch return error.InvalidLineCommand;
	520	if (linenum == 0) return error.InvalidLineCommand;
222	521
223	522	var filename_literal = tokenizer.rest();
224	523	while (filename_literal.len > 0 and std.ascii.isWhitespace(filename_literal[filename_literal.len - 1])) {
225	524	filename_literal.len -= 1;
226	525	}
227		if (filename_literal.len < 2) return;
	526	if (filename_literal.len < 2) return error.InvalidLineCommand;
228	527	const is_quoted = filename_literal[0] == '"' and filename_literal[filename_literal.len - 1] == '"';
229		if (!is_quoted) return;
230		const filename = parseFilename(allocator, filename_literal[1 .. filename_literal.len - 1]) catch \|err\| switch (err) {
	528	if (!is_quoted) return error.InvalidLineCommand;
	529	const unquoted_filename = filename_literal[1 .. filename_literal.len - 1];
	530
	531	// Ignore <builtin> and <command line>
	532	if (std.mem.eql(u8, unquoted_filename, "<builtin>") or std.mem.eql(u8, unquoted_filename, "<command line>")) return;
	533
	534	const filename = parseFilename(allocator, unquoted_filename) catch \|err\| switch (err) {
231	535	error.OutOfMemory => \|e\| return e,
232		else => return,
	536	else => return error.InvalidLineCommand,
233	537	};
234	538	defer allocator.free(filename);
235	539
236	540	// \x00 bytes in the filename is incompatible with how StringTable works
237		if (std.mem.indexOfScalar(u8, filename, '\x00') != null) return;
	541	if (std.mem.indexOfScalar(u8, filename, '\x00') != null) return error.InvalidLineCommand;
238	542
239	543	current_mapping.line_num = linenum;
240	544	current_mapping.filename.clearRetainingCapacity();
...	...	@@ -494,8 +798,12 @@ pub const SourceMappings = struct {
494	798	if (node.key.filename_offset != filename_offset) {
495	799	break :need_new_node true;
496	800	}
497		const exist_delta = @as(i64, @intCast(node.key.corresponding_start_line)) - @as(i64, @intCast(node.key.start_line));
498		const cur_delta = @as(i64, @intCast(corresponding_line_num)) - @as(i64, @intCast(line_num));
	801	// TODO: These use i65 to avoid truncation when any of the line number values
	802	// use all 64 bits of the usize. In reality, line numbers can't really
	803	// get that large so limiting the line number and using a smaller iX
	804	// type here might be a better solution.
	805	const exist_delta = @as(i65, @intCast(node.key.corresponding_start_line)) - @as(i65, @intCast(node.key.start_line));
	806	const cur_delta = @as(i65, @intCast(corresponding_line_num)) - @as(i65, @intCast(line_num));
499	807	if (exist_delta != cur_delta) {
500	808	break :need_new_node true;
501	809	}
...	...	@@ -578,15 +886,8 @@ pub const SourceMappings = struct {
578	886	inorder_node.key.start_line -= span_diff;
579	887
580	888	// This can only really happen if there are #line commands within
581		// a multiline comment, which in theory should be skipped over.
582		// However, currently, parseAndRemoveLineCommands is not aware of
583		// comments at all.
584		//
585		// TODO: Make parseAndRemoveLineCommands aware of comments/strings
586		// and turn this into an assertion
587		if (prev.key.start_line > inorder_node.key.start_line) {
588		return error.InvalidSourceMappingCollapse;
589		}
	889	// a multiline comment, which should be skipped over.
	890	std.debug.assert(prev.key.start_line <= inorder_node.key.start_line);
590	891	prev = inorder_node;
591	892	}
592	893	self.end_line -= span_diff;
...	...	@@ -594,7 +895,7 @@ pub const SourceMappings = struct {
594	895
595	896	/// Returns true if the line is from the main/root file (i.e. not a file that has been
596	897	/// `#include`d).
597		pub fn isRootFile(self: *SourceMappings, line_num: usize) bool {
	898	pub fn isRootFile(self: *const SourceMappings, line_num: usize) bool {
598	899	const source = self.get(line_num) orelse return false;
599	900	return source.filename_offset == self.root_filename_offset;
600	901	}
...	...	@@ -803,9 +1104,6 @@ test "in place" {
803	1104	}
804	1105
805	1106	test "line command within a multiline comment" {
806		// TODO: Enable once parseAndRemoveLineCommands is comment-aware
807		if (true) return error.SkipZigTest;
808
809	1107	try testParseAndRemoveLineCommands(
810	1108	\\/*
811	1109	\\#line 1 "irrelevant.rc"
...	...	@@ -825,4 +1123,103 @@ test "line command within a multiline comment" {
825	1123	\\
826	1124	\\*/
827	1125	, .{ .initial_filename = "blah.rc" });
	1126
	1127	// * but without / directly after
	1128	try testParseAndRemoveLineCommands(
	1129	\\/** /
	1130	\\#line 1 "irrelevant.rc"
	1131	\\*/
	1132	, &[_]ExpectedSourceSpan{
	1133	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
	1134	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
	1135	.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
	1136	},
	1137	\\/** /
	1138	\\#line 1 "irrelevant.rc"
	1139	\\*/
	1140	, .{ .initial_filename = "blah.rc" });
	1141
	1142	// / and /
	1143	try testParseAndRemoveLineCommands(
	1144	\\/**
	1145	\\#line 1 "irrelevant.rc"
	1146	\\**/
	1147	\\foo
	1148	, &[_]ExpectedSourceSpan{
	1149	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
	1150	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
	1151	.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
	1152	.{ .start_line = 20, .end_line = 20, .filename = "blah.rc" },
	1153	},
	1154	\\/**
	1155	\\#line 1 "irrelevant.rc"
	1156	\\**/
	1157	\\#line 20 "blah.rc"
	1158	\\foo
	1159	, .{ .initial_filename = "blah.rc" });
	1160	}
	1161
	1162	test "whitespace preservation" {
	1163	try testParseAndRemoveLineCommands(
	1164	\\ /
	1165	\\/
	1166	, &[_]ExpectedSourceSpan{
	1167	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
	1168	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
	1169	},
	1170	\\ /
	1171	\\/
	1172	, .{ .initial_filename = "blah.rc" });
	1173	}
	1174
	1175	test "preprocessor line with a multiline comment after" {
	1176	try testParseAndRemoveLineCommands(
	1177	\\#pragma test /*
	1178	\\
	1179	\\*/
	1180	, &[_]ExpectedSourceSpan{
	1181	.{ .start_line = 1, .end_line = 1, .filename = "blah.rc" },
	1182	.{ .start_line = 2, .end_line = 2, .filename = "blah.rc" },
	1183	.{ .start_line = 3, .end_line = 3, .filename = "blah.rc" },
	1184	},
	1185	\\#pragma test /*
	1186	\\
	1187	\\*/
	1188	, .{ .initial_filename = "blah.rc" });
	1189	}
	1190
	1191	test "comment after line command" {
	1192	var mut_source = "#line 1 \"blah.rc\" /".;
	1193	try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
	1194	}
	1195
	1196	test "line command with 0 as line number" {
	1197	var mut_source = "#line 0 \"blah.rc\"".*;
	1198	try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
	1199	}
	1200
	1201	test "line number limits" {
	1202	// TODO: Avoid usize for line numbers
	1203	if (@sizeOf(usize) != 8) return error.SkipZigTest;
	1204
	1205	// greater than i64 max
	1206	try testParseAndRemoveLineCommands(
	1207	\\
	1208	, &[_]ExpectedSourceSpan{
	1209	.{ .start_line = 11111111111111111111, .end_line = 11111111111111111111, .filename = "blah.rc" },
	1210	},
	1211	\\#line 11111111111111111111 "blah.rc"
	1212	, .{ .initial_filename = "blah.rc" });
	1213
	1214	// equal to u64 max, overflows on line number increment
	1215	{
	1216	var mut_source = "#line 18446744073709551615 \"blah.rc\"".*;
	1217	try std.testing.expectError(error.LineNumberOverflow, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
	1218	}
	1219
	1220	// greater than u64 max
	1221	{
	1222	var mut_source = "#line 18446744073709551616 \"blah.rc\"".*;
	1223	try std.testing.expectError(error.InvalidLineCommand, parseAndRemoveLineCommands(std.testing.allocator, &mut_source, &mut_source, .{}));
	1224	}
828	1225	}