| ... | @@ -2,6 +2,7 @@ | ... | @@ -2,6 +2,7 @@ |
| 2 | //! and stage2. | 2 | //! and stage2. |
| 3 | | 3 | |
| 4 | const std = @import("std"); | 4 | const std = @import("std"); |
| | 5 | const testing = std.testing; |
| 5 | const assert = std.debug.assert; | 6 | const assert = std.debug.assert; |
| 6 | const clang = @import("clang.zig"); | 7 | const clang = @import("clang.zig"); |
| 7 | const ctok = std.c.tokenizer; | 8 | const ctok = std.c.tokenizer; |
| ... | @@ -18,6 +19,7 @@ const CallingConvention = std.builtin.CallingConvention; | ... | @@ -18,6 +19,7 @@ const CallingConvention = std.builtin.CallingConvention; |
| 18 | pub const ClangErrMsg = clang.Stage2ErrorMsg; | 19 | pub const ClangErrMsg = clang.Stage2ErrorMsg; |
| 19 | | 20 | |
| 20 | pub const Error = std.mem.Allocator.Error; | 21 | pub const Error = std.mem.Allocator.Error; |
| | 22 | const MacroProcessingError = Error || error{UnexpectedMacroToken}; |
| 21 | const TypeError = Error || error{UnsupportedType}; | 23 | const TypeError = Error || error{UnsupportedType}; |
| 22 | const TransError = TypeError || error{UnsupportedTranslation}; | 24 | const TransError = TypeError || error{UnsupportedTranslation}; |
| 23 | | 25 | |
| ... | @@ -27,6 +29,10 @@ const AliasList = std.ArrayList(struct { | ... | @@ -27,6 +29,10 @@ const AliasList = std.ArrayList(struct { |
| 27 | name: []const u8, | 29 | name: []const u8, |
| 28 | }); | 30 | }); |
| 29 | | 31 | |
| | 32 | // Maps macro parameter names to token position, for determining if different |
| | 33 | // identifiers refer to the same positional argument in different macros. |
| | 34 | const ArgsPositionMap = std.StringArrayHashMapUnmanaged(usize); |
| | 35 | |
| 30 | const Scope = struct { | 36 | const Scope = struct { |
| 31 | id: Id, | 37 | id: Id, |
| 32 | parent: ?*Scope, | 38 | parent: ?*Scope, |
| ... | @@ -322,6 +328,8 @@ pub const Context = struct { | ... | @@ -322,6 +328,8 @@ pub const Context = struct { |
| 322 | /// up front in a pre-processing step. | 328 | /// up front in a pre-processing step. |
| 323 | global_names: std.StringArrayHashMapUnmanaged(void) = .{}, | 329 | global_names: std.StringArrayHashMapUnmanaged(void) = .{}, |
| 324 | | 330 | |
| | 331 | pattern_list: PatternList, |
| | 332 | |
| 325 | fn getMangle(c: *Context) u32 { | 333 | fn getMangle(c: *Context) u32 { |
| 326 | c.mangle_count += 1; | 334 | c.mangle_count += 1; |
| 327 | return c.mangle_count; | 335 | return c.mangle_count; |
| ... | @@ -375,6 +383,7 @@ pub fn translate( | ... | @@ -375,6 +383,7 @@ pub fn translate( |
| 375 | .alias_list = AliasList.init(gpa), | 383 | .alias_list = AliasList.init(gpa), |
| 376 | .global_scope = try arena.allocator.create(Scope.Root), | 384 | .global_scope = try arena.allocator.create(Scope.Root), |
| 377 | .clang_context = ast_unit.getASTContext(), | 385 | .clang_context = ast_unit.getASTContext(), |
| | 386 | .pattern_list = try PatternList.init(gpa), |
| 378 | }; | 387 | }; |
| 379 | context.global_scope.* = Scope.Root.init(&context); | 388 | context.global_scope.* = Scope.Root.init(&context); |
| 380 | defer { | 389 | defer { |
| ... | @@ -385,6 +394,7 @@ pub fn translate( | ... | @@ -385,6 +394,7 @@ pub fn translate( |
| 385 | context.unnamed_typedefs.deinit(gpa); | 394 | context.unnamed_typedefs.deinit(gpa); |
| 386 | context.typedefs.deinit(gpa); | 395 | context.typedefs.deinit(gpa); |
| 387 | context.global_scope.deinit(); | 396 | context.global_scope.deinit(); |
| | 397 | context.pattern_list.deinit(gpa); |
| 388 | } | 398 | } |
| 389 | | 399 | |
| 390 | try context.global_scope.nodes.append(Tag.usingnamespace_builtins.init()); | 400 | try context.global_scope.nodes.append(Tag.usingnamespace_builtins.init()); |
| ... | @@ -4829,6 +4839,217 @@ fn isZigPrimitiveType(name: []const u8) bool { | ... | @@ -4829,6 +4839,217 @@ fn isZigPrimitiveType(name: []const u8) bool { |
| 4829 | return @import("AstGen.zig").simple_types.has(name); | 4839 | return @import("AstGen.zig").simple_types.has(name); |
| 4830 | } | 4840 | } |
| 4831 | | 4841 | |
| | 4842 | const PatternList = struct { |
| | 4843 | patterns: []Pattern, |
| | 4844 | |
| | 4845 | /// Templates must be function-like macros |
| | 4846 | /// first element is macro source, second element is the name of the function |
| | 4847 | /// in std.lib.zig.c_translation.Macros which implements it |
| | 4848 | const templates = [_][2][]const u8{ |
| | 4849 | [2][]const u8{ "f_SUFFIX(X) (X ## f)", "F_SUFFIX" }, |
| | 4850 | [2][]const u8{ "F_SUFFIX(X) (X ## F)", "F_SUFFIX" }, |
| | 4851 | |
| | 4852 | [2][]const u8{ "u_SUFFIX(X) (X ## u)", "U_SUFFIX" }, |
| | 4853 | [2][]const u8{ "U_SUFFIX(X) (X ## U)", "U_SUFFIX" }, |
| | 4854 | |
| | 4855 | [2][]const u8{ "l_SUFFIX(X) (X ## l)", "L_SUFFIX" }, |
| | 4856 | [2][]const u8{ "L_SUFFIX(X) (X ## L)", "L_SUFFIX" }, |
| | 4857 | |
| | 4858 | [2][]const u8{ "ul_SUFFIX(X) (X ## ul)", "UL_SUFFIX" }, |
| | 4859 | [2][]const u8{ "uL_SUFFIX(X) (X ## uL)", "UL_SUFFIX" }, |
| | 4860 | [2][]const u8{ "Ul_SUFFIX(X) (X ## Ul)", "UL_SUFFIX" }, |
| | 4861 | [2][]const u8{ "UL_SUFFIX(X) (X ## UL)", "UL_SUFFIX" }, |
| | 4862 | |
| | 4863 | [2][]const u8{ "ll_SUFFIX(X) (X ## ll)", "LL_SUFFIX" }, |
| | 4864 | [2][]const u8{ "LL_SUFFIX(X) (X ## LL)", "LL_SUFFIX" }, |
| | 4865 | |
| | 4866 | [2][]const u8{ "ull_SUFFIX(X) (X ## ull)", "ULL_SUFFIX" }, |
| | 4867 | [2][]const u8{ "uLL_SUFFIX(X) (X ## uLL)", "ULL_SUFFIX" }, |
| | 4868 | [2][]const u8{ "Ull_SUFFIX(X) (X ## Ull)", "ULL_SUFFIX" }, |
| | 4869 | [2][]const u8{ "ULL_SUFFIX(X) (X ## ULL)", "ULL_SUFFIX" }, |
| | 4870 | |
| | 4871 | [2][]const u8{ |
| | 4872 | \\wl_container_of(ptr, sample, member) \ |
| | 4873 | \\(__typeof__(sample))((char *)(ptr) - \ |
| | 4874 | \\ offsetof(__typeof__(*sample), member)) |
| | 4875 | , |
| | 4876 | "WL_CONTAINER_OF", |
| | 4877 | }, |
| | 4878 | }; |
| | 4879 | |
| | 4880 | /// Assumes that `ms` represents a tokenized function-like macro. |
| | 4881 | fn buildArgsHash(allocator: *mem.Allocator, ms: MacroSlicer, hash: *ArgsPositionMap) MacroProcessingError!void { |
| | 4882 | assert(ms.tokens.len > 2); |
| | 4883 | assert(ms.tokens[0].id == .Identifier); |
| | 4884 | assert(ms.tokens[1].id == .LParen); |
| | 4885 | |
| | 4886 | var i: usize = 2; |
| | 4887 | while (true) : (i += 1) { |
| | 4888 | const token = ms.tokens[i]; |
| | 4889 | switch (token.id) { |
| | 4890 | .RParen => break, |
| | 4891 | .Comma => continue, |
| | 4892 | .Identifier => { |
| | 4893 | const identifier = ms.slice(token); |
| | 4894 | try hash.put(allocator, identifier, i); |
| | 4895 | }, |
| | 4896 | else => return error.UnexpectedMacroToken, |
| | 4897 | } |
| | 4898 | } |
| | 4899 | } |
| | 4900 | |
| | 4901 | const Pattern = struct { |
| | 4902 | tokens: []const CToken, |
| | 4903 | source: []const u8, |
| | 4904 | impl: []const u8, |
| | 4905 | args_hash: ArgsPositionMap, |
| | 4906 | |
| | 4907 | fn init(self: *Pattern, allocator: *mem.Allocator, template: [2][]const u8) Error!void { |
| | 4908 | const source = template[0]; |
| | 4909 | const impl = template[1]; |
| | 4910 | |
| | 4911 | var tok_list = std.ArrayList(CToken).init(allocator); |
| | 4912 | defer tok_list.deinit(); |
| | 4913 | try tokenizeMacro(source, &tok_list); |
| | 4914 | const tokens = try allocator.dupe(CToken, tok_list.items); |
| | 4915 | |
| | 4916 | self.* = .{ |
| | 4917 | .tokens = tokens, |
| | 4918 | .source = source, |
| | 4919 | .impl = impl, |
| | 4920 | .args_hash = .{}, |
| | 4921 | }; |
| | 4922 | const ms = MacroSlicer{ .source = source, .tokens = tokens }; |
| | 4923 | buildArgsHash(allocator, ms, &self.args_hash) catch |err| switch (err) { |
| | 4924 | error.UnexpectedMacroToken => unreachable, |
| | 4925 | else => |e| return e, |
| | 4926 | }; |
| | 4927 | } |
| | 4928 | |
| | 4929 | fn deinit(self: *Pattern, allocator: *mem.Allocator) void { |
| | 4930 | self.args_hash.deinit(allocator); |
| | 4931 | allocator.free(self.tokens); |
| | 4932 | } |
| | 4933 | |
| | 4934 | /// This function assumes that `ms` has already been validated to contain a function-like |
| | 4935 | /// macro, and that the parsed template macro in `self` also contains a function-like |
| | 4936 | /// macro. Please review this logic carefully if changing that assumption. Two |
| | 4937 | /// function-like macros are considered equivalent if and only if they contain the same |
| | 4938 | /// list of tokens, modulo parameter names. |
| | 4939 | fn isEquivalent(self: Pattern, ms: MacroSlicer, args_hash: ArgsPositionMap) bool { |
| | 4940 | if (self.tokens.len != ms.tokens.len) return false; |
| | 4941 | if (args_hash.count() != self.args_hash.count()) return false; |
| | 4942 | |
| | 4943 | var i: usize = 2; |
| | 4944 | while (self.tokens[i].id != .RParen) : (i += 1) {} |
| | 4945 | |
| | 4946 | const pattern_slicer = MacroSlicer{ .source = self.source, .tokens = self.tokens }; |
| | 4947 | while (i < self.tokens.len) : (i += 1) { |
| | 4948 | const pattern_token = self.tokens[i]; |
| | 4949 | const macro_token = ms.tokens[i]; |
| | 4950 | if (meta.activeTag(pattern_token.id) != meta.activeTag(macro_token.id)) return false; |
| | 4951 | |
| | 4952 | const pattern_bytes = pattern_slicer.slice(pattern_token); |
| | 4953 | const macro_bytes = ms.slice(macro_token); |
| | 4954 | switch (pattern_token.id) { |
| | 4955 | .Identifier => { |
| | 4956 | const pattern_arg_index = self.args_hash.get(pattern_bytes); |
| | 4957 | const macro_arg_index = args_hash.get(macro_bytes); |
| | 4958 | |
| | 4959 | if (pattern_arg_index == null and macro_arg_index == null) { |
| | 4960 | if (!mem.eql(u8, pattern_bytes, macro_bytes)) return false; |
| | 4961 | } else if (pattern_arg_index != null and macro_arg_index != null) { |
| | 4962 | if (pattern_arg_index.? != macro_arg_index.?) return false; |
| | 4963 | } else { |
| | 4964 | return false; |
| | 4965 | } |
| | 4966 | }, |
| | 4967 | .MacroString, .StringLiteral, .CharLiteral, .IntegerLiteral, .FloatLiteral => { |
| | 4968 | if (!mem.eql(u8, pattern_bytes, macro_bytes)) return false; |
| | 4969 | }, |
| | 4970 | else => { |
| | 4971 | // other tags correspond to keywords and operators that do not contain a "payload" |
| | 4972 | // that can vary |
| | 4973 | }, |
| | 4974 | } |
| | 4975 | } |
| | 4976 | return true; |
| | 4977 | } |
| | 4978 | }; |
| | 4979 | |
| | 4980 | fn init(allocator: *mem.Allocator) Error!PatternList { |
| | 4981 | const patterns = try allocator.alloc(Pattern, templates.len); |
| | 4982 | for (templates) |template, i| { |
| | 4983 | try patterns[i].init(allocator, template); |
| | 4984 | } |
| | 4985 | return PatternList{ .patterns = patterns }; |
| | 4986 | } |
| | 4987 | |
| | 4988 | fn deinit(self: *PatternList, allocator: *mem.Allocator) void { |
| | 4989 | for (self.patterns) |*pattern| pattern.deinit(allocator); |
| | 4990 | allocator.free(self.patterns); |
| | 4991 | } |
| | 4992 | |
| | 4993 | fn match(self: PatternList, allocator: *mem.Allocator, ms: MacroSlicer) Error!?Pattern { |
| | 4994 | var args_hash: ArgsPositionMap = .{}; |
| | 4995 | defer args_hash.deinit(allocator); |
| | 4996 | |
| | 4997 | buildArgsHash(allocator, ms, &args_hash) catch |err| switch (err) { |
| | 4998 | error.UnexpectedMacroToken => return null, |
| | 4999 | else => |e| return e, |
| | 5000 | }; |
| | 5001 | |
| | 5002 | for (self.patterns) |pattern| if (pattern.isEquivalent(ms, args_hash)) return pattern; |
| | 5003 | return null; |
| | 5004 | } |
| | 5005 | }; |
| | 5006 | |
| | 5007 | const MacroSlicer = struct { |
| | 5008 | source: []const u8, |
| | 5009 | tokens: []const CToken, |
| | 5010 | fn slice(self: MacroSlicer, token: CToken) []const u8 { |
| | 5011 | return self.source[token.start..token.end]; |
| | 5012 | } |
| | 5013 | }; |
| | 5014 | |
| | 5015 | // Testing here instead of test/translate_c.zig allows us to also test that the |
| | 5016 | // mapped function exists in `std.zig.c_translation.Macros` |
| | 5017 | test "Macro matching" { |
| | 5018 | const helper = struct { |
| | 5019 | const MacroFunctions = @import("std").zig.c_translation.Macros; |
| | 5020 | fn checkMacro(allocator: *mem.Allocator, pattern_list: PatternList, source: []const u8, comptime expected_match: ?[]const u8) !void { |
| | 5021 | var tok_list = std.ArrayList(CToken).init(allocator); |
| | 5022 | defer tok_list.deinit(); |
| | 5023 | try tokenizeMacro(source, &tok_list); |
| | 5024 | const macro_slicer = MacroSlicer{ .source = source, .tokens = tok_list.items }; |
| | 5025 | const matched = try pattern_list.match(allocator, macro_slicer); |
| | 5026 | if (expected_match) |expected| { |
| | 5027 | try testing.expectEqualStrings(expected, matched.?.impl); |
| | 5028 | try testing.expect(@hasDecl(MacroFunctions, expected)); |
| | 5029 | } else { |
| | 5030 | try testing.expectEqual(@as(@TypeOf(matched), null), matched); |
| | 5031 | } |
| | 5032 | } |
| | 5033 | }; |
| | 5034 | const allocator = std.testing.allocator; |
| | 5035 | var pattern_list = try PatternList.init(allocator); |
| | 5036 | defer pattern_list.deinit(allocator); |
| | 5037 | |
| | 5038 | try helper.checkMacro(allocator, pattern_list, "BAR(Z) (Z ## F)", "F_SUFFIX"); |
| | 5039 | try helper.checkMacro(allocator, pattern_list, "BAR(Z) (Z ## U)", "U_SUFFIX"); |
| | 5040 | try helper.checkMacro(allocator, pattern_list, "BAR(Z) (Z ## L)", "L_SUFFIX"); |
| | 5041 | try helper.checkMacro(allocator, pattern_list, "BAR(Z) (Z ## LL)", "LL_SUFFIX"); |
| | 5042 | try helper.checkMacro(allocator, pattern_list, "BAR(Z) (Z ## UL)", "UL_SUFFIX"); |
| | 5043 | try helper.checkMacro(allocator, pattern_list, "BAR(Z) (Z ## ULL)", "ULL_SUFFIX"); |
| | 5044 | try helper.checkMacro(allocator, pattern_list, |
| | 5045 | \\container_of(a, b, c) \ |
| | 5046 | \\(__typeof__(b))((char *)(a) - \ |
| | 5047 | \\ offsetof(__typeof__(*b), c)) |
| | 5048 | , "WL_CONTAINER_OF"); |
| | 5049 | |
| | 5050 | try helper.checkMacro(allocator, pattern_list, "NO_MATCH(X, Y) (X + Y)", null); |
| | 5051 | } |
| | 5052 | |
| 4832 | const MacroCtx = struct { | 5053 | const MacroCtx = struct { |
| 4833 | source: []const u8, | 5054 | source: []const u8, |
| 4834 | list: []const CToken, | 5055 | list: []const CToken, |
| ... | @@ -4855,8 +5076,30 @@ const MacroCtx = struct { | ... | @@ -4855,8 +5076,30 @@ const MacroCtx = struct { |
| 4855 | fn fail(self: *MacroCtx, c: *Context, comptime fmt: []const u8, args: anytype) !void { | 5076 | fn fail(self: *MacroCtx, c: *Context, comptime fmt: []const u8, args: anytype) !void { |
| 4856 | return failDecl(c, self.loc, self.name, fmt, args); | 5077 | return failDecl(c, self.loc, self.name, fmt, args); |
| 4857 | } | 5078 | } |
| | 5079 | |
| | 5080 | fn makeSlicer(self: *const MacroCtx) MacroSlicer { |
| | 5081 | return MacroSlicer{ .source = self.source, .tokens = self.list }; |
| | 5082 | } |
| 4858 | }; | 5083 | }; |
| 4859 | | 5084 | |
| | 5085 | fn tokenizeMacro(source: []const u8, tok_list: *std.ArrayList(CToken)) Error!void { |
| | 5086 | var tokenizer = std.c.Tokenizer{ |
| | 5087 | .buffer = source, |
| | 5088 | }; |
| | 5089 | while (true) { |
| | 5090 | const tok = tokenizer.next(); |
| | 5091 | switch (tok.id) { |
| | 5092 | .Nl, .Eof => { |
| | 5093 | try tok_list.append(tok); |
| | 5094 | break; |
| | 5095 | }, |
| | 5096 | .LineComment, .MultiLineComment => continue, |
| | 5097 | else => {}, |
| | 5098 | } |
| | 5099 | try tok_list.append(tok); |
| | 5100 | } |
| | 5101 | } |
| | 5102 | |
| 4860 | fn transPreprocessorEntities(c: *Context, unit: *clang.ASTUnit) Error!void { | 5103 | fn transPreprocessorEntities(c: *Context, unit: *clang.ASTUnit) Error!void { |
| 4861 | // TODO if we see #undef, delete it from the table | 5104 | // TODO if we see #undef, delete it from the table |
| 4862 | var it = unit.getLocalPreprocessingEntities_begin(); | 5105 | var it = unit.getLocalPreprocessingEntities_begin(); |
| ... | @@ -4888,21 +5131,7 @@ fn transPreprocessorEntities(c: *Context, unit: *clang.ASTUnit) Error!void { | ... | @@ -4888,21 +5131,7 @@ fn transPreprocessorEntities(c: *Context, unit: *clang.ASTUnit) Error!void { |
| 4888 | const slice_len = @ptrToInt(end_c) - @ptrToInt(begin_c); | 5131 | const slice_len = @ptrToInt(end_c) - @ptrToInt(begin_c); |
| 4889 | const slice = begin_c[0..slice_len]; | 5132 | const slice = begin_c[0..slice_len]; |
| 4890 | | 5133 | |
| 4891 | var tokenizer = std.c.Tokenizer{ | 5134 | try tokenizeMacro(slice, &tok_list); |
| 4892 | .buffer = slice, | | |
| 4893 | }; | | |
| 4894 | while (true) { | | |
| 4895 | const tok = tokenizer.next(); | | |
| 4896 | switch (tok.id) { | | |
| 4897 | .Nl, .Eof => { | | |
| 4898 | try tok_list.append(tok); | | |
| 4899 | break; | | |
| 4900 | }, | | |
| 4901 | .LineComment, .MultiLineComment => continue, | | |
| 4902 | else => {}, | | |
| 4903 | } | | |
| 4904 | try tok_list.append(tok); | | |
| 4905 | } | | |
| 4906 | | 5135 | |
| 4907 | var macro_ctx = MacroCtx{ | 5136 | var macro_ctx = MacroCtx{ |
| 4908 | .source = slice, | 5137 | .source = slice, |
| ... | @@ -4960,6 +5189,16 @@ fn transMacroDefine(c: *Context, m: *MacroCtx) ParseError!void { | ... | @@ -4960,6 +5189,16 @@ fn transMacroDefine(c: *Context, m: *MacroCtx) ParseError!void { |
| 4960 | } | 5189 | } |
| 4961 | | 5190 | |
| 4962 | fn transMacroFnDefine(c: *Context, m: *MacroCtx) ParseError!void { | 5191 | fn transMacroFnDefine(c: *Context, m: *MacroCtx) ParseError!void { |
| | 5192 | const macro_slicer = m.makeSlicer(); |
| | 5193 | if (try c.pattern_list.match(c.gpa, macro_slicer)) |pattern| { |
| | 5194 | const decl = try Tag.pub_var_simple.create(c.arena, .{ |
| | 5195 | .name = m.name, |
| | 5196 | .init = try Tag.helpers_macro.create(c.arena, pattern.impl), |
| | 5197 | }); |
| | 5198 | try c.global_scope.macro_table.put(m.name, decl); |
| | 5199 | return; |
| | 5200 | } |
| | 5201 | |
| 4963 | var block_scope = try Scope.Block.init(c, &c.global_scope.base, false); | 5202 | var block_scope = try Scope.Block.init(c, &c.global_scope.base, false); |
| 4964 | defer block_scope.deinit(); | 5203 | defer block_scope.deinit(); |
| 4965 | const scope = &block_scope.base; | 5204 | const scope = &block_scope.base; |