authorgravatar for evan@lagerdata.comEvan Haas <evan@lagerdata.com> 2022-09-08 20:19:10-07:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2022-09-10 01:50:17+03:00
log8e631ee3e7b4e7b4466c0efafaffb4151447785f
tree1873bad537882c07d240b8ceed55e0221f81705c
parent9e070b653c89a9216f9dd9f78ed7c78c11460ac7

translate-c: Escape non-ASCII characters that appear in macros

Macro definitions are simply a slice of bytes, which may not be UTF-8 encoded. If they are not UTF-8 encoded, escape non-printable and non-ASCII characters as `\xNN`. Fixes #12784

3 files changed, 35 insertions(+), 2 deletions(-)

src/translate_c.zig+18-2
...@@ -5957,20 +5957,36 @@ fn zigifyEscapeSequences(ctx: *Context, m: *MacroCtx) ![]const u8 {...@@ -5957,20 +5957,36 @@ fn zigifyEscapeSequences(ctx: *Context, m: *MacroCtx) ![]const u8 {
5957 return bytes[0..i];5957 return bytes[0..i];
5958}5958}
59595959
5960/// non-ASCII characters (c > 127) are also treated as non-printable by fmtSliceEscapeLower.
5961/// If a C string literal or char literal in a macro is not valid UTF-8, we need to escape
5962/// non-ASCII characters so that the Zig source we output will itself be UTF-8.
5963fn escapeUnprintables(ctx: *Context, m: *MacroCtx) ![]const u8 {
5964 const zigified = try zigifyEscapeSequences(ctx, m);
5965 if (std.unicode.utf8ValidateSlice(zigified)) return zigified;
5966
5967 const formatter = std.fmt.fmtSliceEscapeLower(zigified);
5968 const encoded_size = @intCast(usize, std.fmt.count("{s}", .{formatter}));
5969 var output = try ctx.arena.alloc(u8, encoded_size);
5970 return std.fmt.bufPrint(output, "{s}", .{formatter}) catch |err| switch (err) {
5971 error.NoSpaceLeft => unreachable,
5972 else => |e| return e,
5973 };
5974}
5975
5960fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!Node {5976fn parseCPrimaryExprInner(c: *Context, m: *MacroCtx, scope: *Scope) ParseError!Node {
5961 const tok = m.next().?;5977 const tok = m.next().?;
5962 const slice = m.slice();5978 const slice = m.slice();
5963 switch (tok) {5979 switch (tok) {
5964 .CharLiteral => {5980 .CharLiteral => {
5965 if (slice[0] != '\'' or slice[1] == '\\' or slice.len == 3) {5981 if (slice[0] != '\'' or slice[1] == '\\' or slice.len == 3) {
5966 return Tag.char_literal.create(c.arena, try zigifyEscapeSequences(c, m));5982 return Tag.char_literal.create(c.arena, try escapeUnprintables(c, m));
5967 } else {5983 } else {
5968 const str = try std.fmt.allocPrint(c.arena, "0x{s}", .{std.fmt.fmtSliceHexLower(slice[1 .. slice.len - 1])});5984 const str = try std.fmt.allocPrint(c.arena, "0x{s}", .{std.fmt.fmtSliceHexLower(slice[1 .. slice.len - 1])});
5969 return Tag.integer_literal.create(c.arena, str);5985 return Tag.integer_literal.create(c.arena, str);
5970 }5986 }
5971 },5987 },
5972 .StringLiteral => {5988 .StringLiteral => {
5973 return Tag.string_literal.create(c.arena, try zigifyEscapeSequences(c, m));5989 return Tag.string_literal.create(c.arena, try escapeUnprintables(c, m));
5974 },5990 },
5975 .IntegerLiteral, .FloatLiteral => {5991 .IntegerLiteral, .FloatLiteral => {
5976 return parseCNumLit(c, m);5992 return parseCNumLit(c, m);
test/behavior/translate_c_macros.zig+12
...@@ -5,6 +5,7 @@ const expectEqual = std.testing.expectEqual;...@@ -5,6 +5,7 @@ const expectEqual = std.testing.expectEqual;
5const expectEqualStrings = std.testing.expectEqualStrings;5const expectEqualStrings = std.testing.expectEqualStrings;
66
7const h = @cImport(@cInclude("behavior/translate_c_macros.h"));7const h = @cImport(@cInclude("behavior/translate_c_macros.h"));
8const latin1 = @cImport(@cInclude("behavior/translate_c_macros_not_utf8.h"));
89
9test "casting to void with a macro" {10test "casting to void with a macro" {
10 h.IGNORE_ME_1(42);11 h.IGNORE_ME_1(42);
...@@ -134,3 +135,14 @@ test "string literal macro with embedded tab character" {...@@ -134,3 +135,14 @@ test "string literal macro with embedded tab character" {
134135
135 try expectEqualStrings("hello\t", h.EMBEDDED_TAB);136 try expectEqualStrings("hello\t", h.EMBEDDED_TAB);
136}137}
138
139test "string and char literals that are not UTF-8 encoded. Issue #12784" {
140 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
141 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO
142 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO
143 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
144 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
145
146 try expectEqual(@as(u8, '\xA9'), latin1.UNPRINTABLE_CHAR);
147 try expectEqualStrings("\xA9\xA9\xA9", latin1.UNPRINTABLE_STRING);
148}
test/behavior/translate_c_macros_not_utf8.h created+5
...@@ -0,0 +1,5 @@
1// Note: This file is encoded with ISO/IEC 8859-1 (latin1), not UTF-8.
2// Do not change the encoding
3
4#define UNPRINTABLE_STRING "���"
5#define UNPRINTABLE_CHAR '�'