authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-21 16:21:14+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-22 12:33:53-08:00
loge60d66711185fa2e3164f5af92b9786e04c4fa19
tree58a35ac4bb210ba0cfc4f781fb977d494c2c2066
parent241e100827fffde710eb0722691eeee592854744

Module: fix `@embedFile` of files containing zero bytes

If an adapted string key with embedded nulls was put in a hash map with `std.hash_map.StringIndexAdapter`, then an incorrect hash would be entered for that entry such that it is possible that when looking for the exact key that matches the prefix of the original key up to the first null would sometimes match this entry due to hash collisions and sometimes not if performed later after a grow + rehash, causing the same key to exist with two different indices breaking every string equality comparison ever, for example claiming that a container type doesn't contain a field because the field name string in the struct and the string representing the identifier to lookup might be equal strings but have different string indices. This could maybe be fixed by changing `std.hash_map.StringIndexAdapter.hash` to only hash up to the first null, therefore ensuring that the entry's hash is correct and that all future lookups will be consistent, but I don't trust anything so instead I assert that there are no embedded nulls.

6 files changed, 28 insertions(+), 18 deletions(-)

lib/std/hash_map.zig+7-10
...@@ -92,27 +92,24 @@ pub fn hashString(s: []const u8) u64 {...@@ -92,27 +92,24 @@ pub fn hashString(s: []const u8) u64 {
92pub const StringIndexContext = struct {92pub const StringIndexContext = struct {
93 bytes: *const std.ArrayListUnmanaged(u8),93 bytes: *const std.ArrayListUnmanaged(u8),
9494
95 pub fn eql(self: @This(), a: u32, b: u32) bool {95 pub fn eql(_: @This(), a: u32, b: u32) bool {
96 _ = self;
97 return a == b;96 return a == b;
98 }97 }
9998
100 pub fn hash(self: @This(), x: u32) u64 {99 pub fn hash(ctx: @This(), key: u32) u64 {
101 const x_slice = mem.sliceTo(@as([*:0]const u8, @ptrCast(self.bytes.items.ptr)) + x, 0);100 return hashString(mem.sliceTo(ctx.bytes.items[key..], 0));
102 return hashString(x_slice);
103 }101 }
104};102};
105103
106pub const StringIndexAdapter = struct {104pub const StringIndexAdapter = struct {
107 bytes: *const std.ArrayListUnmanaged(u8),105 bytes: *const std.ArrayListUnmanaged(u8),
108106
109 pub fn eql(self: @This(), a_slice: []const u8, b: u32) bool {107 pub fn eql(ctx: @This(), a: []const u8, b: u32) bool {
110 const b_slice = mem.sliceTo(@as([*:0]const u8, @ptrCast(self.bytes.items.ptr)) + b, 0);108 return mem.eql(u8, a, mem.sliceTo(ctx.bytes.items[b..], 0));
111 return mem.eql(u8, a_slice, b_slice);
112 }109 }
113110
114 pub fn hash(self: @This(), adapted_key: []const u8) u64 {111 pub fn hash(_: @This(), adapted_key: []const u8) u64 {
115 _ = self;112 assert(mem.indexOfScalar(u8, adapted_key, 0) == null);
116 return hashString(adapted_key);113 return hashString(adapted_key);
117 }114 }
118};115};
src/AstGen.zig+6-2
...@@ -11461,6 +11461,10 @@ fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {...@@ -11461,6 +11461,10 @@ fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
11461 const token_bytes = astgen.tree.tokenSlice(str_lit_token);11461 const token_bytes = astgen.tree.tokenSlice(str_lit_token);
11462 try astgen.parseStrLit(str_lit_token, string_bytes, token_bytes, 0);11462 try astgen.parseStrLit(str_lit_token, string_bytes, token_bytes, 0);
11463 const key: []const u8 = string_bytes.items[str_index..];11463 const key: []const u8 = string_bytes.items[str_index..];
11464 if (std.mem.indexOfScalar(u8, key, 0)) |_| return .{
11465 .index = @enumFromInt(str_index),
11466 .len = @intCast(key.len),
11467 };
11464 const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{11468 const gop = try astgen.string_table.getOrPutContextAdapted(gpa, key, StringIndexAdapter{
11465 .bytes = string_bytes,11469 .bytes = string_bytes,
11466 }, StringIndexContext{11470 }, StringIndexContext{
...@@ -11468,7 +11472,7 @@ fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {...@@ -11468,7 +11472,7 @@ fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
11468 });11472 });
11469 if (gop.found_existing) {11473 if (gop.found_existing) {
11470 string_bytes.shrinkRetainingCapacity(str_index);11474 string_bytes.shrinkRetainingCapacity(str_index);
11471 return IndexSlice{11475 return .{
11472 .index = @enumFromInt(gop.key_ptr.*),11476 .index = @enumFromInt(gop.key_ptr.*),
11473 .len = @intCast(key.len),11477 .len = @intCast(key.len),
11474 };11478 };
...@@ -11478,7 +11482,7 @@ fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {...@@ -11478,7 +11482,7 @@ fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
11478 // to lookup null terminated strings, so if we get a match, it has to11482 // to lookup null terminated strings, so if we get a match, it has to
11479 // be null terminated for that to work.11483 // be null terminated for that to work.
11480 try string_bytes.append(gpa, 0);11484 try string_bytes.append(gpa, 0);
11481 return IndexSlice{11485 return .{
11482 .index = @enumFromInt(str_index),11486 .index = @enumFromInt(str_index),
11483 .len = @intCast(key.len),11487 .len = @intCast(key.len),
11484 };11488 };
src/InternPool.zig+2-1
...@@ -7985,7 +7985,8 @@ pub fn getTrailingAggregate(...@@ -7985,7 +7985,8 @@ pub fn getTrailingAggregate(
7985) Allocator.Error!Index {7985) Allocator.Error!Index {
7986 try ip.items.ensureUnusedCapacity(gpa, 1);7986 try ip.items.ensureUnusedCapacity(gpa, 1);
7987 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Bytes).Struct.fields.len);7987 try ip.extra.ensureUnusedCapacity(gpa, @typeInfo(Bytes).Struct.fields.len);
7988 const str: String = @enumFromInt(@intFromEnum(try getOrPutTrailingString(ip, gpa, len)));7988
7989 const str: String = @enumFromInt(ip.string_bytes.items.len - len);
7989 const adapter: KeyAdapter = .{ .intern_pool = ip };7990 const adapter: KeyAdapter = .{ .intern_pool = ip };
7990 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .aggregate = .{7991 const gop = try ip.map.getOrPutAdapted(gpa, Key{ .aggregate = .{
7991 .ty = ty,7992 .ty = ty,
src/Module.zig+6-5
...@@ -4400,6 +4400,7 @@ fn newEmbedFile(...@@ -4400,6 +4400,7 @@ fn newEmbedFile(
4400 src_loc: SrcLoc,4400 src_loc: SrcLoc,
4401) !InternPool.Index {4401) !InternPool.Index {
4402 const gpa = mod.gpa;4402 const gpa = mod.gpa;
4403 const ip = &mod.intern_pool;
44034404
4404 const new_file = try gpa.create(EmbedFile);4405 const new_file = try gpa.create(EmbedFile);
4405 errdefer gpa.destroy(new_file);4406 errdefer gpa.destroy(new_file);
...@@ -4414,11 +4415,11 @@ fn newEmbedFile(...@@ -4414,11 +4415,11 @@ fn newEmbedFile(
4414 .mtime = actual_stat.mtime,4415 .mtime = actual_stat.mtime,
4415 };4416 };
4416 const size = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;4417 const size = std.math.cast(usize, actual_stat.size) orelse return error.Overflow;
4417 const ip = &mod.intern_pool;
44184418
4419 const ptr = try ip.string_bytes.addManyAsSlice(gpa, size);4419 const bytes = try ip.string_bytes.addManyAsSlice(gpa, try std.math.add(usize, size, 1));
4420 const actual_read = try file.readAll(ptr);4420 const actual_read = try file.readAll(bytes[0..size]);
4421 if (actual_read != size) return error.UnexpectedEndOfFile;4421 if (actual_read != size) return error.UnexpectedEndOfFile;
4422 bytes[size] = 0;
44224423
4423 const comp = mod.comp;4424 const comp = mod.comp;
4424 switch (comp.cache_use) {4425 switch (comp.cache_use) {
...@@ -4427,7 +4428,7 @@ fn newEmbedFile(...@@ -4427,7 +4428,7 @@ fn newEmbedFile(
4427 errdefer gpa.free(copied_resolved_path);4428 errdefer gpa.free(copied_resolved_path);
4428 whole.cache_manifest_mutex.lock();4429 whole.cache_manifest_mutex.lock();
4429 defer whole.cache_manifest_mutex.unlock();4430 defer whole.cache_manifest_mutex.unlock();
4430 try man.addFilePostContents(copied_resolved_path, ptr, stat);4431 try man.addFilePostContents(copied_resolved_path, bytes[0..size], stat);
4431 },4432 },
4432 .incremental => {},4433 .incremental => {},
4433 }4434 }
...@@ -4437,7 +4438,7 @@ fn newEmbedFile(...@@ -4437,7 +4438,7 @@ fn newEmbedFile(
4437 .sentinel = .zero_u8,4438 .sentinel = .zero_u8,
4438 .child = .u8_type,4439 .child = .u8_type,
4439 } });4440 } });
4440 const array_val = try ip.getTrailingAggregate(gpa, array_ty, size);4441 const array_val = try ip.getTrailingAggregate(gpa, array_ty, bytes.len);
44414442
4442 const ptr_ty = (try mod.ptrType(.{4443 const ptr_ty = (try mod.ptrType(.{
4443 .child = array_ty,4444 .child = array_ty,
test/behavior.zig+7
...@@ -127,3 +127,10 @@ test {...@@ -127,3 +127,10 @@ test {
127 _ = @import("behavior/export_keyword.zig");127 _ = @import("behavior/export_keyword.zig");
128 }128 }
129}129}
130
131// This bug only repros in the root file
132test "deference @embedFile() of a file full of zero bytes" {
133 const contents = @embedFile("behavior/zero.bin").*;
134 try @import("std").testing.expect(contents.len == 456);
135 for (contents) |byte| try @import("std").testing.expect(byte == 0);
136}
test/behavior/zero.bin created
Binary files /dev/null and b/test/behavior/zero.bin differ