authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-01 18:15:08-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-02 17:02:24-07:00
loga4352982b3ad4370543e0d4486347b58a958ed6b
treee3e0e9fe8340c08b75d7b221917f26329aaad291
parenta5144d19b7a3585122dafbe05f7a1ce21f61a992

compiler: extract package hashing logic to separate file

There are no functional changes in this commit.

3 files changed, 133 insertions(+), 127 deletions(-)

CMakeLists.txt+1
...@@ -527,6 +527,7 @@ set(ZIG_STAGE2_SOURCES...@@ -527,6 +527,7 @@ set(ZIG_STAGE2_SOURCES
527 "${CMAKE_SOURCE_DIR}/src/Liveness.zig"527 "${CMAKE_SOURCE_DIR}/src/Liveness.zig"
528 "${CMAKE_SOURCE_DIR}/src/Module.zig"528 "${CMAKE_SOURCE_DIR}/src/Module.zig"
529 "${CMAKE_SOURCE_DIR}/src/Package.zig"529 "${CMAKE_SOURCE_DIR}/src/Package.zig"
530 "${CMAKE_SOURCE_DIR}/src/Package/hash.zig"
530 "${CMAKE_SOURCE_DIR}/src/RangeSet.zig"531 "${CMAKE_SOURCE_DIR}/src/RangeSet.zig"
531 "${CMAKE_SOURCE_DIR}/src/Sema.zig"532 "${CMAKE_SOURCE_DIR}/src/Sema.zig"
532 "${CMAKE_SOURCE_DIR}/src/TypedValue.zig"533 "${CMAKE_SOURCE_DIR}/src/TypedValue.zig"
src/Package.zig+1-127
...@@ -10,7 +10,6 @@ const assert = std.debug.assert;...@@ -10,7 +10,6 @@ const assert = std.debug.assert;
10const log = std.log.scoped(.package);10const log = std.log.scoped(.package);
11const main = @import("main.zig");11const main = @import("main.zig");
12const ThreadPool = std.Thread.Pool;12const ThreadPool = std.Thread.Pool;
13const WaitGroup = std.Thread.WaitGroup;
1413
15const Compilation = @import("Compilation.zig");14const Compilation = @import("Compilation.zig");
16const Module = @import("Module.zig");15const Module = @import("Module.zig");
...@@ -18,6 +17,7 @@ const Cache = std.Build.Cache;...@@ -18,6 +17,7 @@ const Cache = std.Build.Cache;
18const build_options = @import("build_options");17const build_options = @import("build_options");
19const Manifest = @import("Manifest.zig");18const Manifest = @import("Manifest.zig");
20const git = @import("git.zig");19const git = @import("git.zig");
20const computePackageHash = @import("Package/hash.zig").compute;
2121
22pub const Table = std.StringHashMapUnmanaged(*Package);22pub const Table = std.StringHashMapUnmanaged(*Package);
2323
...@@ -1147,81 +1147,6 @@ fn unpackGitPack(...@@ -1147,81 +1147,6 @@ fn unpackGitPack(
1147 try out_dir.deleteTree(".git");1147 try out_dir.deleteTree(".git");
1148}1148}
11491149
1150const HashedFile = struct {
1151 fs_path: []const u8,
1152 normalized_path: []const u8,
1153 hash: [Manifest.Hash.digest_length]u8,
1154 failure: Error!void,
1155
1156 const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
1157
1158 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
1159 _ = context;
1160 return mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
1161 }
1162};
1163
1164fn computePackageHash(
1165 thread_pool: *ThreadPool,
1166 pkg_dir: fs.IterableDir,
1167) ![Manifest.Hash.digest_length]u8 {
1168 const gpa = thread_pool.allocator;
1169
1170 // We'll use an arena allocator for the path name strings since they all
1171 // need to be in memory for sorting.
1172 var arena_instance = std.heap.ArenaAllocator.init(gpa);
1173 defer arena_instance.deinit();
1174 const arena = arena_instance.allocator();
1175
1176 // Collect all files, recursively, then sort.
1177 var all_files = std.ArrayList(*HashedFile).init(gpa);
1178 defer all_files.deinit();
1179
1180 var walker = try pkg_dir.walk(gpa);
1181 defer walker.deinit();
1182
1183 {
1184 // The final hash will be a hash of each file hashed independently. This
1185 // allows hashing in parallel.
1186 var wait_group: WaitGroup = .{};
1187 defer wait_group.wait();
1188
1189 while (try walker.next()) |entry| {
1190 switch (entry.kind) {
1191 .directory => continue,
1192 .file => {},
1193 else => return error.IllegalFileTypeInPackage,
1194 }
1195 const hashed_file = try arena.create(HashedFile);
1196 const fs_path = try arena.dupe(u8, entry.path);
1197 hashed_file.* = .{
1198 .fs_path = fs_path,
1199 .normalized_path = try normalizePath(arena, fs_path),
1200 .hash = undefined, // to be populated by the worker
1201 .failure = undefined, // to be populated by the worker
1202 };
1203 wait_group.start();
1204 try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
1205
1206 try all_files.append(hashed_file);
1207 }
1208 }
1209
1210 mem.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1211
1212 var hasher = Manifest.Hash.init(.{});
1213 var any_failures = false;
1214 for (all_files.items) |hashed_file| {
1215 hashed_file.failure catch |err| {
1216 any_failures = true;
1217 std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
1218 };
1219 hasher.update(&hashed_file.hash);
1220 }
1221 if (any_failures) return error.PackageHashUnavailable;
1222 return hasher.finalResult();
1223}
1224
1225/// Compute the hash of a file path.1150/// Compute the hash of a file path.
1226fn computePathHash(gpa: Allocator, dir: Compilation.Directory, path: []const u8) ![Manifest.Hash.digest_length]u8 {1151fn computePathHash(gpa: Allocator, dir: Compilation.Directory, path: []const u8) ![Manifest.Hash.digest_length]u8 {
1227 const resolved_path = try std.fs.path.resolve(gpa, &.{ dir.path.?, path });1152 const resolved_path = try std.fs.path.resolve(gpa, &.{ dir.path.?, path });
...@@ -1240,57 +1165,6 @@ fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {...@@ -1240,57 +1165,6 @@ fn isDirectory(root_dir: Compilation.Directory, path: []const u8) !bool {
1240 return true;1165 return true;
1241}1166}
12421167
1243/// Make a file system path identical independently of operating system path inconsistencies.
1244/// This converts backslashes into forward slashes.
1245fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
1246 const canonical_sep = '/';
1247
1248 if (fs.path.sep == canonical_sep)
1249 return fs_path;
1250
1251 const normalized = try arena.dupe(u8, fs_path);
1252 for (normalized) |*byte| {
1253 switch (byte.*) {
1254 fs.path.sep => byte.* = canonical_sep,
1255 else => continue,
1256 }
1257 }
1258 return normalized;
1259}
1260
1261fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
1262 defer wg.finish();
1263 hashed_file.failure = hashFileFallible(dir, hashed_file);
1264}
1265
1266fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1267 var buf: [8000]u8 = undefined;
1268 var file = try dir.openFile(hashed_file.fs_path, .{});
1269 defer file.close();
1270 var hasher = Manifest.Hash.init(.{});
1271 hasher.update(hashed_file.normalized_path);
1272 hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
1273 while (true) {
1274 const bytes_read = try file.read(&buf);
1275 if (bytes_read == 0) break;
1276 hasher.update(buf[0..bytes_read]);
1277 }
1278 hasher.final(&hashed_file.hash);
1279}
1280
1281fn isExecutable(file: fs.File) !bool {
1282 if (builtin.os.tag == .windows) {
1283 // TODO check the ACL on Windows.
1284 // Until this is implemented, this could be a false negative on
1285 // Windows, which is why we do not yet set executable_bit_only above
1286 // when unpacking the tarball.
1287 return false;
1288 } else {
1289 const stat = try file.stat();
1290 return (stat.mode & std.os.S.IXUSR) != 0;
1291 }
1292}
1293
1294fn renameTmpIntoCache(1168fn renameTmpIntoCache(
1295 cache_dir: fs.Dir,1169 cache_dir: fs.Dir,
1296 tmp_dir_sub_path: []const u8,1170 tmp_dir_sub_path: []const u8,
src/Package/hash.zig created+131
...@@ -0,0 +1,131 @@
1const builtin = @import("builtin");
2const std = @import("std");
3const fs = std.fs;
4const ThreadPool = std.Thread.Pool;
5const WaitGroup = std.Thread.WaitGroup;
6const Allocator = std.mem.Allocator;
7
8const Hash = @import("../Manifest.zig").Hash;
9
10pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_length]u8 {
11 const gpa = thread_pool.allocator;
12
13 // We'll use an arena allocator for the path name strings since they all
14 // need to be in memory for sorting.
15 var arena_instance = std.heap.ArenaAllocator.init(gpa);
16 defer arena_instance.deinit();
17 const arena = arena_instance.allocator();
18
19 // Collect all files, recursively, then sort.
20 var all_files = std.ArrayList(*HashedFile).init(gpa);
21 defer all_files.deinit();
22
23 var walker = try pkg_dir.walk(gpa);
24 defer walker.deinit();
25
26 {
27 // The final hash will be a hash of each file hashed independently. This
28 // allows hashing in parallel.
29 var wait_group: WaitGroup = .{};
30 defer wait_group.wait();
31
32 while (try walker.next()) |entry| {
33 switch (entry.kind) {
34 .directory => continue,
35 .file => {},
36 else => return error.IllegalFileTypeInPackage,
37 }
38 const hashed_file = try arena.create(HashedFile);
39 const fs_path = try arena.dupe(u8, entry.path);
40 hashed_file.* = .{
41 .fs_path = fs_path,
42 .normalized_path = try normalizePath(arena, fs_path),
43 .hash = undefined, // to be populated by the worker
44 .failure = undefined, // to be populated by the worker
45 };
46 wait_group.start();
47 try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
48
49 try all_files.append(hashed_file);
50 }
51 }
52
53 std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
54
55 var hasher = Hash.init(.{});
56 var any_failures = false;
57 for (all_files.items) |hashed_file| {
58 hashed_file.failure catch |err| {
59 any_failures = true;
60 std.log.err("unable to hash '{s}': {s}", .{ hashed_file.fs_path, @errorName(err) });
61 };
62 hasher.update(&hashed_file.hash);
63 }
64 if (any_failures) return error.PackageHashUnavailable;
65 return hasher.finalResult();
66}
67
68const HashedFile = struct {
69 fs_path: []const u8,
70 normalized_path: []const u8,
71 hash: [Hash.digest_length]u8,
72 failure: Error!void,
73
74 const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
75
76 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
77 _ = context;
78 return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
79 }
80};
81
82/// Make a file system path identical independently of operating system path inconsistencies.
83/// This converts backslashes into forward slashes.
84fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
85 const canonical_sep = '/';
86
87 if (fs.path.sep == canonical_sep)
88 return fs_path;
89
90 const normalized = try arena.dupe(u8, fs_path);
91 for (normalized) |*byte| {
92 switch (byte.*) {
93 fs.path.sep => byte.* = canonical_sep,
94 else => continue,
95 }
96 }
97 return normalized;
98}
99
100fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
101 defer wg.finish();
102 hashed_file.failure = hashFileFallible(dir, hashed_file);
103}
104
105fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
106 var buf: [8000]u8 = undefined;
107 var file = try dir.openFile(hashed_file.fs_path, .{});
108 defer file.close();
109 var hasher = Hash.init(.{});
110 hasher.update(hashed_file.normalized_path);
111 hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
112 while (true) {
113 const bytes_read = try file.read(&buf);
114 if (bytes_read == 0) break;
115 hasher.update(buf[0..bytes_read]);
116 }
117 hasher.final(&hashed_file.hash);
118}
119
120fn isExecutable(file: fs.File) !bool {
121 if (builtin.os.tag == .windows) {
122 // TODO check the ACL on Windows.
123 // Until this is implemented, this could be a false negative on
124 // Windows, which is why we do not yet set executable_bit_only above
125 // when unpacking the tarball.
126 return false;
127 } else {
128 const stat = try file.stat();
129 return (stat.mode & std.os.S.IXUSR) != 0;
130 }
131}