| ... | @@ -10,6 +10,7 @@ const Hash = std.crypto.hash.sha2.Sha256; | ... | @@ -10,6 +10,7 @@ const Hash = std.crypto.hash.sha2.Sha256; |
| 10 | const Compilation = @import("Compilation.zig"); | 10 | const Compilation = @import("Compilation.zig"); |
| 11 | const Module = @import("Module.zig"); | 11 | const Module = @import("Module.zig"); |
| 12 | const ThreadPool = @import("ThreadPool.zig"); | 12 | const ThreadPool = @import("ThreadPool.zig"); |
| | 13 | const WaitGroup = @import("WaitGroup.zig"); |
| 13 | | 14 | |
| 14 | pub const Table = std.StringHashMapUnmanaged(*Package); | 15 | pub const Table = std.StringHashMapUnmanaged(*Package); |
| 15 | | 16 | |
| ... | @@ -201,7 +202,13 @@ pub fn fetchAndAddDependencies( | ... | @@ -201,7 +202,13 @@ pub fn fetchAndAddDependencies( |
| 201 | continue; | 202 | continue; |
| 202 | }; | 203 | }; |
| 203 | | 204 | |
| 204 | const sub_pkg = try fetchAndUnpack(http_client, global_cache_directory, url, expected_hash); | 205 | const sub_pkg = try fetchAndUnpack( |
| | 206 | thread_pool, |
| | 207 | http_client, |
| | 208 | global_cache_directory, |
| | 209 | url, |
| | 210 | expected_hash, |
| | 211 | ); |
| 205 | | 212 | |
| 206 | try sub_pkg.fetchAndAddDependencies( | 213 | try sub_pkg.fetchAndAddDependencies( |
| 207 | thread_pool, | 214 | thread_pool, |
| ... | @@ -218,6 +225,7 @@ pub fn fetchAndAddDependencies( | ... | @@ -218,6 +225,7 @@ pub fn fetchAndAddDependencies( |
| 218 | } | 225 | } |
| 219 | | 226 | |
| 220 | fn fetchAndUnpack( | 227 | fn fetchAndUnpack( |
| | 228 | thread_pool: *ThreadPool, |
| 221 | http_client: *std.http.Client, | 229 | http_client: *std.http.Client, |
| 222 | global_cache_directory: Compilation.Directory, | 230 | global_cache_directory: Compilation.Directory, |
| 223 | url: []const u8, | 231 | url: []const u8, |
| ... | @@ -225,71 +233,99 @@ fn fetchAndUnpack( | ... | @@ -225,71 +233,99 @@ fn fetchAndUnpack( |
| 225 | ) !*Package { | 233 | ) !*Package { |
| 226 | const gpa = http_client.allocator; | 234 | const gpa = http_client.allocator; |
| 227 | | 235 | |
| 228 | // TODO check if the expected_hash is already present in the global package cache, and | 236 | // Check if the expected_hash is already present in the global package |
| 229 | // thereby avoid both fetching and unpacking. | 237 | // cache, and thereby avoid both fetching and unpacking. |
| | 238 | const s = fs.path.sep_str; |
| | 239 | if (expected_hash) |h| { |
| | 240 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(h); |
| | 241 | _ = pkg_dir_sub_path; |
| | 242 | @panic("TODO check the p dir for the package"); |
| | 243 | } |
| 230 | | 244 | |
| 231 | const uri = try std.Uri.parse(url); | 245 | const uri = try std.Uri.parse(url); |
| 232 | | 246 | |
| 233 | var tmp_directory: Compilation.Directory = d: { | 247 | const rand_int = std.crypto.random.int(u64); |
| 234 | const s = fs.path.sep_str; | 248 | const tmp_dir_sub_path = "tmp" ++ s ++ hex64(rand_int); |
| 235 | const rand_int = std.crypto.random.int(u64); | | |
| 236 | | 249 | |
| 237 | const tmp_dir_sub_path = try std.fmt.allocPrint(gpa, "tmp" ++ s ++ "{x}", .{rand_int}); | 250 | const actual_hash = a: { |
| | 251 | var tmp_directory: Compilation.Directory = d: { |
| | 252 | const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path}); |
| | 253 | errdefer gpa.free(path); |
| 238 | | 254 | |
| 239 | const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path}); | 255 | const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}); |
| 240 | errdefer gpa.free(path); | 256 | errdefer iterable_dir.close(); |
| 241 | | 257 | |
| 242 | const handle = try global_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); | 258 | break :d .{ |
| 243 | errdefer handle.close(); | 259 | .path = path, |
| 244 | | 260 | .handle = iterable_dir.dir, |
| 245 | break :d .{ | 261 | }; |
| 246 | .path = path, | | |
| 247 | .handle = handle, | | |
| 248 | }; | 262 | }; |
| 249 | }; | 263 | defer tmp_directory.closeAndFree(gpa); |
| 250 | defer tmp_directory.closeAndFree(gpa); | | |
| 251 | | 264 | |
| 252 | var req = try http_client.request(uri, .{}, .{}); | 265 | var req = try http_client.request(uri, .{}, .{}); |
| 253 | defer req.deinit(); | 266 | defer req.deinit(); |
| 254 | | 267 | |
| 255 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { | 268 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { |
| 256 | // I observed the gzip stream to read 1 byte at a time, so I am using a | 269 | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| 257 | // buffered reader on the front of it. | 270 | // buffered reader on the front of it. |
| 258 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); | 271 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); |
| 259 | | 272 | |
| 260 | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); | 273 | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); |
| 261 | defer gzip_stream.deinit(); | 274 | defer gzip_stream.deinit(); |
| 262 | | 275 | |
| 263 | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{}); | 276 | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{ |
| 264 | } else { | 277 | .strip_components = 1, |
| 265 | // TODO: show the build.zig.ini file and line number | 278 | }); |
| 266 | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); | 279 | } else { |
| 267 | return error.UnknownPackageExtension; | 280 | // TODO: show the build.zig.ini file and line number |
| 268 | } | 281 | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); |
| | 282 | return error.UnknownPackageExtension; |
| | 283 | } |
| 269 | | 284 | |
| 270 | // TODO: delete files not included in the package prior to computing the package hash. | 285 | // TODO: delete files not included in the package prior to computing the package hash. |
| 271 | // for example, if the ini file has directives to include/not include certain files, | 286 | // for example, if the ini file has directives to include/not include certain files, |
| 272 | // apply those rules directly to the filesystem right here. This ensures that files | 287 | // apply those rules directly to the filesystem right here. This ensures that files |
| 273 | // not protected by the hash are not present on the file system. | 288 | // not protected by the hash are not present on the file system. |
| 274 | | 289 | |
| 275 | const actual_hash = try computePackageHash(tmp_directory); | 290 | const actual_hash = try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); |
| 276 | | 291 | |
| 277 | if (expected_hash) |h| { | 292 | if (expected_hash) |h| { |
| 278 | if (!mem.eql(u8, &h, &actual_hash)) { | 293 | if (!mem.eql(u8, &h, &actual_hash)) { |
| 279 | // TODO: show the build.zig.ini file and line number | 294 | // TODO: show the build.zig.ini file and line number |
| 280 | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ | 295 | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ |
| 281 | url, h, actual_hash, | 296 | url, h, actual_hash, |
| 282 | }); | 297 | }); |
| 283 | return error.PackageHashMismatch; | 298 | return error.PackageHashMismatch; |
| | 299 | } |
| 284 | } | 300 | } |
| 285 | } | | |
| 286 | | 301 | |
| 287 | if (true) @panic("TODO move the tmp dir into place"); | 302 | break :a actual_hash; |
| | 303 | }; |
| | 304 | |
| | 305 | { |
| | 306 | // Rename the temporary directory into the global package cache. |
| | 307 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); |
| | 308 | var handled_missing_dir = false; |
| | 309 | while (true) { |
| | 310 | global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { |
| | 311 | error.FileNotFound => { |
| | 312 | if (handled_missing_dir) return err; |
| | 313 | global_cache_directory.handle.makeDir("p") catch |mkd_err| switch (mkd_err) { |
| | 314 | error.PathAlreadyExists => handled_missing_dir = true, |
| | 315 | else => |e| return e, |
| | 316 | }; |
| | 317 | continue; |
| | 318 | }, |
| | 319 | else => |e| return e, |
| | 320 | }; |
| | 321 | break; |
| | 322 | } |
| | 323 | } |
| 288 | | 324 | |
| 289 | if (expected_hash == null) { | 325 | if (expected_hash == null) { |
| 290 | // TODO: show the build.zig.ini file and line number | 326 | // TODO: show the build.zig.ini file and line number |
| 291 | std.log.err("{s}: missing hash:\nhash={s}", .{ | 327 | std.log.err("{s}: missing hash:\nhash={s}", .{ |
| 292 | url, actual_hash, | 328 | url, std.fmt.fmtSliceHexLower(&actual_hash), |
| 293 | }); | 329 | }); |
| 294 | return error.PackageDependencyMissingHash; | 330 | return error.PackageDependencyMissingHash; |
| 295 | } | 331 | } |
| ... | @@ -303,7 +339,117 @@ fn fetchAndUnpack( | ... | @@ -303,7 +339,117 @@ fn fetchAndUnpack( |
| 303 | //root_src_path: []const u8, | 339 | //root_src_path: []const u8, |
| 304 | } | 340 | } |
| 305 | | 341 | |
| 306 | fn computePackageHash(pkg_directory: Compilation.Directory) ![Hash.digest_length]u8 { | 342 | const HashedFile = struct { |
| 307 | _ = pkg_directory; | 343 | path: []const u8, |
| 308 | @panic("TODO computePackageHash"); | 344 | hash: [Hash.digest_length]u8, |
| | 345 | failure: Error!void, |
| | 346 | |
| | 347 | const Error = fs.File.OpenError || fs.File.ReadError; |
| | 348 | |
| | 349 | fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { |
| | 350 | _ = context; |
| | 351 | return mem.lessThan(u8, lhs.path, rhs.path); |
| | 352 | } |
| | 353 | }; |
| | 354 | |
| | 355 | fn computePackageHash( |
| | 356 | thread_pool: *ThreadPool, |
| | 357 | pkg_dir: fs.IterableDir, |
| | 358 | ) ![Hash.digest_length]u8 { |
| | 359 | const gpa = thread_pool.allocator; |
| | 360 | |
| | 361 | // We'll use an arena allocator for the path name strings since they all |
| | 362 | // need to be in memory for sorting. |
| | 363 | var arena_instance = std.heap.ArenaAllocator.init(gpa); |
| | 364 | defer arena_instance.deinit(); |
| | 365 | const arena = arena_instance.allocator(); |
| | 366 | |
| | 367 | // Collect all files, recursively, then sort. |
| | 368 | var all_files = std.ArrayList(*HashedFile).init(gpa); |
| | 369 | defer all_files.deinit(); |
| | 370 | |
| | 371 | var walker = try pkg_dir.walk(gpa); |
| | 372 | defer walker.deinit(); |
| | 373 | |
| | 374 | { |
| | 375 | // The final hash will be a hash of each file hashed independently. This |
| | 376 | // allows hashing in parallel. |
| | 377 | var wait_group: WaitGroup = .{}; |
| | 378 | defer wait_group.wait(); |
| | 379 | |
| | 380 | while (try walker.next()) |entry| { |
| | 381 | switch (entry.kind) { |
| | 382 | .Directory => continue, |
| | 383 | .File => {}, |
| | 384 | else => return error.IllegalFileTypeInPackage, |
| | 385 | } |
| | 386 | const hashed_file = try arena.create(HashedFile); |
| | 387 | hashed_file.* = .{ |
| | 388 | .path = try arena.dupe(u8, entry.path), |
| | 389 | .hash = undefined, // to be populated by the worker |
| | 390 | .failure = undefined, // to be populated by the worker |
| | 391 | }; |
| | 392 | |
| | 393 | wait_group.start(); |
| | 394 | try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); |
| | 395 | } |
| | 396 | } |
| | 397 | |
| | 398 | std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan); |
| | 399 | |
| | 400 | var hasher = Hash.init(.{}); |
| | 401 | var any_failures = false; |
| | 402 | for (all_files.items) |hashed_file| { |
| | 403 | hashed_file.failure catch |err| { |
| | 404 | any_failures = true; |
| | 405 | std.log.err("unable to hash '{s}': {s}", .{ hashed_file.path, @errorName(err) }); |
| | 406 | }; |
| | 407 | hasher.update(&hashed_file.hash); |
| | 408 | } |
| | 409 | if (any_failures) return error.PackageHashUnavailable; |
| | 410 | return hasher.finalResult(); |
| | 411 | } |
| | 412 | |
| | 413 | fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { |
| | 414 | defer wg.finish(); |
| | 415 | hashed_file.failure = hashFileFallible(dir, hashed_file); |
| | 416 | } |
| | 417 | |
| | 418 | fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { |
| | 419 | var buf: [8000]u8 = undefined; |
| | 420 | var file = try dir.openFile(hashed_file.path, .{}); |
| | 421 | var hasher = Hash.init(.{}); |
| | 422 | while (true) { |
| | 423 | const bytes_read = try file.read(&buf); |
| | 424 | if (bytes_read == 0) break; |
| | 425 | hasher.update(buf[0..bytes_read]); |
| | 426 | } |
| | 427 | hasher.final(&hashed_file.hash); |
| | 428 | } |
| | 429 | |
| | 430 | const hex_charset = "0123456789abcdef"; |
| | 431 | |
| | 432 | fn hex64(x: u64) [16]u8 { |
| | 433 | var result: [16]u8 = undefined; |
| | 434 | var i: usize = 0; |
| | 435 | while (i < 8) : (i += 1) { |
| | 436 | const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); |
| | 437 | result[i * 2 + 0] = hex_charset[byte >> 4]; |
| | 438 | result[i * 2 + 1] = hex_charset[byte & 15]; |
| | 439 | } |
| | 440 | return result; |
| | 441 | } |
| | 442 | |
| | 443 | test hex64 { |
| | 444 | const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; |
| | 445 | try std.testing.expectEqualStrings("[00efcdab78563412]", s); |
| | 446 | } |
| | 447 | |
| | 448 | fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 { |
| | 449 | var result: [Hash.digest_length * 2]u8 = undefined; |
| | 450 | for (digest) |byte, i| { |
| | 451 | result[i * 2 + 0] = hex_charset[byte >> 4]; |
| | 452 | result[i * 2 + 1] = hex_charset[byte & 15]; |
| | 453 | } |
| | 454 | return result; |
| 309 | } | 455 | } |
Andrew Kelley <andrew@ziglang.org> 2023-01-10 00:38:36-07:00
src