| ... | ... | @@ -10,6 +10,7 @@ const Hash = std.crypto.hash.sha2.Sha256; |
| 10 | 10 | const Compilation = @import("Compilation.zig"); |
| 11 | 11 | const Module = @import("Module.zig"); |
| 12 | 12 | const ThreadPool = @import("ThreadPool.zig"); |
| 13 | const WaitGroup = @import("WaitGroup.zig"); |
| 13 | 14 | |
| 14 | 15 | pub const Table = std.StringHashMapUnmanaged(*Package); |
| 15 | 16 | |
| ... | ... | @@ -201,7 +202,13 @@ pub fn fetchAndAddDependencies( |
| 201 | 202 | continue; |
| 202 | 203 | }; |
| 203 | 204 | |
| 204 | | const sub_pkg = try fetchAndUnpack(http_client, global_cache_directory, url, expected_hash); |
| 205 | const sub_pkg = try fetchAndUnpack( |
| 206 | thread_pool, |
| 207 | http_client, |
| 208 | global_cache_directory, |
| 209 | url, |
| 210 | expected_hash, |
| 211 | ); |
| 205 | 212 | |
| 206 | 213 | try sub_pkg.fetchAndAddDependencies( |
| 207 | 214 | thread_pool, |
| ... | ... | @@ -218,6 +225,7 @@ pub fn fetchAndAddDependencies( |
| 218 | 225 | } |
| 219 | 226 | |
| 220 | 227 | fn fetchAndUnpack( |
| 228 | thread_pool: *ThreadPool, |
| 221 | 229 | http_client: *std.http.Client, |
| 222 | 230 | global_cache_directory: Compilation.Directory, |
| 223 | 231 | url: []const u8, |
| ... | ... | @@ -225,71 +233,99 @@ fn fetchAndUnpack( |
| 225 | 233 | ) !*Package { |
| 226 | 234 | const gpa = http_client.allocator; |
| 227 | 235 | |
| 228 | | // TODO check if the expected_hash is already present in the global package cache, and |
| 229 | | // thereby avoid both fetching and unpacking. |
| 236 | // Check if the expected_hash is already present in the global package |
| 237 | // cache, and thereby avoid both fetching and unpacking. |
| 238 | const s = fs.path.sep_str; |
| 239 | if (expected_hash) |h| { |
| 240 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(h); |
| 241 | _ = pkg_dir_sub_path; |
| 242 | @panic("TODO check the p dir for the package"); |
| 243 | } |
| 230 | 244 | |
| 231 | 245 | const uri = try std.Uri.parse(url); |
| 232 | 246 | |
| 233 | | var tmp_directory: Compilation.Directory = d: { |
| 234 | | const s = fs.path.sep_str; |
| 235 | | const rand_int = std.crypto.random.int(u64); |
| 247 | const rand_int = std.crypto.random.int(u64); |
| 248 | const tmp_dir_sub_path = "tmp" ++ s ++ hex64(rand_int); |
| 236 | 249 | |
| 237 | | const tmp_dir_sub_path = try std.fmt.allocPrint(gpa, "tmp" ++ s ++ "{x}", .{rand_int}); |
| 250 | const actual_hash = a: { |
| 251 | var tmp_directory: Compilation.Directory = d: { |
| 252 | const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path}); |
| 253 | errdefer gpa.free(path); |
| 238 | 254 | |
| 239 | | const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path}); |
| 240 | | errdefer gpa.free(path); |
| 255 | const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}); |
| 256 | errdefer iterable_dir.close(); |
| 241 | 257 | |
| 242 | | const handle = try global_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); |
| 243 | | errdefer handle.close(); |
| 244 | | |
| 245 | | break :d .{ |
| 246 | | .path = path, |
| 247 | | .handle = handle, |
| 258 | break :d .{ |
| 259 | .path = path, |
| 260 | .handle = iterable_dir.dir, |
| 261 | }; |
| 248 | 262 | }; |
| 249 | | }; |
| 250 | | defer tmp_directory.closeAndFree(gpa); |
| 263 | defer tmp_directory.closeAndFree(gpa); |
| 251 | 264 | |
| 252 | | var req = try http_client.request(uri, .{}, .{}); |
| 253 | | defer req.deinit(); |
| 265 | var req = try http_client.request(uri, .{}, .{}); |
| 266 | defer req.deinit(); |
| 254 | 267 | |
| 255 | | if (mem.endsWith(u8, uri.path, ".tar.gz")) { |
| 256 | | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| 257 | | // buffered reader on the front of it. |
| 258 | | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); |
| 268 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { |
| 269 | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| 270 | // buffered reader on the front of it. |
| 271 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); |
| 259 | 272 | |
| 260 | | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); |
| 261 | | defer gzip_stream.deinit(); |
| 273 | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); |
| 274 | defer gzip_stream.deinit(); |
| 262 | 275 | |
| 263 | | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{}); |
| 264 | | } else { |
| 265 | | // TODO: show the build.zig.ini file and line number |
| 266 | | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); |
| 267 | | return error.UnknownPackageExtension; |
| 268 | | } |
| 276 | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{ |
| 277 | .strip_components = 1, |
| 278 | }); |
| 279 | } else { |
| 280 | // TODO: show the build.zig.ini file and line number |
| 281 | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); |
| 282 | return error.UnknownPackageExtension; |
| 283 | } |
| 269 | 284 | |
| 270 | | // TODO: delete files not included in the package prior to computing the package hash. |
| 271 | | // for example, if the ini file has directives to include/not include certain files, |
| 272 | | // apply those rules directly to the filesystem right here. This ensures that files |
| 273 | | // not protected by the hash are not present on the file system. |
| 285 | // TODO: delete files not included in the package prior to computing the package hash. |
| 286 | // for example, if the ini file has directives to include/not include certain files, |
| 287 | // apply those rules directly to the filesystem right here. This ensures that files |
| 288 | // not protected by the hash are not present on the file system. |
| 274 | 289 | |
| 275 | | const actual_hash = try computePackageHash(tmp_directory); |
| 290 | const actual_hash = try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); |
| 276 | 291 | |
| 277 | | if (expected_hash) |h| { |
| 278 | | if (!mem.eql(u8, &h, &actual_hash)) { |
| 279 | | // TODO: show the build.zig.ini file and line number |
| 280 | | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ |
| 281 | | url, h, actual_hash, |
| 282 | | }); |
| 283 | | return error.PackageHashMismatch; |
| 292 | if (expected_hash) |h| { |
| 293 | if (!mem.eql(u8, &h, &actual_hash)) { |
| 294 | // TODO: show the build.zig.ini file and line number |
| 295 | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ |
| 296 | url, h, actual_hash, |
| 297 | }); |
| 298 | return error.PackageHashMismatch; |
| 299 | } |
| 284 | 300 | } |
| 285 | | } |
| 286 | 301 | |
| 287 | | if (true) @panic("TODO move the tmp dir into place"); |
| 302 | break :a actual_hash; |
| 303 | }; |
| 304 | |
| 305 | { |
| 306 | // Rename the temporary directory into the global package cache. |
| 307 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); |
| 308 | var handled_missing_dir = false; |
| 309 | while (true) { |
| 310 | global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { |
| 311 | error.FileNotFound => { |
| 312 | if (handled_missing_dir) return err; |
| 313 | global_cache_directory.handle.makeDir("p") catch |mkd_err| switch (mkd_err) { |
| 314 | error.PathAlreadyExists => handled_missing_dir = true, |
| 315 | else => |e| return e, |
| 316 | }; |
| 317 | continue; |
| 318 | }, |
| 319 | else => |e| return e, |
| 320 | }; |
| 321 | break; |
| 322 | } |
| 323 | } |
| 288 | 324 | |
| 289 | 325 | if (expected_hash == null) { |
| 290 | 326 | // TODO: show the build.zig.ini file and line number |
| 291 | 327 | std.log.err("{s}: missing hash:\nhash={s}", .{ |
| 292 | | url, actual_hash, |
| 328 | url, std.fmt.fmtSliceHexLower(&actual_hash), |
| 293 | 329 | }); |
| 294 | 330 | return error.PackageDependencyMissingHash; |
| 295 | 331 | } |
| ... | ... | @@ -303,7 +339,117 @@ fn fetchAndUnpack( |
| 303 | 339 | //root_src_path: []const u8, |
| 304 | 340 | } |
| 305 | 341 | |
| 306 | | fn computePackageHash(pkg_directory: Compilation.Directory) ![Hash.digest_length]u8 { |
| 307 | | _ = pkg_directory; |
| 308 | | @panic("TODO computePackageHash"); |
| 342 | const HashedFile = struct { |
| 343 | path: []const u8, |
| 344 | hash: [Hash.digest_length]u8, |
| 345 | failure: Error!void, |
| 346 | |
| 347 | const Error = fs.File.OpenError || fs.File.ReadError; |
| 348 | |
| 349 | fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool { |
| 350 | _ = context; |
| 351 | return mem.lessThan(u8, lhs.path, rhs.path); |
| 352 | } |
| 353 | }; |
| 354 | |
| 355 | fn computePackageHash( |
| 356 | thread_pool: *ThreadPool, |
| 357 | pkg_dir: fs.IterableDir, |
| 358 | ) ![Hash.digest_length]u8 { |
| 359 | const gpa = thread_pool.allocator; |
| 360 | |
| 361 | // We'll use an arena allocator for the path name strings since they all |
| 362 | // need to be in memory for sorting. |
| 363 | var arena_instance = std.heap.ArenaAllocator.init(gpa); |
| 364 | defer arena_instance.deinit(); |
| 365 | const arena = arena_instance.allocator(); |
| 366 | |
| 367 | // Collect all files, recursively, then sort. |
| 368 | var all_files = std.ArrayList(*HashedFile).init(gpa); |
| 369 | defer all_files.deinit(); |
| 370 | |
| 371 | var walker = try pkg_dir.walk(gpa); |
| 372 | defer walker.deinit(); |
| 373 | |
| 374 | { |
| 375 | // The final hash will be a hash of each file hashed independently. This |
| 376 | // allows hashing in parallel. |
| 377 | var wait_group: WaitGroup = .{}; |
| 378 | defer wait_group.wait(); |
| 379 | |
| 380 | while (try walker.next()) |entry| { |
| 381 | switch (entry.kind) { |
| 382 | .Directory => continue, |
| 383 | .File => {}, |
| 384 | else => return error.IllegalFileTypeInPackage, |
| 385 | } |
| 386 | const hashed_file = try arena.create(HashedFile); |
| 387 | hashed_file.* = .{ |
| 388 | .path = try arena.dupe(u8, entry.path), |
| 389 | .hash = undefined, // to be populated by the worker |
| 390 | .failure = undefined, // to be populated by the worker |
| 391 | }; |
| 392 | |
| 393 | wait_group.start(); |
| 394 | try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan); |
| 399 | |
| 400 | var hasher = Hash.init(.{}); |
| 401 | var any_failures = false; |
| 402 | for (all_files.items) |hashed_file| { |
| 403 | hashed_file.failure catch |err| { |
| 404 | any_failures = true; |
| 405 | std.log.err("unable to hash '{s}': {s}", .{ hashed_file.path, @errorName(err) }); |
| 406 | }; |
| 407 | hasher.update(&hashed_file.hash); |
| 408 | } |
| 409 | if (any_failures) return error.PackageHashUnavailable; |
| 410 | return hasher.finalResult(); |
| 411 | } |
| 412 | |
| 413 | fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void { |
| 414 | defer wg.finish(); |
| 415 | hashed_file.failure = hashFileFallible(dir, hashed_file); |
| 416 | } |
| 417 | |
| 418 | fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { |
| 419 | var buf: [8000]u8 = undefined; |
| 420 | var file = try dir.openFile(hashed_file.path, .{}); |
| 421 | var hasher = Hash.init(.{}); |
| 422 | while (true) { |
| 423 | const bytes_read = try file.read(&buf); |
| 424 | if (bytes_read == 0) break; |
| 425 | hasher.update(buf[0..bytes_read]); |
| 426 | } |
| 427 | hasher.final(&hashed_file.hash); |
| 428 | } |
| 429 | |
| 430 | const hex_charset = "0123456789abcdef"; |
| 431 | |
| 432 | fn hex64(x: u64) [16]u8 { |
| 433 | var result: [16]u8 = undefined; |
| 434 | var i: usize = 0; |
| 435 | while (i < 8) : (i += 1) { |
| 436 | const byte = @truncate(u8, x >> @intCast(u6, 8 * i)); |
| 437 | result[i * 2 + 0] = hex_charset[byte >> 4]; |
| 438 | result[i * 2 + 1] = hex_charset[byte & 15]; |
| 439 | } |
| 440 | return result; |
| 441 | } |
| 442 | |
| 443 | test hex64 { |
| 444 | const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; |
| 445 | try std.testing.expectEqualStrings("[00efcdab78563412]", s); |
| 446 | } |
| 447 | |
| 448 | fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 { |
| 449 | var result: [Hash.digest_length * 2]u8 = undefined; |
| 450 | for (digest) |byte, i| { |
| 451 | result[i * 2 + 0] = hex_charset[byte >> 4]; |
| 452 | result[i * 2 + 1] = hex_charset[byte & 15]; |
| 453 | } |
| 454 | return result; |
| 309 | 455 | } |
Andrew Kelley <andrew@ziglang.org> 2023-01-10 00:38:36-07:00
src