| ... | @@ -6,6 +6,7 @@ const mem = std.mem; | ... | @@ -6,6 +6,7 @@ const mem = std.mem; |
| 6 | const Allocator = mem.Allocator; | 6 | const Allocator = mem.Allocator; |
| 7 | const assert = std.debug.assert; | 7 | const assert = std.debug.assert; |
| 8 | const Hash = std.crypto.hash.sha2.Sha256; | 8 | const Hash = std.crypto.hash.sha2.Sha256; |
| | 9 | const log = std.log.scoped(.package); |
| 9 | | 10 | |
| 10 | const Compilation = @import("Compilation.zig"); | 11 | const Compilation = @import("Compilation.zig"); |
| 11 | const Module = @import("Module.zig"); | 12 | const Module = @import("Module.zig"); |
| ... | @@ -128,6 +129,9 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P | ... | @@ -128,6 +129,9 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P |
| 128 | return parent.add(gpa, name, child); | 129 | return parent.add(gpa, name, child); |
| 129 | } | 130 | } |
| 130 | | 131 | |
| | 132 | pub const build_zig_basename = "build.zig"; |
| | 133 | pub const ini_basename = build_zig_basename ++ ".ini"; |
| | 134 | |
| 131 | pub fn fetchAndAddDependencies( | 135 | pub fn fetchAndAddDependencies( |
| 132 | pkg: *Package, | 136 | pkg: *Package, |
| 133 | thread_pool: *ThreadPool, | 137 | thread_pool: *ThreadPool, |
| ... | @@ -138,7 +142,7 @@ pub fn fetchAndAddDependencies( | ... | @@ -138,7 +142,7 @@ pub fn fetchAndAddDependencies( |
| 138 | ) !void { | 142 | ) !void { |
| 139 | const max_bytes = 10 * 1024 * 1024; | 143 | const max_bytes = 10 * 1024 * 1024; |
| 140 | const gpa = thread_pool.allocator; | 144 | const gpa = thread_pool.allocator; |
| 141 | const build_zig_ini = directory.handle.readFileAlloc(gpa, "build.zig.ini", max_bytes) catch |err| switch (err) { | 145 | const build_zig_ini = directory.handle.readFileAlloc(gpa, ini_basename, max_bytes) catch |err| switch (err) { |
| 142 | error.FileNotFound => { | 146 | error.FileNotFound => { |
| 143 | // Handle the same as no dependencies. | 147 | // Handle the same as no dependencies. |
| 144 | return; | 148 | return; |
| ... | @@ -154,7 +158,7 @@ pub fn fetchAndAddDependencies( | ... | @@ -154,7 +158,7 @@ pub fn fetchAndAddDependencies( |
| 154 | var line_it = mem.split(u8, dep, "\n"); | 158 | var line_it = mem.split(u8, dep, "\n"); |
| 155 | var opt_id: ?[]const u8 = null; | 159 | var opt_id: ?[]const u8 = null; |
| 156 | var opt_url: ?[]const u8 = null; | 160 | var opt_url: ?[]const u8 = null; |
| 157 | var expected_hash: ?[Hash.digest_length]u8 = null; | 161 | var expected_hash: ?[]const u8 = null; |
| 158 | while (line_it.next()) |kv| { | 162 | while (line_it.next()) |kv| { |
| 159 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; | 163 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; |
| 160 | const key = kv[0..eq_pos]; | 164 | const key = kv[0..eq_pos]; |
| ... | @@ -164,8 +168,7 @@ pub fn fetchAndAddDependencies( | ... | @@ -164,8 +168,7 @@ pub fn fetchAndAddDependencies( |
| 164 | } else if (mem.eql(u8, key, "url")) { | 168 | } else if (mem.eql(u8, key, "url")) { |
| 165 | opt_url = value; | 169 | opt_url = value; |
| 166 | } else if (mem.eql(u8, key, "hash")) { | 170 | } else if (mem.eql(u8, key, "hash")) { |
| 167 | @panic("TODO parse hex digits of value into expected_hash"); | 171 | expected_hash = value; |
| 168 | //expected_hash = value; | | |
| 169 | } else { | 172 | } else { |
| 170 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); | 173 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 171 | std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ | 174 | std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ |
| ... | @@ -208,6 +211,8 @@ pub fn fetchAndAddDependencies( | ... | @@ -208,6 +211,8 @@ pub fn fetchAndAddDependencies( |
| 208 | global_cache_directory, | 211 | global_cache_directory, |
| 209 | url, | 212 | url, |
| 210 | expected_hash, | 213 | expected_hash, |
| | 214 | ini, |
| | 215 | directory, |
| 211 | ); | 216 | ); |
| 212 | | 217 | |
| 213 | try sub_pkg.fetchAndAddDependencies( | 218 | try sub_pkg.fetchAndAddDependencies( |
| ... | @@ -229,17 +234,48 @@ fn fetchAndUnpack( | ... | @@ -229,17 +234,48 @@ fn fetchAndUnpack( |
| 229 | http_client: *std.http.Client, | 234 | http_client: *std.http.Client, |
| 230 | global_cache_directory: Compilation.Directory, | 235 | global_cache_directory: Compilation.Directory, |
| 231 | url: []const u8, | 236 | url: []const u8, |
| 232 | expected_hash: ?[Hash.digest_length]u8, | 237 | expected_hash: ?[]const u8, |
| | 238 | ini: std.Ini, |
| | 239 | comp_directory: Compilation.Directory, |
| 233 | ) !*Package { | 240 | ) !*Package { |
| 234 | const gpa = http_client.allocator; | 241 | const gpa = http_client.allocator; |
| | 242 | const s = fs.path.sep_str; |
| 235 | | 243 | |
| 236 | // Check if the expected_hash is already present in the global package | 244 | // Check if the expected_hash is already present in the global package |
| 237 | // cache, and thereby avoid both fetching and unpacking. | 245 | // cache, and thereby avoid both fetching and unpacking. |
| 238 | const s = fs.path.sep_str; | 246 | if (expected_hash) |h| cached: { |
| 239 | if (expected_hash) |h| { | 247 | if (h.len != 2 * Hash.digest_length) { |
| 240 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(h); | 248 | return reportError( |
| 241 | _ = pkg_dir_sub_path; | 249 | ini, |
| 242 | @panic("TODO check the p dir for the package"); | 250 | comp_directory, |
| | 251 | h.ptr, |
| | 252 | "wrong hash size. expected: {d}, found: {d}", |
| | 253 | .{ Hash.digest_length, h.len }, |
| | 254 | ); |
| | 255 | } |
| | 256 | const hex_digest = h[0 .. 2 * Hash.digest_length]; |
| | 257 | const pkg_dir_sub_path = "p" ++ s ++ hex_digest; |
| | 258 | var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) { |
| | 259 | error.FileNotFound => break :cached, |
| | 260 | else => |e| return e, |
| | 261 | }; |
| | 262 | errdefer pkg_dir.close(); |
| | 263 | |
| | 264 | const ptr = try gpa.create(Package); |
| | 265 | errdefer gpa.destroy(ptr); |
| | 266 | |
| | 267 | const owned_src_path = try gpa.dupe(u8, build_zig_basename); |
| | 268 | errdefer gpa.free(owned_src_path); |
| | 269 | |
| | 270 | ptr.* = .{ |
| | 271 | .root_src_directory = .{ |
| | 272 | .path = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}), |
| | 273 | .handle = pkg_dir, |
| | 274 | }, |
| | 275 | .root_src_directory_owned = true, |
| | 276 | .root_src_path = owned_src_path, |
| | 277 | }; |
| | 278 | return ptr; |
| 243 | } | 279 | } |
| 244 | | 280 | |
| 245 | const uri = try std.Uri.parse(url); | 281 | const uri = try std.Uri.parse(url); |
| ... | @@ -277,9 +313,13 @@ fn fetchAndUnpack( | ... | @@ -277,9 +313,13 @@ fn fetchAndUnpack( |
| 277 | .strip_components = 1, | 313 | .strip_components = 1, |
| 278 | }); | 314 | }); |
| 279 | } else { | 315 | } else { |
| 280 | // TODO: show the build.zig.ini file and line number | 316 | return reportError( |
| 281 | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); | 317 | ini, |
| 282 | return error.UnknownPackageExtension; | 318 | comp_directory, |
| | 319 | uri.path.ptr, |
| | 320 | "unknown file extension for path '{s}'", |
| | 321 | .{uri.path}, |
| | 322 | ); |
| 283 | } | 323 | } |
| 284 | | 324 | |
| 285 | // TODO: delete files not included in the package prior to computing the package hash. | 325 | // TODO: delete files not included in the package prior to computing the package hash. |
| ... | @@ -287,24 +327,13 @@ fn fetchAndUnpack( | ... | @@ -287,24 +327,13 @@ fn fetchAndUnpack( |
| 287 | // apply those rules directly to the filesystem right here. This ensures that files | 327 | // apply those rules directly to the filesystem right here. This ensures that files |
| 288 | // not protected by the hash are not present on the file system. | 328 | // not protected by the hash are not present on the file system. |
| 289 | | 329 | |
| 290 | const actual_hash = try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); | 330 | break :a try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle }); |
| 291 | | | |
| 292 | if (expected_hash) |h| { | | |
| 293 | if (!mem.eql(u8, &h, &actual_hash)) { | | |
| 294 | // TODO: show the build.zig.ini file and line number | | |
| 295 | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ | | |
| 296 | url, h, actual_hash, | | |
| 297 | }); | | |
| 298 | return error.PackageHashMismatch; | | |
| 299 | } | | |
| 300 | } | | |
| 301 | | | |
| 302 | break :a actual_hash; | | |
| 303 | }; | 331 | }; |
| 304 | | 332 | |
| | 333 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); |
| | 334 | |
| 305 | { | 335 | { |
| 306 | // Rename the temporary directory into the global package cache. | 336 | // Rename the temporary directory into the global package cache. |
| 307 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); | | |
| 308 | var handled_missing_dir = false; | 337 | var handled_missing_dir = false; |
| 309 | while (true) { | 338 | while (true) { |
| 310 | global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { | 339 | global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { |
| ... | @@ -316,27 +345,60 @@ fn fetchAndUnpack( | ... | @@ -316,27 +345,60 @@ fn fetchAndUnpack( |
| 316 | }; | 345 | }; |
| 317 | continue; | 346 | continue; |
| 318 | }, | 347 | }, |
| | 348 | error.PathAlreadyExists => { |
| | 349 | // Package has been already downloaded and may already be in use on the system. |
| | 350 | global_cache_directory.handle.deleteTree(tmp_dir_sub_path) catch |del_err| { |
| | 351 | std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); |
| | 352 | }; |
| | 353 | }, |
| 319 | else => |e| return e, | 354 | else => |e| return e, |
| 320 | }; | 355 | }; |
| 321 | break; | 356 | break; |
| 322 | } | 357 | } |
| 323 | } | 358 | } |
| 324 | | 359 | |
| 325 | if (expected_hash == null) { | 360 | if (expected_hash) |h| { |
| 326 | // TODO: show the build.zig.ini file and line number | 361 | const actual_hex = hexDigest(actual_hash); |
| 327 | std.log.err("{s}: missing hash:\nhash={s}", .{ | 362 | if (!mem.eql(u8, h, &actual_hex)) { |
| 328 | url, std.fmt.fmtSliceHexLower(&actual_hash), | 363 | return reportError( |
| 329 | }); | 364 | ini, |
| 330 | return error.PackageDependencyMissingHash; | 365 | comp_directory, |
| | 366 | h.ptr, |
| | 367 | "hash mismatch: expected: {s}, found: {s}", |
| | 368 | .{ h, actual_hex }, |
| | 369 | ); |
| | 370 | } |
| | 371 | } else { |
| | 372 | return reportError( |
| | 373 | ini, |
| | 374 | comp_directory, |
| | 375 | url.ptr, |
| | 376 | "url field is missing corresponding hash field: hash={s}", |
| | 377 | .{std.fmt.fmtSliceHexLower(&actual_hash)}, |
| | 378 | ); |
| 331 | } | 379 | } |
| 332 | | 380 | |
| 333 | @panic("TODO create package and set root_src_directory"); | 381 | return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename); |
| 334 | //return create(gpa, root_src | 382 | } |
| 335 | //gpa: Allocator, | 383 | |
| 336 | ///// Null indicates the current working directory | 384 | fn reportError( |
| 337 | //root_src_dir_path: ?[]const u8, | 385 | ini: std.Ini, |
| 338 | ///// Relative to root_src_dir_path | 386 | comp_directory: Compilation.Directory, |
| 339 | //root_src_path: []const u8, | 387 | src_ptr: [*]const u8, |
| | 388 | comptime fmt_string: []const u8, |
| | 389 | fmt_args: anytype, |
| | 390 | ) error{PackageFetchFailed} { |
| | 391 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(src_ptr) - @ptrToInt(ini.bytes.ptr)); |
| | 392 | if (comp_directory.path) |p| { |
| | 393 | std.debug.print("{s}{c}{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ |
| | 394 | p, fs.path.sep, ini_basename, loc.line + 1, loc.column + 1, |
| | 395 | } ++ fmt_args); |
| | 396 | } else { |
| | 397 | std.debug.print("{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{ |
| | 398 | ini_basename, loc.line + 1, loc.column + 1, |
| | 399 | } ++ fmt_args); |
| | 400 | } |
| | 401 | return error.PackageFetchFailed; |
| 340 | } | 402 | } |
| 341 | | 403 | |
| 342 | const HashedFile = struct { | 404 | const HashedFile = struct { |
| ... | @@ -389,9 +451,10 @@ fn computePackageHash( | ... | @@ -389,9 +451,10 @@ fn computePackageHash( |
| 389 | .hash = undefined, // to be populated by the worker | 451 | .hash = undefined, // to be populated by the worker |
| 390 | .failure = undefined, // to be populated by the worker | 452 | .failure = undefined, // to be populated by the worker |
| 391 | }; | 453 | }; |
| 392 | | | |
| 393 | wait_group.start(); | 454 | wait_group.start(); |
| 394 | try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); | 455 | try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); |
| | 456 | |
| | 457 | try all_files.append(hashed_file); |
| 395 | } | 458 | } |
| 396 | } | 459 | } |
| 397 | | 460 | |
Andrew Kelley <andrew@ziglang.org> 2023-01-10 16:26:27-07:00