| ... | @@ -5,9 +5,11 @@ const fs = std.fs; | ... | @@ -5,9 +5,11 @@ const fs = std.fs; |
| 5 | const mem = std.mem; | 5 | const mem = std.mem; |
| 6 | const Allocator = mem.Allocator; | 6 | const Allocator = mem.Allocator; |
| 7 | const assert = std.debug.assert; | 7 | const assert = std.debug.assert; |
| | 8 | const Hash = std.crypto.hash.sha2.Sha256; |
| 8 | | 9 | |
| 9 | const Compilation = @import("Compilation.zig"); | 10 | const Compilation = @import("Compilation.zig"); |
| 10 | const Module = @import("Module.zig"); | 11 | const Module = @import("Module.zig"); |
| | 12 | const ThreadPool = @import("ThreadPool.zig"); |
| 11 | | 13 | |
| 12 | pub const Table = std.StringHashMapUnmanaged(*Package); | 14 | pub const Table = std.StringHashMapUnmanaged(*Package); |
| 13 | | 15 | |
| ... | @@ -124,3 +126,184 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P | ... | @@ -124,3 +126,184 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P |
| 124 | child.parent = parent; | 126 | child.parent = parent; |
| 125 | return parent.add(gpa, name, child); | 127 | return parent.add(gpa, name, child); |
| 126 | } | 128 | } |
| | 129 | |
| | 130 | pub fn fetchAndAddDependencies( |
| | 131 | pkg: *Package, |
| | 132 | thread_pool: *ThreadPool, |
| | 133 | http_client: *std.http.Client, |
| | 134 | directory: Compilation.Directory, |
| | 135 | global_cache_directory: Compilation.Directory, |
| | 136 | local_cache_directory: Compilation.Directory, |
| | 137 | ) !void { |
| | 138 | const max_bytes = 10 * 1024 * 1024; |
| | 139 | const gpa = thread_pool.allocator; |
| | 140 | const build_zig_ini = directory.handle.readFileAlloc(gpa, "build.zig.ini", max_bytes) catch |err| switch (err) { |
| | 141 | error.FileNotFound => { |
| | 142 | // Handle the same as no dependencies. |
| | 143 | return; |
| | 144 | }, |
| | 145 | else => |e| return e, |
| | 146 | }; |
| | 147 | defer gpa.free(build_zig_ini); |
| | 148 | |
| | 149 | const ini: std.Ini = .{ .bytes = build_zig_ini }; |
| | 150 | var any_error = false; |
| | 151 | var it = ini.iterateSection("\n[dependency]\n"); |
| | 152 | while (it.next()) |dep| { |
| | 153 | var line_it = mem.split(u8, dep, "\n"); |
| | 154 | var opt_id: ?[]const u8 = null; |
| | 155 | var opt_url: ?[]const u8 = null; |
| | 156 | var expected_hash: ?[Hash.digest_length]u8 = null; |
| | 157 | while (line_it.next()) |kv| { |
| | 158 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; |
| | 159 | const key = kv[0..eq_pos]; |
| | 160 | const value = kv[eq_pos + 1 ..]; |
| | 161 | if (mem.eql(u8, key, "id")) { |
| | 162 | opt_id = value; |
| | 163 | } else if (mem.eql(u8, key, "url")) { |
| | 164 | opt_url = value; |
| | 165 | } else if (mem.eql(u8, key, "hash")) { |
| | 166 | @panic("TODO parse hex digits of value into expected_hash"); |
| | 167 | //expected_hash = value; |
| | 168 | } else { |
| | 169 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); |
| | 170 | std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ |
| | 171 | directory.path orelse ".", |
| | 172 | "build.zig.ini", |
| | 173 | loc.line, |
| | 174 | loc.column, |
| | 175 | key, |
| | 176 | }); |
| | 177 | } |
| | 178 | } |
| | 179 | |
| | 180 | const id = opt_id orelse { |
| | 181 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); |
| | 182 | std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ |
| | 183 | directory.path orelse ".", |
| | 184 | "build.zig.ini", |
| | 185 | loc.line, |
| | 186 | loc.column, |
| | 187 | }); |
| | 188 | any_error = true; |
| | 189 | continue; |
| | 190 | }; |
| | 191 | |
| | 192 | const url = opt_url orelse { |
| | 193 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); |
| | 194 | std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ |
| | 195 | directory.path orelse ".", |
| | 196 | "build.zig.ini", |
| | 197 | loc.line, |
| | 198 | loc.column, |
| | 199 | }); |
| | 200 | any_error = true; |
| | 201 | continue; |
| | 202 | }; |
| | 203 | |
| | 204 | const sub_pkg = try fetchAndUnpack(http_client, global_cache_directory, url, expected_hash); |
| | 205 | |
| | 206 | try sub_pkg.fetchAndAddDependencies( |
| | 207 | thread_pool, |
| | 208 | http_client, |
| | 209 | sub_pkg.root_src_directory, |
| | 210 | global_cache_directory, |
| | 211 | local_cache_directory, |
| | 212 | ); |
| | 213 | |
| | 214 | try addAndAdopt(pkg, gpa, id, sub_pkg); |
| | 215 | } |
| | 216 | |
| | 217 | if (any_error) return error.InvalidBuildZigIniFile; |
| | 218 | } |
| | 219 | |
| | 220 | fn fetchAndUnpack( |
| | 221 | http_client: *std.http.Client, |
| | 222 | global_cache_directory: Compilation.Directory, |
| | 223 | url: []const u8, |
| | 224 | expected_hash: ?[Hash.digest_length]u8, |
| | 225 | ) !*Package { |
| | 226 | const gpa = http_client.allocator; |
| | 227 | |
| | 228 | // TODO check if the expected_hash is already present in the global package cache, and |
| | 229 | // thereby avoid both fetching and unpacking. |
| | 230 | |
| | 231 | const uri = try std.Uri.parse(url); |
| | 232 | |
| | 233 | var tmp_directory: Compilation.Directory = d: { |
| | 234 | const s = fs.path.sep_str; |
| | 235 | const rand_int = std.crypto.random.int(u64); |
| | 236 | |
| | 237 | const tmp_dir_sub_path = try std.fmt.allocPrint(gpa, "tmp" ++ s ++ "{x}", .{rand_int}); |
| | 238 | |
| | 239 | const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path}); |
| | 240 | errdefer gpa.free(path); |
| | 241 | |
| | 242 | const handle = try global_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); |
| | 243 | errdefer handle.close(); |
| | 244 | |
| | 245 | break :d .{ |
| | 246 | .path = path, |
| | 247 | .handle = handle, |
| | 248 | }; |
| | 249 | }; |
| | 250 | defer tmp_directory.closeAndFree(gpa); |
| | 251 | |
| | 252 | var req = try http_client.request(uri, .{}, .{}); |
| | 253 | defer req.deinit(); |
| | 254 | |
| | 255 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { |
| | 256 | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| | 257 | // buffered reader on the front of it. |
| | 258 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); |
| | 259 | |
| | 260 | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); |
| | 261 | defer gzip_stream.deinit(); |
| | 262 | |
| | 263 | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{}); |
| | 264 | } else { |
| | 265 | // TODO: show the build.zig.ini file and line number |
| | 266 | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); |
| | 267 | return error.UnknownPackageExtension; |
| | 268 | } |
| | 269 | |
| | 270 | // TODO: delete files not included in the package prior to computing the package hash. |
| | 271 | // for example, if the ini file has directives to include/not include certain files, |
| | 272 | // apply those rules directly to the filesystem right here. This ensures that files |
| | 273 | // not protected by the hash are not present on the file system. |
| | 274 | |
| | 275 | const actual_hash = try computePackageHash(tmp_directory); |
| | 276 | |
| | 277 | if (expected_hash) |h| { |
| | 278 | if (!mem.eql(u8, &h, &actual_hash)) { |
| | 279 | // TODO: show the build.zig.ini file and line number |
| | 280 | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ |
| | 281 | url, h, actual_hash, |
| | 282 | }); |
| | 283 | return error.PackageHashMismatch; |
| | 284 | } |
| | 285 | } |
| | 286 | |
| | 287 | if (true) @panic("TODO move the tmp dir into place"); |
| | 288 | |
| | 289 | if (expected_hash == null) { |
| | 290 | // TODO: show the build.zig.ini file and line number |
| | 291 | std.log.err("{s}: missing hash:\nhash={s}", .{ |
| | 292 | url, actual_hash, |
| | 293 | }); |
| | 294 | return error.PackageDependencyMissingHash; |
| | 295 | } |
| | 296 | |
| | 297 | @panic("TODO create package and set root_src_directory"); |
| | 298 | //return create(gpa, root_src |
| | 299 | //gpa: Allocator, |
| | 300 | ///// Null indicates the current working directory |
| | 301 | //root_src_dir_path: ?[]const u8, |
| | 302 | ///// Relative to root_src_dir_path |
| | 303 | //root_src_path: []const u8, |
| | 304 | } |
| | 305 | |
| | 306 | fn computePackageHash(pkg_directory: Compilation.Directory) ![Hash.digest_length]u8 { |
| | 307 | _ = pkg_directory; |
| | 308 | @panic("TODO computePackageHash"); |
| | 309 | } |
Andrew Kelley <andrew@ziglang.org> 2023-01-09 22:37:17-07:00