| ... | ... | @@ -5,9 +5,11 @@ const fs = std.fs; |
| 5 | 5 | const mem = std.mem; |
| 6 | 6 | const Allocator = mem.Allocator; |
| 7 | 7 | const assert = std.debug.assert; |
| 8 | const Hash = std.crypto.hash.sha2.Sha256; |
| 8 | 9 | |
| 9 | 10 | const Compilation = @import("Compilation.zig"); |
| 10 | 11 | const Module = @import("Module.zig"); |
| 12 | const ThreadPool = @import("ThreadPool.zig"); |
| 11 | 13 | |
| 12 | 14 | pub const Table = std.StringHashMapUnmanaged(*Package); |
| 13 | 15 | |
| ... | ... | @@ -124,3 +126,184 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P |
| 124 | 126 | child.parent = parent; |
| 125 | 127 | return parent.add(gpa, name, child); |
| 126 | 128 | } |
| 129 | |
| 130 | pub fn fetchAndAddDependencies( |
| 131 | pkg: *Package, |
| 132 | thread_pool: *ThreadPool, |
| 133 | http_client: *std.http.Client, |
| 134 | directory: Compilation.Directory, |
| 135 | global_cache_directory: Compilation.Directory, |
| 136 | local_cache_directory: Compilation.Directory, |
| 137 | ) !void { |
| 138 | const max_bytes = 10 * 1024 * 1024; |
| 139 | const gpa = thread_pool.allocator; |
| 140 | const build_zig_ini = directory.handle.readFileAlloc(gpa, "build.zig.ini", max_bytes) catch |err| switch (err) { |
| 141 | error.FileNotFound => { |
| 142 | // Handle the same as no dependencies. |
| 143 | return; |
| 144 | }, |
| 145 | else => |e| return e, |
| 146 | }; |
| 147 | defer gpa.free(build_zig_ini); |
| 148 | |
| 149 | const ini: std.Ini = .{ .bytes = build_zig_ini }; |
| 150 | var any_error = false; |
| 151 | var it = ini.iterateSection("\n[dependency]\n"); |
| 152 | while (it.next()) |dep| { |
| 153 | var line_it = mem.split(u8, dep, "\n"); |
| 154 | var opt_id: ?[]const u8 = null; |
| 155 | var opt_url: ?[]const u8 = null; |
| 156 | var expected_hash: ?[Hash.digest_length]u8 = null; |
| 157 | while (line_it.next()) |kv| { |
| 158 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; |
| 159 | const key = kv[0..eq_pos]; |
| 160 | const value = kv[eq_pos + 1 ..]; |
| 161 | if (mem.eql(u8, key, "id")) { |
| 162 | opt_id = value; |
| 163 | } else if (mem.eql(u8, key, "url")) { |
| 164 | opt_url = value; |
| 165 | } else if (mem.eql(u8, key, "hash")) { |
| 166 | @panic("TODO parse hex digits of value into expected_hash"); |
| 167 | //expected_hash = value; |
| 168 | } else { |
| 169 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 170 | std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{ |
| 171 | directory.path orelse ".", |
| 172 | "build.zig.ini", |
| 173 | loc.line, |
| 174 | loc.column, |
| 175 | key, |
| 176 | }); |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | const id = opt_id orelse { |
| 181 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 182 | std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ |
| 183 | directory.path orelse ".", |
| 184 | "build.zig.ini", |
| 185 | loc.line, |
| 186 | loc.column, |
| 187 | }); |
| 188 | any_error = true; |
| 189 | continue; |
| 190 | }; |
| 191 | |
| 192 | const url = opt_url orelse { |
| 193 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 194 | std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ |
| 195 | directory.path orelse ".", |
| 196 | "build.zig.ini", |
| 197 | loc.line, |
| 198 | loc.column, |
| 199 | }); |
| 200 | any_error = true; |
| 201 | continue; |
| 202 | }; |
| 203 | |
| 204 | const sub_pkg = try fetchAndUnpack(http_client, global_cache_directory, url, expected_hash); |
| 205 | |
| 206 | try sub_pkg.fetchAndAddDependencies( |
| 207 | thread_pool, |
| 208 | http_client, |
| 209 | sub_pkg.root_src_directory, |
| 210 | global_cache_directory, |
| 211 | local_cache_directory, |
| 212 | ); |
| 213 | |
| 214 | try addAndAdopt(pkg, gpa, id, sub_pkg); |
| 215 | } |
| 216 | |
| 217 | if (any_error) return error.InvalidBuildZigIniFile; |
| 218 | } |
| 219 | |
| 220 | fn fetchAndUnpack( |
| 221 | http_client: *std.http.Client, |
| 222 | global_cache_directory: Compilation.Directory, |
| 223 | url: []const u8, |
| 224 | expected_hash: ?[Hash.digest_length]u8, |
| 225 | ) !*Package { |
| 226 | const gpa = http_client.allocator; |
| 227 | |
| 228 | // TODO check if the expected_hash is already present in the global package cache, and |
| 229 | // thereby avoid both fetching and unpacking. |
| 230 | |
| 231 | const uri = try std.Uri.parse(url); |
| 232 | |
| 233 | var tmp_directory: Compilation.Directory = d: { |
| 234 | const s = fs.path.sep_str; |
| 235 | const rand_int = std.crypto.random.int(u64); |
| 236 | |
| 237 | const tmp_dir_sub_path = try std.fmt.allocPrint(gpa, "tmp" ++ s ++ "{x}", .{rand_int}); |
| 238 | |
| 239 | const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path}); |
| 240 | errdefer gpa.free(path); |
| 241 | |
| 242 | const handle = try global_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); |
| 243 | errdefer handle.close(); |
| 244 | |
| 245 | break :d .{ |
| 246 | .path = path, |
| 247 | .handle = handle, |
| 248 | }; |
| 249 | }; |
| 250 | defer tmp_directory.closeAndFree(gpa); |
| 251 | |
| 252 | var req = try http_client.request(uri, .{}, .{}); |
| 253 | defer req.deinit(); |
| 254 | |
| 255 | if (mem.endsWith(u8, uri.path, ".tar.gz")) { |
| 256 | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| 257 | // buffered reader on the front of it. |
| 258 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); |
| 259 | |
| 260 | var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader()); |
| 261 | defer gzip_stream.deinit(); |
| 262 | |
| 263 | try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{}); |
| 264 | } else { |
| 265 | // TODO: show the build.zig.ini file and line number |
| 266 | std.log.err("{s}: unknown package extension for path '{s}'", .{ url, uri.path }); |
| 267 | return error.UnknownPackageExtension; |
| 268 | } |
| 269 | |
| 270 | // TODO: delete files not included in the package prior to computing the package hash. |
| 271 | // for example, if the ini file has directives to include/not include certain files, |
| 272 | // apply those rules directly to the filesystem right here. This ensures that files |
| 273 | // not protected by the hash are not present on the file system. |
| 274 | |
| 275 | const actual_hash = try computePackageHash(tmp_directory); |
| 276 | |
| 277 | if (expected_hash) |h| { |
| 278 | if (!mem.eql(u8, &h, &actual_hash)) { |
| 279 | // TODO: show the build.zig.ini file and line number |
| 280 | std.log.err("{s}: hash mismatch: expected: {s}, actual: {s}", .{ |
| 281 | url, h, actual_hash, |
| 282 | }); |
| 283 | return error.PackageHashMismatch; |
| 284 | } |
| 285 | } |
| 286 | |
| 287 | if (true) @panic("TODO move the tmp dir into place"); |
| 288 | |
| 289 | if (expected_hash == null) { |
| 290 | // TODO: show the build.zig.ini file and line number |
| 291 | std.log.err("{s}: missing hash:\nhash={s}", .{ |
| 292 | url, actual_hash, |
| 293 | }); |
| 294 | return error.PackageDependencyMissingHash; |
| 295 | } |
| 296 | |
| 297 | @panic("TODO create package and set root_src_directory"); |
| 298 | //return create(gpa, root_src |
| 299 | //gpa: Allocator, |
| 300 | ///// Null indicates the current working directory |
| 301 | //root_src_dir_path: ?[]const u8, |
| 302 | ///// Relative to root_src_dir_path |
| 303 | //root_src_path: []const u8, |
| 304 | } |
| 305 | |
| 306 | fn computePackageHash(pkg_directory: Compilation.Directory) ![Hash.digest_length]u8 { |
| 307 | _ = pkg_directory; |
| 308 | @panic("TODO computePackageHash"); |
| 309 | } |
Andrew Kelley <andrew@ziglang.org> 2023-01-09 22:37:17-07:00