| ... | @@ -0,0 +1,1421 @@ |
| 1 | //! Git support for package fetching. |
| 2 | //! |
| 3 | //! This is not intended to support all features of Git: it is limited to the |
| 4 | //! basic functionality needed to clone a repository for the purpose of fetching |
| 5 | //! a package. |
| 6 | |
| 7 | const std = @import("std"); |
| 8 | const mem = std.mem; |
| 9 | const testing = std.testing; |
| 10 | const Allocator = mem.Allocator; |
| 11 | const Sha1 = std.crypto.hash.Sha1; |
| 12 | const assert = std.debug.assert; |
| 13 | |
| 14 | const ProgressReader = @import("Package.zig").ProgressReader; |
| 15 | |
| 16 | pub const oid_length = Sha1.digest_length; |
| 17 | pub const fmt_oid_length = 2 * oid_length; |
| 18 | /// The ID of a Git object (an SHA-1 hash). |
| 19 | pub const Oid = [oid_length]u8; |
| 20 | |
| 21 | pub fn parseOid(s: []const u8) !Oid { |
| 22 | if (s.len != fmt_oid_length) return error.InvalidOid; |
| 23 | var oid: Oid = undefined; |
| 24 | for (&oid, 0..) |*b, i| { |
| 25 | b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; |
| 26 | } |
| 27 | return oid; |
| 28 | } |
| 29 | |
| 30 | test parseOid { |
| 31 | try testing.expectEqualSlices( |
| 32 | u8, |
| 33 | &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, |
| 34 | &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"), |
| 35 | ); |
| 36 | try testing.expectError(error.InvalidOid, parseOid("ce919ccf")); |
| 37 | try testing.expectError(error.InvalidOid, parseOid("master")); |
| 38 | try testing.expectError(error.InvalidOid, parseOid("HEAD")); |
| 39 | } |
| 40 | |
| 41 | pub const Repository = struct { |
| 42 | odb: Odb, |
| 43 | |
| 44 | pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository { |
| 45 | return .{ .odb = try Odb.init(allocator, pack_file, index_file) }; |
| 46 | } |
| 47 | |
| 48 | pub fn deinit(repository: *Repository) void { |
| 49 | repository.odb.deinit(); |
| 50 | repository.* = undefined; |
| 51 | } |
| 52 | |
| 53 | /// Checks out the repository at `commit_oid` to `worktree`. |
| 54 | pub fn checkout( |
| 55 | repository: *Repository, |
| 56 | worktree: std.fs.Dir, |
| 57 | commit_oid: Oid, |
| 58 | ) !void { |
| 59 | try repository.odb.seekOid(commit_oid); |
| 60 | const tree_oid = tree_oid: { |
| 61 | var commit_object = try repository.odb.readObject(); |
| 62 | if (commit_object.type != .commit) return error.NotACommit; |
| 63 | break :tree_oid try getCommitTree(commit_object.data); |
| 64 | }; |
| 65 | try repository.checkoutTree(worktree, tree_oid); |
| 66 | } |
| 67 | |
| 68 | /// Checks out the tree at `tree_oid` to `worktree`. |
| 69 | fn checkoutTree( |
| 70 | repository: *Repository, |
| 71 | dir: std.fs.Dir, |
| 72 | tree_oid: Oid, |
| 73 | ) !void { |
| 74 | try repository.odb.seekOid(tree_oid); |
| 75 | const tree_object = try repository.odb.readObject(); |
| 76 | if (tree_object.type != .tree) return error.NotATree; |
| 77 | // The tree object may be evicted from the object cache while we're |
| 78 | // iterating over it, so we can make a defensive copy here to make sure |
| 79 | // it remains valid until we're done with it |
| 80 | const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); |
| 81 | defer repository.odb.allocator.free(tree_data); |
| 82 | |
| 83 | var tree_iter: TreeIterator = .{ .data = tree_data }; |
| 84 | while (try tree_iter.next()) |entry| { |
| 85 | switch (entry.type) { |
| 86 | .directory => { |
| 87 | try dir.makeDir(entry.name); |
| 88 | var subdir = try dir.openDir(entry.name, .{}); |
| 89 | defer subdir.close(); |
| 90 | try repository.checkoutTree(subdir, entry.oid); |
| 91 | }, |
| 92 | .file => { |
| 93 | var file = try dir.createFile(entry.name, .{}); |
| 94 | defer file.close(); |
| 95 | try repository.odb.seekOid(entry.oid); |
| 96 | var file_object = try repository.odb.readObject(); |
| 97 | if (file_object.type != .blob) return error.InvalidFile; |
| 98 | try file.writeAll(file_object.data); |
| 99 | try file.sync(); |
| 100 | }, |
| 101 | .symlink => return error.SymlinkNotSupported, |
| 102 | .gitlink => { |
| 103 | // Consistent with git archive behavior, create the directory but |
| 104 | // do nothing else |
| 105 | try dir.makeDir(entry.name); |
| 106 | }, |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | /// Returns the ID of the tree associated with the given commit (provided as |
| 112 | /// raw object data). |
| 113 | fn getCommitTree(commit_data: []const u8) !Oid { |
| 114 | if (!mem.startsWith(u8, commit_data, "tree ") or |
| 115 | commit_data.len < "tree ".len + fmt_oid_length + "\n".len or |
| 116 | commit_data["tree ".len + fmt_oid_length] != '\n') |
| 117 | { |
| 118 | return error.InvalidCommit; |
| 119 | } |
| 120 | return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]); |
| 121 | } |
| 122 | |
| 123 | const TreeIterator = struct { |
| 124 | data: []const u8, |
| 125 | pos: usize = 0, |
| 126 | |
| 127 | const Entry = struct { |
| 128 | type: Type, |
| 129 | executable: bool, |
| 130 | name: [:0]const u8, |
| 131 | oid: Oid, |
| 132 | |
| 133 | const Type = enum(u4) { |
| 134 | directory = 0o4, |
| 135 | file = 0o10, |
| 136 | symlink = 0o12, |
| 137 | gitlink = 0o16, |
| 138 | }; |
| 139 | }; |
| 140 | |
| 141 | fn next(iterator: *TreeIterator) !?Entry { |
| 142 | if (iterator.pos == iterator.data.len) return null; |
| 143 | |
| 144 | const mode_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, ' ') orelse return error.InvalidTree; |
| 145 | const mode: packed struct { |
| 146 | permission: u9, |
| 147 | unused: u3, |
| 148 | type: u4, |
| 149 | } = @bitCast(std.fmt.parseUnsigned(u16, iterator.data[iterator.pos..mode_end], 8) catch return error.InvalidTree); |
| 150 | const @"type" = std.meta.intToEnum(Entry.Type, mode.type) catch return error.InvalidTree; |
| 151 | const executable = switch (mode.permission) { |
| 152 | 0 => if (@"type" == .file) return error.InvalidTree else false, |
| 153 | 0o644 => if (@"type" != .file) return error.InvalidTree else false, |
| 154 | 0o755 => if (@"type" != .file) return error.InvalidTree else true, |
| 155 | else => return error.InvalidTree, |
| 156 | }; |
| 157 | iterator.pos = mode_end + 1; |
| 158 | |
| 159 | const name_end = mem.indexOfScalarPos(u8, iterator.data, iterator.pos, 0) orelse return error.InvalidTree; |
| 160 | const name = iterator.data[iterator.pos..name_end :0]; |
| 161 | iterator.pos = name_end + 1; |
| 162 | |
| 163 | if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; |
| 164 | const oid = iterator.data[iterator.pos..][0..oid_length].*; |
| 165 | iterator.pos += oid_length; |
| 166 | |
| 167 | return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; |
| 168 | } |
| 169 | }; |
| 170 | }; |
| 171 | |
| 172 | /// A Git object database backed by a packfile. A packfile index is also used |
| 173 | /// for efficient access to objects in the packfile. |
| 174 | /// |
| 175 | /// The format of the packfile and its associated index are documented in |
| 176 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 177 | const Odb = struct { |
| 178 | pack_file: std.fs.File, |
| 179 | index_header: IndexHeader, |
| 180 | index_file: std.fs.File, |
| 181 | cache: ObjectCache = .{}, |
| 182 | allocator: Allocator, |
| 183 | |
| 184 | /// Initializes the database from open pack and index files. |
| 185 | fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb { |
| 186 | try pack_file.seekTo(0); |
| 187 | try index_file.seekTo(0); |
| 188 | const index_header = try IndexHeader.read(index_file.reader()); |
| 189 | return .{ |
| 190 | .pack_file = pack_file, |
| 191 | .index_header = index_header, |
| 192 | .index_file = index_file, |
| 193 | .allocator = allocator, |
| 194 | }; |
| 195 | } |
| 196 | |
| 197 | fn deinit(odb: *Odb) void { |
| 198 | odb.cache.deinit(odb.allocator); |
| 199 | odb.* = undefined; |
| 200 | } |
| 201 | |
| 202 | /// Reads the object at the current position in the database. |
| 203 | fn readObject(odb: *Odb) !Object { |
| 204 | var base_offset = try odb.pack_file.getPos(); |
| 205 | var base_header: EntryHeader = undefined; |
| 206 | var delta_offsets = std.ArrayListUnmanaged(u64){}; |
| 207 | defer delta_offsets.deinit(odb.allocator); |
| 208 | const base_object = while (true) { |
| 209 | if (odb.cache.get(base_offset)) |base_object| break base_object; |
| 210 | |
| 211 | base_header = try EntryHeader.read(odb.pack_file.reader()); |
| 212 | switch (base_header) { |
| 213 | .ofs_delta => |ofs_delta| { |
| 214 | try delta_offsets.append(odb.allocator, base_offset); |
| 215 | base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; |
| 216 | try odb.pack_file.seekTo(base_offset); |
| 217 | }, |
| 218 | .ref_delta => |ref_delta| { |
| 219 | try delta_offsets.append(odb.allocator, base_offset); |
| 220 | try odb.seekOid(ref_delta.base_object); |
| 221 | base_offset = try odb.pack_file.getPos(); |
| 222 | }, |
| 223 | else => { |
| 224 | const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); |
| 225 | errdefer odb.allocator.free(base_data); |
| 226 | const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; |
| 227 | try odb.cache.put(odb.allocator, base_offset, base_object); |
| 228 | break base_object; |
| 229 | }, |
| 230 | } |
| 231 | }; |
| 232 | |
| 233 | const base_data = try resolveDeltaChain( |
| 234 | odb.allocator, |
| 235 | odb.pack_file, |
| 236 | base_object, |
| 237 | delta_offsets.items, |
| 238 | &odb.cache, |
| 239 | ); |
| 240 | |
| 241 | return .{ .type = base_object.type, .data = base_data }; |
| 242 | } |
| 243 | |
| 244 | /// Seeks to the beginning of the object with the given ID. |
| 245 | fn seekOid(odb: *Odb, oid: Oid) !void { |
| 246 | const key = oid[0]; |
| 247 | var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; |
| 248 | var end_index = odb.index_header.fan_out_table[key]; |
| 249 | const found_index = while (start_index < end_index) { |
| 250 | const mid_index = start_index + (end_index - start_index) / 2; |
| 251 | try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); |
| 252 | const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length); |
| 253 | switch (mem.order(u8, &mid_oid, &oid)) { |
| 254 | .lt => start_index = mid_index + 1, |
| 255 | .gt => end_index = mid_index, |
| 256 | .eq => break mid_index, |
| 257 | } |
| 258 | } else return error.ObjectNotFound; |
| 259 | |
| 260 | const n_objects = odb.index_header.fan_out_table[255]; |
| 261 | const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); |
| 262 | try odb.index_file.seekTo(offset_values_start + found_index * 4); |
| 263 | const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readIntBig(u32)); |
| 264 | const pack_offset = pack_offset: { |
| 265 | if (l1_offset.big) { |
| 266 | const l2_offset_values_start = offset_values_start + n_objects * 4; |
| 267 | try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); |
| 268 | break :pack_offset try odb.index_file.reader().readIntBig(u64); |
| 269 | } else { |
| 270 | break :pack_offset l1_offset.value; |
| 271 | } |
| 272 | }; |
| 273 | |
| 274 | try odb.pack_file.seekTo(pack_offset); |
| 275 | } |
| 276 | }; |
| 277 | |
| 278 | const Object = struct { |
| 279 | type: Type, |
| 280 | data: []const u8, |
| 281 | |
| 282 | const Type = enum { |
| 283 | commit, |
| 284 | tree, |
| 285 | blob, |
| 286 | tag, |
| 287 | }; |
| 288 | }; |
| 289 | |
| 290 | /// A cache for object data. |
| 291 | /// |
| 292 | /// The purpose of this cache is to speed up resolution of deltas by caching the |
| 293 | /// results of resolving delta objects, while maintaining a maximum cache size |
| 294 | /// to avoid excessive memory usage. If the total size of the objects in the |
| 295 | /// cache exceeds the maximum, the cache will begin evicting the least recently |
| 296 | /// used objects: when resolving delta chains, the most recently used objects |
| 297 | /// will likely be more helpful as they will be further along in the chain |
| 298 | /// (skipping earlier reconstruction steps). |
| 299 | /// |
| 300 | /// Object data stored in the cache is managed by the cache. It should not be |
| 301 | /// freed by the caller at any point after inserting it into the cache. Any |
| 302 | /// objects remaining in the cache will be freed when the cache itself is freed. |
| 303 | const ObjectCache = struct { |
| 304 | objects: std.AutoHashMapUnmanaged(u64, CacheEntry) = .{}, |
| 305 | lru_nodes: LruList = .{}, |
| 306 | byte_size: usize = 0, |
| 307 | |
| 308 | const max_byte_size = 128 * 1024 * 1024; // 128MiB |
| 309 | /// A list of offsets stored in the cache, with the most recently used |
| 310 | /// entries at the end. |
| 311 | const LruList = std.DoublyLinkedList(u64); |
| 312 | const CacheEntry = struct { object: Object, lru_node: *LruList.Node }; |
| 313 | |
| 314 | fn deinit(cache: *ObjectCache, allocator: Allocator) void { |
| 315 | var object_iterator = cache.objects.iterator(); |
| 316 | while (object_iterator.next()) |object| { |
| 317 | allocator.free(object.value_ptr.object.data); |
| 318 | allocator.destroy(object.value_ptr.lru_node); |
| 319 | } |
| 320 | cache.objects.deinit(allocator); |
| 321 | cache.* = undefined; |
| 322 | } |
| 323 | |
| 324 | /// Gets an object from the cache, moving it to the most recently used |
| 325 | /// position if it is present. |
| 326 | fn get(cache: *ObjectCache, offset: u64) ?Object { |
| 327 | if (cache.objects.get(offset)) |entry| { |
| 328 | cache.lru_nodes.remove(entry.lru_node); |
| 329 | cache.lru_nodes.append(entry.lru_node); |
| 330 | return entry.object; |
| 331 | } else { |
| 332 | return null; |
| 333 | } |
| 334 | } |
| 335 | |
| 336 | /// Puts an object in the cache, possibly evicting older entries if the |
| 337 | /// cache exceeds its maximum size. Note that, although old objects may |
| 338 | /// be evicted, the object just added to the cache with this function |
| 339 | /// will not be evicted before the next call to `put` or `deinit` even if |
| 340 | /// it exceeds the maximum cache size. |
| 341 | fn put(cache: *ObjectCache, allocator: Allocator, offset: u64, object: Object) !void { |
| 342 | const lru_node = try allocator.create(LruList.Node); |
| 343 | errdefer allocator.destroy(lru_node); |
| 344 | lru_node.data = offset; |
| 345 | |
| 346 | const gop = try cache.objects.getOrPut(allocator, offset); |
| 347 | if (gop.found_existing) { |
| 348 | cache.byte_size -= gop.value_ptr.object.data.len; |
| 349 | cache.lru_nodes.remove(gop.value_ptr.lru_node); |
| 350 | allocator.destroy(gop.value_ptr.lru_node); |
| 351 | allocator.free(gop.value_ptr.object.data); |
| 352 | } |
| 353 | gop.value_ptr.* = .{ .object = object, .lru_node = lru_node }; |
| 354 | cache.byte_size += object.data.len; |
| 355 | cache.lru_nodes.append(lru_node); |
| 356 | |
| 357 | while (cache.byte_size > max_byte_size and cache.lru_nodes.len > 1) { |
| 358 | // The > 1 check is to make sure that we don't evict the most |
| 359 | // recently added node, even if it by itself happens to exceed the |
| 360 | // maximum size of the cache. |
| 361 | const evict_node = cache.lru_nodes.popFirst().?; |
| 362 | const evict_offset = evict_node.data; |
| 363 | allocator.destroy(evict_node); |
| 364 | const evict_object = cache.objects.get(evict_offset).?.object; |
| 365 | cache.byte_size -= evict_object.data.len; |
| 366 | allocator.free(evict_object.data); |
| 367 | _ = cache.objects.remove(evict_offset); |
| 368 | } |
| 369 | } |
| 370 | }; |
| 371 | |
| 372 | /// A single pkt-line in the Git protocol. |
| 373 | /// |
| 374 | /// The format of a pkt-line is documented in |
| 375 | /// [protocol-common](https://git-scm.com/docs/protocol-common). The special |
| 376 | /// meanings of the delimiter and response-end packets are documented in |
| 377 | /// [protocol-v2](https://git-scm.com/docs/protocol-v2). |
| 378 | const Packet = union(enum) { |
| 379 | flush, |
| 380 | delimiter, |
| 381 | response_end, |
| 382 | data: []const u8, |
| 383 | |
| 384 | const max_data_length = 65516; |
| 385 | |
| 386 | /// Reads a packet in pkt-line format. |
| 387 | fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { |
| 388 | const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; |
| 389 | switch (length) { |
| 390 | 0 => return .flush, |
| 391 | 1 => return .delimiter, |
| 392 | 2 => return .response_end, |
| 393 | 3 => return error.InvalidPacket, |
| 394 | else => if (length - 4 > max_data_length) return error.InvalidPacket, |
| 395 | } |
| 396 | const data = buf[0 .. length - 4]; |
| 397 | try reader.readNoEof(data); |
| 398 | return .{ .data = data }; |
| 399 | } |
| 400 | |
| 401 | /// Writes a packet in pkt-line format. |
| 402 | fn write(packet: Packet, writer: anytype) !void { |
| 403 | switch (packet) { |
| 404 | .flush => try writer.writeAll("0000"), |
| 405 | .delimiter => try writer.writeAll("0001"), |
| 406 | .response_end => try writer.writeAll("0002"), |
| 407 | .data => |data| { |
| 408 | assert(data.len <= max_data_length); |
| 409 | try writer.print("{x:0>4}", .{data.len + 4}); |
| 410 | try writer.writeAll(data); |
| 411 | }, |
| 412 | } |
| 413 | } |
| 414 | }; |
| 415 | |
| 416 | /// A client session for the Git protocol, currently limited to an HTTP(S) |
| 417 | /// transport. Only protocol version 2 is supported, as documented in |
| 418 | /// [protocol-v2](https://git-scm.com/docs/protocol-v2). |
| 419 | pub const Session = struct { |
| 420 | transport: *std.http.Client, |
| 421 | uri: std.Uri, |
| 422 | supports_agent: bool = false, |
| 423 | supports_shallow: bool = false, |
| 424 | |
| 425 | const agent = "zig/" ++ @import("builtin").zig_version_string; |
| 426 | const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent}); |
| 427 | |
| 428 | /// Discovers server capabilities. This should be called before using any |
| 429 | /// other client functionality, or the client will be forced to default to |
| 430 | /// the bare minimum server requirements, which may be considerably less |
| 431 | /// efficient (e.g. no shallow fetches). |
| 432 | /// |
| 433 | /// See the note on `getCapabilities` regarding `redirect_uri`. |
| 434 | pub fn discoverCapabilities( |
| 435 | session: *Session, |
| 436 | allocator: Allocator, |
| 437 | redirect_uri: *[]u8, |
| 438 | ) !void { |
| 439 | var capability_iterator = try session.getCapabilities(allocator, redirect_uri); |
| 440 | defer capability_iterator.deinit(); |
| 441 | while (try capability_iterator.next()) |capability| { |
| 442 | if (mem.eql(u8, capability.key, "agent")) { |
| 443 | session.supports_agent = true; |
| 444 | } else if (mem.eql(u8, capability.key, "fetch")) { |
| 445 | var feature_iterator = mem.splitScalar(u8, capability.value orelse continue, ' '); |
| 446 | while (feature_iterator.next()) |feature| { |
| 447 | if (mem.eql(u8, feature, "shallow")) { |
| 448 | session.supports_shallow = true; |
| 449 | } |
| 450 | } |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | /// Returns an iterator over capabilities supported by the server. |
| 456 | /// |
| 457 | /// If the server redirects the request, `error.Redirected` is returned and |
| 458 | /// `redirect_uri` is populated with the URI resulting from the redirects. |
| 459 | /// When this occurs, the value of `redirect_uri` must be freed with |
| 460 | /// `allocator` when the caller is done with it. |
| 461 | fn getCapabilities( |
| 462 | session: Session, |
| 463 | allocator: Allocator, |
| 464 | redirect_uri: *[]u8, |
| 465 | ) !CapabilityIterator { |
| 466 | var info_refs_uri = session.uri; |
| 467 | info_refs_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "info/refs" }); |
| 468 | defer allocator.free(info_refs_uri.path); |
| 469 | info_refs_uri.query = "service=git-upload-pack"; |
| 470 | info_refs_uri.fragment = null; |
| 471 | |
| 472 | var headers = std.http.Headers.init(allocator); |
| 473 | defer headers.deinit(); |
| 474 | try headers.append("Git-Protocol", "version=2"); |
| 475 | |
| 476 | var request = try session.transport.request(.GET, info_refs_uri, headers, .{ |
| 477 | .max_redirects = 3, |
| 478 | }); |
| 479 | errdefer request.deinit(); |
| 480 | try request.start(.{}); |
| 481 | try request.finish(); |
| 482 | |
| 483 | try request.wait(); |
| 484 | if (request.response.status != .ok) return error.ProtocolError; |
| 485 | if (request.redirects_left < 3) { |
| 486 | if (!mem.endsWith(u8, request.uri.path, "/info/refs")) return error.UnparseableRedirect; |
| 487 | var new_uri = request.uri; |
| 488 | new_uri.path = new_uri.path[0 .. new_uri.path.len - "/info/refs".len]; |
| 489 | new_uri.query = null; |
| 490 | redirect_uri.* = try std.fmt.allocPrint(allocator, "{+/}", .{new_uri}); |
| 491 | return error.Redirected; |
| 492 | } |
| 493 | |
| 494 | const reader = request.reader(); |
| 495 | var buf: [Packet.max_data_length]u8 = undefined; |
| 496 | var state: enum { response_start, response_content } = .response_start; |
| 497 | while (true) { |
| 498 | // Some Git servers (at least GitHub) include an additional |
| 499 | // '# service=git-upload-pack' informative response before sending |
| 500 | // the expected 'version 2' packet and capability information. |
| 501 | // This is not universal: SourceHut, for example, does not do this. |
| 502 | // Thus, we need to skip any such useless additional responses |
| 503 | // before we get the one we're actually looking for. The responses |
| 504 | // will be delimited by flush packets. |
| 505 | const packet = Packet.read(reader, &buf) catch |e| switch (e) { |
| 506 | error.EndOfStream => return error.UnsupportedProtocol, // 'version 2' packet not found |
| 507 | else => |other| return other, |
| 508 | }; |
| 509 | switch (packet) { |
| 510 | .flush => state = .response_start, |
| 511 | .data => |data| switch (state) { |
| 512 | .response_start => if (mem.eql(u8, data, "version 2\n")) { |
| 513 | return .{ .request = request }; |
| 514 | } else { |
| 515 | state = .response_content; |
| 516 | }, |
| 517 | else => {}, |
| 518 | }, |
| 519 | else => return error.UnexpectedPacket, |
| 520 | } |
| 521 | } |
| 522 | } |
| 523 | |
| 524 | const CapabilityIterator = struct { |
| 525 | request: std.http.Client.Request, |
| 526 | buf: [Packet.max_data_length]u8 = undefined, |
| 527 | |
| 528 | const Capability = struct { |
| 529 | key: []const u8, |
| 530 | value: ?[]const u8 = null, |
| 531 | }; |
| 532 | |
| 533 | fn deinit(iterator: *CapabilityIterator) void { |
| 534 | iterator.request.deinit(); |
| 535 | iterator.* = undefined; |
| 536 | } |
| 537 | |
| 538 | fn next(iterator: *CapabilityIterator) !?Capability { |
| 539 | switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { |
| 540 | .flush => return null, |
| 541 | .data => |data| if (data.len > 0 and data[data.len - 1] == '\n') { |
| 542 | if (mem.indexOfScalar(u8, data, '=')) |separator_pos| { |
| 543 | return .{ .key = data[0..separator_pos], .value = data[separator_pos + 1 .. data.len - 1] }; |
| 544 | } else { |
| 545 | return .{ .key = data[0 .. data.len - 1] }; |
| 546 | } |
| 547 | } else return error.UnexpectedPacket, |
| 548 | else => return error.UnexpectedPacket, |
| 549 | } |
| 550 | } |
| 551 | }; |
| 552 | |
| 553 | const ListRefsOptions = struct { |
| 554 | /// The ref prefixes (if any) to use to filter the refs available on the |
| 555 | /// server. Note that the client must still check the returned refs |
| 556 | /// against its desired filters itself: the server is not required to |
| 557 | /// respect these prefix filters and may return other refs as well. |
| 558 | ref_prefixes: []const []const u8 = &.{}, |
| 559 | /// Whether to include symref targets for returned symbolic refs. |
| 560 | include_symrefs: bool = false, |
| 561 | /// Whether to include the peeled object ID for returned tag refs. |
| 562 | include_peeled: bool = false, |
| 563 | }; |
| 564 | |
| 565 | /// Returns an iterator over refs known to the server. |
| 566 | pub fn listRefs(session: Session, allocator: Allocator, options: ListRefsOptions) !RefIterator { |
| 567 | var upload_pack_uri = session.uri; |
| 568 | upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" }); |
| 569 | defer allocator.free(upload_pack_uri.path); |
| 570 | upload_pack_uri.query = null; |
| 571 | upload_pack_uri.fragment = null; |
| 572 | |
| 573 | var headers = std.http.Headers.init(allocator); |
| 574 | defer headers.deinit(); |
| 575 | try headers.append("Content-Type", "application/x-git-upload-pack-request"); |
| 576 | try headers.append("Git-Protocol", "version=2"); |
| 577 | |
| 578 | var body = std.ArrayListUnmanaged(u8){}; |
| 579 | defer body.deinit(allocator); |
| 580 | const body_writer = body.writer(allocator); |
| 581 | try Packet.write(.{ .data = "command=ls-refs\n" }, body_writer); |
| 582 | if (session.supports_agent) { |
| 583 | try Packet.write(.{ .data = agent_capability }, body_writer); |
| 584 | } |
| 585 | try Packet.write(.delimiter, body_writer); |
| 586 | for (options.ref_prefixes) |ref_prefix| { |
| 587 | const ref_prefix_packet = try std.fmt.allocPrint(allocator, "ref-prefix {s}\n", .{ref_prefix}); |
| 588 | defer allocator.free(ref_prefix_packet); |
| 589 | try Packet.write(.{ .data = ref_prefix_packet }, body_writer); |
| 590 | } |
| 591 | if (options.include_symrefs) { |
| 592 | try Packet.write(.{ .data = "symrefs\n" }, body_writer); |
| 593 | } |
| 594 | if (options.include_peeled) { |
| 595 | try Packet.write(.{ .data = "peel\n" }, body_writer); |
| 596 | } |
| 597 | try Packet.write(.flush, body_writer); |
| 598 | |
| 599 | var request = try session.transport.request(.POST, upload_pack_uri, headers, .{ |
| 600 | .handle_redirects = false, |
| 601 | }); |
| 602 | errdefer request.deinit(); |
| 603 | request.transfer_encoding = .{ .content_length = body.items.len }; |
| 604 | try request.start(.{}); |
| 605 | try request.writeAll(body.items); |
| 606 | try request.finish(); |
| 607 | |
| 608 | try request.wait(); |
| 609 | if (request.response.status != .ok) return error.ProtocolError; |
| 610 | |
| 611 | return .{ .request = request }; |
| 612 | } |
| 613 | |
| 614 | pub const RefIterator = struct { |
| 615 | request: std.http.Client.Request, |
| 616 | buf: [Packet.max_data_length]u8 = undefined, |
| 617 | |
| 618 | pub const Ref = struct { |
| 619 | oid: Oid, |
| 620 | name: []const u8, |
| 621 | symref_target: ?[]const u8, |
| 622 | peeled: ?Oid, |
| 623 | }; |
| 624 | |
| 625 | pub fn deinit(iterator: *RefIterator) void { |
| 626 | iterator.request.deinit(); |
| 627 | iterator.* = undefined; |
| 628 | } |
| 629 | |
| 630 | pub fn next(iterator: *RefIterator) !?Ref { |
| 631 | switch (try Packet.read(iterator.request.reader(), &iterator.buf)) { |
| 632 | .flush => return null, |
| 633 | .data => |data| { |
| 634 | const oid_sep_pos = mem.indexOfScalar(u8, data, ' ') orelse return error.InvalidRefPacket; |
| 635 | const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket; |
| 636 | |
| 637 | const name_sep_pos = mem.indexOfAnyPos(u8, data, oid_sep_pos + 1, " \n") orelse return error.InvalidRefPacket; |
| 638 | const name = data[oid_sep_pos + 1 .. name_sep_pos]; |
| 639 | |
| 640 | var symref_target: ?[]const u8 = null; |
| 641 | var peeled: ?Oid = null; |
| 642 | var last_sep_pos = name_sep_pos; |
| 643 | while (data[last_sep_pos] == ' ') { |
| 644 | const next_sep_pos = mem.indexOfAnyPos(u8, data, last_sep_pos + 1, " \n") orelse return error.InvalidRefPacket; |
| 645 | const attribute = data[last_sep_pos + 1 .. next_sep_pos]; |
| 646 | if (mem.startsWith(u8, attribute, "symref-target:")) { |
| 647 | symref_target = attribute["symref-target:".len..]; |
| 648 | } else if (mem.startsWith(u8, attribute, "peeled:")) { |
| 649 | peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket; |
| 650 | } |
| 651 | last_sep_pos = next_sep_pos; |
| 652 | } |
| 653 | |
| 654 | return .{ .oid = oid, .name = name, .symref_target = symref_target, .peeled = peeled }; |
| 655 | }, |
| 656 | else => return error.UnexpectedPacket, |
| 657 | } |
| 658 | } |
| 659 | }; |
| 660 | |
| 661 | /// Fetches the given refs from the server. A shallow fetch (depth 1) is |
| 662 | /// performed if the server supports it. |
| 663 | pub fn fetch(session: Session, allocator: Allocator, wants: []const []const u8) !FetchStream { |
| 664 | var upload_pack_uri = session.uri; |
| 665 | upload_pack_uri.path = try std.fs.path.resolvePosix(allocator, &.{ "/", session.uri.path, "git-upload-pack" }); |
| 666 | defer allocator.free(upload_pack_uri.path); |
| 667 | upload_pack_uri.query = null; |
| 668 | upload_pack_uri.fragment = null; |
| 669 | |
| 670 | var headers = std.http.Headers.init(allocator); |
| 671 | defer headers.deinit(); |
| 672 | try headers.append("Content-Type", "application/x-git-upload-pack-request"); |
| 673 | try headers.append("Git-Protocol", "version=2"); |
| 674 | |
| 675 | var body = std.ArrayListUnmanaged(u8){}; |
| 676 | defer body.deinit(allocator); |
| 677 | const body_writer = body.writer(allocator); |
| 678 | try Packet.write(.{ .data = "command=fetch\n" }, body_writer); |
| 679 | if (session.supports_agent) { |
| 680 | try Packet.write(.{ .data = agent_capability }, body_writer); |
| 681 | } |
| 682 | try Packet.write(.delimiter, body_writer); |
| 683 | // Our packfile parser supports the OFS_DELTA object type |
| 684 | try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); |
| 685 | // We do not currently convey server progress information to the user |
| 686 | try Packet.write(.{ .data = "no-progress\n" }, body_writer); |
| 687 | if (session.supports_shallow) { |
| 688 | try Packet.write(.{ .data = "deepen 1\n" }, body_writer); |
| 689 | } |
| 690 | for (wants) |want| { |
| 691 | var buf: [Packet.max_data_length]u8 = undefined; |
| 692 | const arg = std.fmt.bufPrint(&buf, "want {s}\n", .{want}) catch unreachable; |
| 693 | try Packet.write(.{ .data = arg }, body_writer); |
| 694 | } |
| 695 | try Packet.write(.{ .data = "done\n" }, body_writer); |
| 696 | try Packet.write(.flush, body_writer); |
| 697 | |
| 698 | var request = try session.transport.request(.POST, upload_pack_uri, headers, .{ |
| 699 | .handle_redirects = false, |
| 700 | }); |
| 701 | errdefer request.deinit(); |
| 702 | request.transfer_encoding = .{ .content_length = body.items.len }; |
| 703 | try request.start(.{}); |
| 704 | try request.writeAll(body.items); |
| 705 | try request.finish(); |
| 706 | |
| 707 | try request.wait(); |
| 708 | if (request.response.status != .ok) return error.ProtocolError; |
| 709 | |
| 710 | const reader = request.reader(); |
| 711 | // We are not interested in any of the sections of the returned fetch |
| 712 | // data other than the packfile section, since we aren't doing anything |
| 713 | // complex like ref negotiation (this is a fresh clone). |
| 714 | var state: enum { section_start, section_content } = .section_start; |
| 715 | while (true) { |
| 716 | var buf: [Packet.max_data_length]u8 = undefined; |
| 717 | const packet = try Packet.read(reader, &buf); |
| 718 | switch (state) { |
| 719 | .section_start => switch (packet) { |
| 720 | .data => |data| if (mem.eql(u8, data, "packfile\n")) { |
| 721 | return .{ .request = request }; |
| 722 | } else { |
| 723 | state = .section_content; |
| 724 | }, |
| 725 | else => return error.UnexpectedPacket, |
| 726 | }, |
| 727 | .section_content => switch (packet) { |
| 728 | .delimiter => state = .section_start, |
| 729 | .data => {}, |
| 730 | else => return error.UnexpectedPacket, |
| 731 | }, |
| 732 | } |
| 733 | } |
| 734 | } |
| 735 | |
| 736 | pub const FetchStream = struct { |
| 737 | request: std.http.Client.Request, |
| 738 | buf: [Packet.max_data_length]u8 = undefined, |
| 739 | pos: usize = 0, |
| 740 | len: usize = 0, |
| 741 | |
| 742 | pub fn deinit(stream: *FetchStream) void { |
| 743 | stream.request.deinit(); |
| 744 | } |
| 745 | |
| 746 | pub const ReadError = std.http.Client.Request.ReadError || error{ |
| 747 | InvalidPacket, |
| 748 | ProtocolError, |
| 749 | UnexpectedPacket, |
| 750 | }; |
| 751 | pub const Reader = std.io.Reader(*FetchStream, ReadError, read); |
| 752 | |
| 753 | const StreamCode = enum(u8) { |
| 754 | pack_data = 1, |
| 755 | progress = 2, |
| 756 | fatal_error = 3, |
| 757 | _, |
| 758 | }; |
| 759 | |
| 760 | pub fn reader(stream: *FetchStream) Reader { |
| 761 | return .{ .context = stream }; |
| 762 | } |
| 763 | |
| 764 | pub fn read(stream: *FetchStream, buf: []u8) !usize { |
| 765 | if (stream.pos == stream.len) { |
| 766 | while (true) { |
| 767 | switch (try Packet.read(stream.request.reader(), &stream.buf)) { |
| 768 | .flush => return 0, |
| 769 | .data => |data| if (data.len > 1) switch (@as(StreamCode, @enumFromInt(data[0]))) { |
| 770 | .pack_data => { |
| 771 | stream.pos = 1; |
| 772 | stream.len = data.len; |
| 773 | break; |
| 774 | }, |
| 775 | .fatal_error => return error.ProtocolError, |
| 776 | else => {}, |
| 777 | }, |
| 778 | else => return error.UnexpectedPacket, |
| 779 | } |
| 780 | } |
| 781 | } |
| 782 | |
| 783 | const size = @min(buf.len, stream.len - stream.pos); |
| 784 | @memcpy(buf[0..size], stream.buf[stream.pos .. stream.pos + size]); |
| 785 | stream.pos += size; |
| 786 | return size; |
| 787 | } |
| 788 | }; |
| 789 | }; |
| 790 | |
| 791 | const PackHeader = struct { |
| 792 | total_objects: u32, |
| 793 | |
| 794 | const signature = "PACK"; |
| 795 | const supported_version = 2; |
| 796 | |
| 797 | fn read(reader: anytype) !PackHeader { |
| 798 | const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { |
| 799 | error.EndOfStream => return error.InvalidHeader, |
| 800 | else => |other| return other, |
| 801 | }; |
| 802 | if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; |
| 803 | const version = reader.readIntBig(u32) catch |e| switch (e) { |
| 804 | error.EndOfStream => return error.InvalidHeader, |
| 805 | else => |other| return other, |
| 806 | }; |
| 807 | if (version != supported_version) return error.UnsupportedVersion; |
| 808 | const total_objects = reader.readIntBig(u32) catch |e| switch (e) { |
| 809 | error.EndOfStream => return error.InvalidHeader, |
| 810 | else => |other| return other, |
| 811 | }; |
| 812 | return .{ .total_objects = total_objects }; |
| 813 | } |
| 814 | }; |
| 815 | |
| 816 | const EntryHeader = union(Type) { |
| 817 | commit: Undeltified, |
| 818 | tree: Undeltified, |
| 819 | blob: Undeltified, |
| 820 | tag: Undeltified, |
| 821 | ofs_delta: OfsDelta, |
| 822 | ref_delta: RefDelta, |
| 823 | |
| 824 | const Type = enum(u3) { |
| 825 | commit = 1, |
| 826 | tree = 2, |
| 827 | blob = 3, |
| 828 | tag = 4, |
| 829 | ofs_delta = 6, |
| 830 | ref_delta = 7, |
| 831 | }; |
| 832 | |
| 833 | const Undeltified = struct { |
| 834 | uncompressed_length: u64, |
| 835 | }; |
| 836 | |
| 837 | const OfsDelta = struct { |
| 838 | offset: u64, |
| 839 | uncompressed_length: u64, |
| 840 | }; |
| 841 | |
| 842 | const RefDelta = struct { |
| 843 | base_object: Oid, |
| 844 | uncompressed_length: u64, |
| 845 | }; |
| 846 | |
| 847 | fn objectType(header: EntryHeader) Object.Type { |
| 848 | return switch (header) { |
| 849 | inline .commit, .tree, .blob, .tag => |_, tag| @field(Object.Type, @tagName(tag)), |
| 850 | else => unreachable, |
| 851 | }; |
| 852 | } |
| 853 | |
| 854 | fn uncompressedLength(header: EntryHeader) u64 { |
| 855 | return switch (header) { |
| 856 | inline else => |entry| entry.uncompressed_length, |
| 857 | }; |
| 858 | } |
| 859 | |
| 860 | fn read(reader: anytype) !EntryHeader { |
| 861 | const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; |
| 862 | const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { |
| 863 | error.EndOfStream => return error.InvalidFormat, |
| 864 | else => |other| return other, |
| 865 | }); |
| 866 | const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; |
| 867 | var uncompressed_length: u64 = initial.len; |
| 868 | uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; |
| 869 | const @"type" = std.meta.intToEnum(EntryHeader.Type, initial.type) catch return error.InvalidFormat; |
| 870 | return switch (@"type") { |
| 871 | inline .commit, .tree, .blob, .tag => |tag| @unionInit(EntryHeader, @tagName(tag), .{ |
| 872 | .uncompressed_length = uncompressed_length, |
| 873 | }), |
| 874 | .ofs_delta => .{ .ofs_delta = .{ |
| 875 | .offset = try readOffsetVarInt(reader), |
| 876 | .uncompressed_length = uncompressed_length, |
| 877 | } }, |
| 878 | .ref_delta => .{ .ref_delta = .{ |
| 879 | .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) { |
| 880 | error.EndOfStream => return error.InvalidFormat, |
| 881 | else => |other| return other, |
| 882 | }, |
| 883 | .uncompressed_length = uncompressed_length, |
| 884 | } }, |
| 885 | }; |
| 886 | } |
| 887 | }; |
| 888 | |
| 889 | fn readSizeVarInt(r: anytype) !u64 { |
| 890 | const Byte = packed struct { value: u7, has_next: bool }; |
| 891 | var b: Byte = @bitCast(try r.readByte()); |
| 892 | var value: u64 = b.value; |
| 893 | var shift: u6 = 0; |
| 894 | while (b.has_next) { |
| 895 | b = @bitCast(try r.readByte()); |
| 896 | shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; |
| 897 | value |= @as(u64, b.value) << shift; |
| 898 | } |
| 899 | return value; |
| 900 | } |
| 901 | |
| 902 | fn readOffsetVarInt(r: anytype) !u64 { |
| 903 | const Byte = packed struct { value: u7, has_next: bool }; |
| 904 | var b: Byte = @bitCast(try r.readByte()); |
| 905 | var value: u64 = b.value; |
| 906 | while (b.has_next) { |
| 907 | b = @bitCast(try r.readByte()); |
| 908 | value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; |
| 909 | value |= b.value; |
| 910 | } |
| 911 | return value; |
| 912 | } |
| 913 | |
| 914 | const IndexHeader = struct { |
| 915 | fan_out_table: [256]u32, |
| 916 | |
| 917 | const signature = "\xFFtOc"; |
| 918 | const supported_version = 2; |
| 919 | const size = 4 + 4 + @sizeOf([256]u32); |
| 920 | |
| 921 | fn read(reader: anytype) !IndexHeader { |
| 922 | var header_bytes = try reader.readBytesNoEof(size); |
| 923 | if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; |
| 924 | const version = mem.readIntBig(u32, header_bytes[4..8]); |
| 925 | if (version != supported_version) return error.UnsupportedVersion; |
| 926 | |
| 927 | var fan_out_table: [256]u32 = undefined; |
| 928 | var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); |
| 929 | const fan_out_table_reader = fan_out_table_stream.reader(); |
| 930 | for (&fan_out_table) |*entry| { |
| 931 | entry.* = fan_out_table_reader.readIntBig(u32) catch unreachable; |
| 932 | } |
| 933 | return .{ .fan_out_table = fan_out_table }; |
| 934 | } |
| 935 | }; |
| 936 | |
| 937 | const IndexEntry = struct { |
| 938 | offset: u64, |
| 939 | crc32: u32, |
| 940 | }; |
| 941 | |
| 942 | /// Writes out a version 2 index for the given packfile, as documented in |
| 943 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 944 | pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void { |
| 945 | try pack.seekTo(0); |
| 946 | |
| 947 | var index_entries = std.AutoHashMapUnmanaged(Oid, IndexEntry){}; |
| 948 | defer index_entries.deinit(allocator); |
| 949 | var pending_deltas = std.ArrayListUnmanaged(IndexEntry){}; |
| 950 | defer pending_deltas.deinit(allocator); |
| 951 | |
| 952 | const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas); |
| 953 | |
| 954 | var cache: ObjectCache = .{}; |
| 955 | defer cache.deinit(allocator); |
| 956 | var remaining_deltas = pending_deltas.items.len; |
| 957 | while (remaining_deltas > 0) { |
| 958 | var i: usize = remaining_deltas; |
| 959 | while (i > 0) { |
| 960 | i -= 1; |
| 961 | const delta = pending_deltas.items[i]; |
| 962 | if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| { |
| 963 | try index_entries.put(allocator, oid, delta); |
| 964 | _ = pending_deltas.swapRemove(i); |
| 965 | } |
| 966 | } |
| 967 | if (pending_deltas.items.len == remaining_deltas) return error.IncompletePack; |
| 968 | remaining_deltas = pending_deltas.items.len; |
| 969 | } |
| 970 | |
| 971 | var oids = std.ArrayListUnmanaged(Oid){}; |
| 972 | defer oids.deinit(allocator); |
| 973 | try oids.ensureTotalCapacityPrecise(allocator, index_entries.count()); |
| 974 | var index_entries_iter = index_entries.iterator(); |
| 975 | while (index_entries_iter.next()) |entry| { |
| 976 | oids.appendAssumeCapacity(entry.key_ptr.*); |
| 977 | } |
| 978 | mem.sortUnstable(Oid, oids.items, {}, struct { |
| 979 | fn lessThan(_: void, o1: Oid, o2: Oid) bool { |
| 980 | return mem.lessThan(u8, &o1, &o2); |
| 981 | } |
| 982 | }.lessThan); |
| 983 | |
| 984 | var fan_out_table: [256]u32 = undefined; |
| 985 | var count: u32 = 0; |
| 986 | var fan_out_index: u8 = 0; |
| 987 | for (oids.items) |oid| { |
| 988 | if (oid[0] > fan_out_index) { |
| 989 | @memset(fan_out_table[fan_out_index..oid[0]], count); |
| 990 | fan_out_index = oid[0]; |
| 991 | } |
| 992 | count += 1; |
| 993 | } |
| 994 | @memset(fan_out_table[fan_out_index..], count); |
| 995 | |
| 996 | var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{})); |
| 997 | const writer = index_hashed_writer.writer(); |
| 998 | try writer.writeAll(IndexHeader.signature); |
| 999 | try writer.writeIntBig(u32, IndexHeader.supported_version); |
| 1000 | for (fan_out_table) |fan_out_entry| { |
| 1001 | try writer.writeIntBig(u32, fan_out_entry); |
| 1002 | } |
| 1003 | |
| 1004 | for (oids.items) |oid| { |
| 1005 | try writer.writeAll(&oid); |
| 1006 | } |
| 1007 | |
| 1008 | for (oids.items) |oid| { |
| 1009 | try writer.writeIntBig(u32, index_entries.get(oid).?.crc32); |
| 1010 | } |
| 1011 | |
| 1012 | var big_offsets = std.ArrayListUnmanaged(u64){}; |
| 1013 | defer big_offsets.deinit(allocator); |
| 1014 | for (oids.items) |oid| { |
| 1015 | const offset = index_entries.get(oid).?.offset; |
| 1016 | if (offset <= std.math.maxInt(u31)) { |
| 1017 | try writer.writeIntBig(u32, @intCast(offset)); |
| 1018 | } else { |
| 1019 | const index = big_offsets.items.len; |
| 1020 | try big_offsets.append(allocator, offset); |
| 1021 | try writer.writeIntBig(u32, @as(u32, @intCast(index)) | (1 << 31)); |
| 1022 | } |
| 1023 | } |
| 1024 | for (big_offsets.items) |offset| { |
| 1025 | try writer.writeIntBig(u64, offset); |
| 1026 | } |
| 1027 | |
| 1028 | try writer.writeAll(&pack_checksum); |
| 1029 | const index_checksum = index_hashed_writer.hasher.finalResult(); |
| 1030 | try index_writer.writeAll(&index_checksum); |
| 1031 | } |
| 1032 | |
| 1033 | /// Performs the first pass over the packfile data for index construction. |
| 1034 | /// This will index all non-delta objects, queue delta objects for further |
| 1035 | /// processing, and return the pack checksum (which is part of the index |
| 1036 | /// format). |
| 1037 | fn indexPackFirstPass( |
| 1038 | allocator: Allocator, |
| 1039 | pack: std.fs.File, |
| 1040 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), |
| 1041 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), |
| 1042 | ) ![Sha1.digest_length]u8 { |
| 1043 | var pack_buffered_reader = std.io.bufferedReader(pack.reader()); |
| 1044 | var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); |
| 1045 | var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{})); |
| 1046 | const pack_reader = pack_hashed_reader.reader(); |
| 1047 | |
| 1048 | const pack_header = try PackHeader.read(pack_reader); |
| 1049 | |
| 1050 | var current_entry: u32 = 0; |
| 1051 | while (current_entry < pack_header.total_objects) : (current_entry += 1) { |
| 1052 | const entry_offset = pack_counting_reader.bytes_read; |
| 1053 | var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); |
| 1054 | const entry_header = try EntryHeader.read(entry_crc32_reader.reader()); |
| 1055 | switch (entry_header) { |
| 1056 | inline .commit, .tree, .blob, .tag => |object, tag| { |
| 1057 | var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); |
| 1058 | defer entry_decompress_stream.deinit(); |
| 1059 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); |
| 1060 | var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); |
| 1061 | const entry_writer = entry_hashed_writer.writer(); |
| 1062 | // The object header is not included in the pack data but is |
| 1063 | // part of the object's ID |
| 1064 | try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length }); |
| 1065 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); |
| 1066 | try fifo.pump(entry_counting_reader.reader(), entry_writer); |
| 1067 | if (entry_counting_reader.bytes_read != object.uncompressed_length) { |
| 1068 | return error.InvalidObject; |
| 1069 | } |
| 1070 | const oid = entry_hashed_writer.hasher.finalResult(); |
| 1071 | try index_entries.put(allocator, oid, .{ |
| 1072 | .offset = entry_offset, |
| 1073 | .crc32 = entry_crc32_reader.hasher.final(), |
| 1074 | }); |
| 1075 | }, |
| 1076 | inline .ofs_delta, .ref_delta => |delta| { |
| 1077 | var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader()); |
| 1078 | defer entry_decompress_stream.deinit(); |
| 1079 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); |
| 1080 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); |
| 1081 | try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); |
| 1082 | if (entry_counting_reader.bytes_read != delta.uncompressed_length) { |
| 1083 | return error.InvalidObject; |
| 1084 | } |
| 1085 | try pending_deltas.append(allocator, .{ |
| 1086 | .offset = entry_offset, |
| 1087 | .crc32 = entry_crc32_reader.hasher.final(), |
| 1088 | }); |
| 1089 | }, |
| 1090 | } |
| 1091 | } |
| 1092 | |
| 1093 | const pack_checksum = pack_hashed_reader.hasher.finalResult(); |
| 1094 | const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length); |
| 1095 | if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) { |
| 1096 | return error.CorruptedPack; |
| 1097 | } |
| 1098 | _ = pack_buffered_reader.reader().readByte() catch |e| switch (e) { |
| 1099 | error.EndOfStream => return pack_checksum, |
| 1100 | else => |other| return other, |
| 1101 | }; |
| 1102 | return error.InvalidFormat; |
| 1103 | } |
| 1104 | |
| 1105 | /// Attempts to determine the final object ID of the given deltified object. |
| 1106 | /// May return null if this is not yet possible (if the delta is a ref-based |
| 1107 | /// delta and we do not yet know the offset of the base object). |
| 1108 | fn indexPackHashDelta( |
| 1109 | allocator: Allocator, |
| 1110 | pack: std.fs.File, |
| 1111 | delta: IndexEntry, |
| 1112 | index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), |
| 1113 | cache: *ObjectCache, |
| 1114 | ) !?Oid { |
| 1115 | // Figure out the chain of deltas to resolve |
| 1116 | var base_offset = delta.offset; |
| 1117 | var base_header: EntryHeader = undefined; |
| 1118 | var delta_offsets = std.ArrayListUnmanaged(u64){}; |
| 1119 | defer delta_offsets.deinit(allocator); |
| 1120 | const base_object = while (true) { |
| 1121 | if (cache.get(base_offset)) |base_object| break base_object; |
| 1122 | |
| 1123 | try pack.seekTo(base_offset); |
| 1124 | base_header = try EntryHeader.read(pack.reader()); |
| 1125 | switch (base_header) { |
| 1126 | .ofs_delta => |ofs_delta| { |
| 1127 | try delta_offsets.append(allocator, base_offset); |
| 1128 | base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidObject; |
| 1129 | }, |
| 1130 | .ref_delta => |ref_delta| { |
| 1131 | try delta_offsets.append(allocator, base_offset); |
| 1132 | base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; |
| 1133 | }, |
| 1134 | else => { |
| 1135 | const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); |
| 1136 | errdefer allocator.free(base_data); |
| 1137 | const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; |
| 1138 | try cache.put(allocator, base_offset, base_object); |
| 1139 | break base_object; |
| 1140 | }, |
| 1141 | } |
| 1142 | }; |
| 1143 | |
| 1144 | const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache); |
| 1145 | |
| 1146 | var entry_hasher = Sha1.init(.{}); |
| 1147 | var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher); |
| 1148 | try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); |
| 1149 | entry_hasher.update(base_data); |
| 1150 | return entry_hasher.finalResult(); |
| 1151 | } |
| 1152 | |
| 1153 | /// Resolves a chain of deltas, returning the final base object data. `pack` is |
| 1154 | /// assumed to be looking at the start of the object data for the base object of |
| 1155 | /// the chain, and will then apply the deltas in `delta_offsets` in reverse order |
| 1156 | /// to obtain the final object. |
| 1157 | fn resolveDeltaChain( |
| 1158 | allocator: Allocator, |
| 1159 | pack: std.fs.File, |
| 1160 | base_object: Object, |
| 1161 | delta_offsets: []const u64, |
| 1162 | cache: *ObjectCache, |
| 1163 | ) ![]const u8 { |
| 1164 | var base_data = base_object.data; |
| 1165 | var i: usize = delta_offsets.len; |
| 1166 | while (i > 0) { |
| 1167 | i -= 1; |
| 1168 | |
| 1169 | const delta_offset = delta_offsets[i]; |
| 1170 | try pack.seekTo(delta_offset); |
| 1171 | const delta_header = try EntryHeader.read(pack.reader()); |
| 1172 | var delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); |
| 1173 | defer allocator.free(delta_data); |
| 1174 | var delta_stream = std.io.fixedBufferStream(delta_data); |
| 1175 | const delta_reader = delta_stream.reader(); |
| 1176 | _ = try readSizeVarInt(delta_reader); // base object size |
| 1177 | const expanded_size = try readSizeVarInt(delta_reader); |
| 1178 | |
| 1179 | const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; |
| 1180 | var expanded_data = try allocator.alloc(u8, expanded_alloc_size); |
| 1181 | errdefer allocator.free(expanded_data); |
| 1182 | var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); |
| 1183 | var base_stream = std.io.fixedBufferStream(base_data); |
| 1184 | try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); |
| 1185 | if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; |
| 1186 | |
| 1187 | try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); |
| 1188 | base_data = expanded_data; |
| 1189 | } |
| 1190 | return base_data; |
| 1191 | } |
| 1192 | |
| 1193 | /// Reads the complete contents of an object from `reader`. This function may |
| 1194 | /// read more bytes than required from `reader`, so the reader position after |
| 1195 | /// returning is not reliable. |
| 1196 | fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { |
| 1197 | const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; |
| 1198 | var buffered_reader = std.io.bufferedReader(reader); |
| 1199 | var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader()); |
| 1200 | defer decompress_stream.deinit(); |
| 1201 | var data = try allocator.alloc(u8, alloc_size); |
| 1202 | errdefer allocator.free(data); |
| 1203 | try decompress_stream.reader().readNoEof(data); |
| 1204 | _ = decompress_stream.reader().readByte() catch |e| switch (e) { |
| 1205 | error.EndOfStream => return data, |
| 1206 | else => |other| return other, |
| 1207 | }; |
| 1208 | return error.InvalidFormat; |
| 1209 | } |
| 1210 | |
| 1211 | /// Expands delta data from `delta_reader` to `writer`. `base_object` must |
| 1212 | /// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). |
| 1213 | /// |
| 1214 | /// The format of the delta data is documented in |
| 1215 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 1216 | fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !void { |
| 1217 | while (true) { |
| 1218 | const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { |
| 1219 | error.EndOfStream => return, |
| 1220 | else => |other| return other, |
| 1221 | }); |
| 1222 | if (inst.copy) { |
| 1223 | const available: packed struct { |
| 1224 | offset1: bool, |
| 1225 | offset2: bool, |
| 1226 | offset3: bool, |
| 1227 | offset4: bool, |
| 1228 | size1: bool, |
| 1229 | size2: bool, |
| 1230 | size3: bool, |
| 1231 | } = @bitCast(inst.value); |
| 1232 | var offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ |
| 1233 | .offset1 = if (available.offset1) try delta_reader.readByte() else 0, |
| 1234 | .offset2 = if (available.offset2) try delta_reader.readByte() else 0, |
| 1235 | .offset3 = if (available.offset3) try delta_reader.readByte() else 0, |
| 1236 | .offset4 = if (available.offset4) try delta_reader.readByte() else 0, |
| 1237 | }; |
| 1238 | const offset: u32 = @bitCast(offset_parts); |
| 1239 | var size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ |
| 1240 | .size1 = if (available.size1) try delta_reader.readByte() else 0, |
| 1241 | .size2 = if (available.size2) try delta_reader.readByte() else 0, |
| 1242 | .size3 = if (available.size3) try delta_reader.readByte() else 0, |
| 1243 | }; |
| 1244 | var size: u24 = @bitCast(size_parts); |
| 1245 | if (size == 0) size = 0x10000; |
| 1246 | try base_object.seekTo(offset); |
| 1247 | var copy_reader = std.io.limitedReader(base_object.reader(), size); |
| 1248 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); |
| 1249 | try fifo.pump(copy_reader.reader(), writer); |
| 1250 | } else if (inst.value != 0) { |
| 1251 | var data_reader = std.io.limitedReader(delta_reader, inst.value); |
| 1252 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); |
| 1253 | try fifo.pump(data_reader.reader(), writer); |
| 1254 | } else { |
| 1255 | return error.InvalidDeltaInstruction; |
| 1256 | } |
| 1257 | } |
| 1258 | } |
| 1259 | |
| 1260 | fn HashedWriter( |
| 1261 | comptime WriterType: anytype, |
| 1262 | comptime HasherType: anytype, |
| 1263 | ) type { |
| 1264 | return struct { |
| 1265 | child_writer: WriterType, |
| 1266 | hasher: HasherType, |
| 1267 | |
| 1268 | const Error = WriterType.Error; |
| 1269 | const Writer = std.io.Writer(*@This(), Error, write); |
| 1270 | |
| 1271 | fn write(hashed_writer: *@This(), buf: []const u8) Error!usize { |
| 1272 | const amt = try hashed_writer.child_writer.write(buf); |
| 1273 | hashed_writer.hasher.update(buf); |
| 1274 | return amt; |
| 1275 | } |
| 1276 | |
| 1277 | fn writer(hashed_writer: *@This()) Writer { |
| 1278 | return .{ .context = hashed_writer }; |
| 1279 | } |
| 1280 | }; |
| 1281 | } |
| 1282 | |
| 1283 | fn hashedWriter( |
| 1284 | writer: anytype, |
| 1285 | hasher: anytype, |
| 1286 | ) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) { |
| 1287 | return .{ .child_writer = writer, .hasher = hasher }; |
| 1288 | } |
| 1289 | |
| 1290 | test "packfile indexing and checkout" { |
| 1291 | // To verify the contents of this packfile without using the code in this |
| 1292 | // file: |
| 1293 | // |
| 1294 | // 1. Create a new empty Git repository (`git init`) |
| 1295 | // 2. `git unpack-objects <path/to/testdata.pack` |
| 1296 | // 3. `git fsck` -> note the "dangling commit" ID (which matches the commit |
| 1297 | // checked out below) |
| 1298 | // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb` |
| 1299 | const testrepo_pack = @embedFile("git/testdata/testrepo.pack"); |
| 1300 | |
| 1301 | var git_dir = testing.tmpDir(.{}); |
| 1302 | defer git_dir.cleanup(); |
| 1303 | var pack_file = try git_dir.dir.createFile("testrepo.pack", .{ .read = true }); |
| 1304 | defer pack_file.close(); |
| 1305 | try pack_file.writeAll(testrepo_pack); |
| 1306 | |
| 1307 | var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); |
| 1308 | defer index_file.close(); |
| 1309 | try indexPack(testing.allocator, pack_file, index_file.writer()); |
| 1310 | |
| 1311 | // Arbitrary size limit on files read while checking the repository contents |
| 1312 | // (all files in the test repo are known to be much smaller than this) |
| 1313 | const max_file_size = 4096; |
| 1314 | |
| 1315 | const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); |
| 1316 | defer testing.allocator.free(index_file_data); |
| 1317 | // testrepo.idx is generated by Git. The index created by this file should |
| 1318 | // match it exactly. Running `git verify-pack -v testrepo.pack` can verify |
| 1319 | // this. |
| 1320 | const testrepo_idx = @embedFile("git/testdata/testrepo.idx"); |
| 1321 | try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); |
| 1322 | |
| 1323 | var repository = try Repository.init(testing.allocator, pack_file, index_file); |
| 1324 | defer repository.deinit(); |
| 1325 | |
| 1326 | var worktree = testing.tmpIterableDir(.{}); |
| 1327 | defer worktree.cleanup(); |
| 1328 | |
| 1329 | const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb"); |
| 1330 | try repository.checkout(worktree.iterable_dir.dir, commit_id); |
| 1331 | |
| 1332 | const expected_files: []const []const u8 = &.{ |
| 1333 | "dir/file", |
| 1334 | "dir/subdir/file", |
| 1335 | "dir/subdir/file2", |
| 1336 | "dir2/file", |
| 1337 | "dir3/file", |
| 1338 | "dir3/file2", |
| 1339 | "file", |
| 1340 | "file2", |
| 1341 | "file3", |
| 1342 | "file4", |
| 1343 | "file5", |
| 1344 | "file6", |
| 1345 | "file7", |
| 1346 | "file8", |
| 1347 | "file9", |
| 1348 | }; |
| 1349 | var actual_files: std.ArrayListUnmanaged([]u8) = .{}; |
| 1350 | defer actual_files.deinit(testing.allocator); |
| 1351 | defer for (actual_files.items) |file| testing.allocator.free(file); |
| 1352 | var walker = try worktree.iterable_dir.walk(testing.allocator); |
| 1353 | defer walker.deinit(); |
| 1354 | while (try walker.next()) |entry| { |
| 1355 | if (entry.kind != .file) continue; |
| 1356 | var path = try testing.allocator.dupe(u8, entry.path); |
| 1357 | errdefer testing.allocator.free(path); |
| 1358 | mem.replaceScalar(u8, path, std.fs.path.sep, '/'); |
| 1359 | try actual_files.append(testing.allocator, path); |
| 1360 | } |
| 1361 | mem.sortUnstable([]u8, actual_files.items, {}, struct { |
| 1362 | fn lessThan(_: void, a: []u8, b: []u8) bool { |
| 1363 | return mem.lessThan(u8, a, b); |
| 1364 | } |
| 1365 | }.lessThan); |
| 1366 | try testing.expectEqualDeep(expected_files, actual_files.items); |
| 1367 | |
| 1368 | const expected_file_contents = |
| 1369 | \\revision 1 |
| 1370 | \\revision 2 |
| 1371 | \\revision 4 |
| 1372 | \\revision 5 |
| 1373 | \\revision 7 |
| 1374 | \\revision 8 |
| 1375 | \\revision 9 |
| 1376 | \\revision 10 |
| 1377 | \\revision 12 |
| 1378 | \\revision 13 |
| 1379 | \\revision 14 |
| 1380 | \\revision 18 |
| 1381 | \\revision 19 |
| 1382 | \\ |
| 1383 | ; |
| 1384 | const actual_file_contents = try worktree.iterable_dir.dir.readFileAlloc(testing.allocator, "file", max_file_size); |
| 1385 | defer testing.allocator.free(actual_file_contents); |
| 1386 | try testing.expectEqualStrings(expected_file_contents, actual_file_contents); |
| 1387 | } |
| 1388 | |
| 1389 | /// Checks out a commit of a packfile. Intended for experimenting with and |
| 1390 | /// benchmarking possible optimizations to the indexing and checkout behavior. |
| 1391 | pub fn main() !void { |
| 1392 | const allocator = std.heap.c_allocator; |
| 1393 | |
| 1394 | const args = try std.process.argsAlloc(allocator); |
| 1395 | defer std.process.argsFree(allocator, args); |
| 1396 | if (args.len != 4) { |
| 1397 | return error.InvalidArguments; // Arguments: packfile commit worktree |
| 1398 | } |
| 1399 | |
| 1400 | var pack_file = try std.fs.cwd().openFile(args[1], .{}); |
| 1401 | defer pack_file.close(); |
| 1402 | const commit = try parseOid(args[2]); |
| 1403 | var worktree = try std.fs.cwd().makeOpenPath(args[3], .{}); |
| 1404 | defer worktree.close(); |
| 1405 | |
| 1406 | var git_dir = try worktree.makeOpenPath(".git", .{}); |
| 1407 | defer git_dir.close(); |
| 1408 | |
| 1409 | std.debug.print("Starting index...\n", .{}); |
| 1410 | var index_file = try git_dir.createFile("idx", .{ .read = true }); |
| 1411 | defer index_file.close(); |
| 1412 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); |
| 1413 | try indexPack(allocator, pack_file, index_buffered_writer.writer()); |
| 1414 | try index_buffered_writer.flush(); |
| 1415 | try index_file.sync(); |
| 1416 | |
| 1417 | std.debug.print("Starting checkout...\n", .{}); |
| 1418 | var repository = try Repository.init(allocator, pack_file, index_file); |
| 1419 | defer repository.deinit(); |
| 1420 | try repository.checkout(worktree, commit); |
| 1421 | } |
Ian Johnson <ian@ianjohnson.dev> 2023-09-23 00:35:41-04:00
Andrew Kelley <andrew@ziglang.org> 2023-09-30 18:30:43-07:00
src