| ... | ... | @@ -9,32 +9,133 @@ const mem = std.mem; |
| 9 | 9 | const testing = std.testing; |
| 10 | 10 | const Allocator = mem.Allocator; |
| 11 | 11 | const Sha1 = std.crypto.hash.Sha1; |
| 12 | const Sha256 = std.crypto.hash.sha2.Sha256; |
| 12 | 13 | const assert = std.debug.assert; |
| 13 | 14 | |
| 14 | | pub const oid_length = Sha1.digest_length; |
| 15 | | pub const fmt_oid_length = 2 * oid_length; |
| 16 | | /// The ID of a Git object (an SHA-1 hash). |
| 17 | | pub const Oid = [oid_length]u8; |
| 15 | /// The ID of a Git object. |
| 16 | pub const Oid = union(Format) { |
| 17 | sha1: [Sha1.digest_length]u8, |
| 18 | sha256: [Sha256.digest_length]u8, |
| 18 | 19 | |
| 19 | | pub fn parseOid(s: []const u8) !Oid { |
| 20 | | if (s.len != fmt_oid_length) return error.InvalidOid; |
| 21 | | var oid: Oid = undefined; |
| 22 | | for (&oid, 0..) |*b, i| { |
| 23 | | b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; |
| 20 | pub const max_formatted_length = len: { |
| 21 | var max: usize = 0; |
| 22 | for (std.enums.values(Format)) |f| { |
| 23 | max = @max(max, f.formattedLength()); |
| 24 | } |
| 25 | break :len max; |
| 26 | }; |
| 27 | |
| 28 | pub const Format = enum { |
| 29 | sha1, |
| 30 | sha256, |
| 31 | |
| 32 | pub fn byteLength(f: Format) usize { |
| 33 | return switch (f) { |
| 34 | .sha1 => Sha1.digest_length, |
| 35 | .sha256 => Sha256.digest_length, |
| 36 | }; |
| 37 | } |
| 38 | |
| 39 | pub fn formattedLength(f: Format) usize { |
| 40 | return 2 * f.byteLength(); |
| 41 | } |
| 42 | }; |
| 43 | |
| 44 | const Hasher = union(Format) { |
| 45 | sha1: Sha1, |
| 46 | sha256: Sha256, |
| 47 | |
| 48 | fn init(oid_format: Format) Hasher { |
| 49 | return switch (oid_format) { |
| 50 | .sha1 => .{ .sha1 = Sha1.init(.{}) }, |
| 51 | .sha256 => .{ .sha256 = Sha256.init(.{}) }, |
| 52 | }; |
| 53 | } |
| 54 | |
| 55 | // Must be public for use from HashedReader and HashedWriter. |
| 56 | pub fn update(hasher: *Hasher, b: []const u8) void { |
| 57 | switch (hasher.*) { |
| 58 | inline else => |*inner| inner.update(b), |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | fn finalResult(hasher: *Hasher) Oid { |
| 63 | return switch (hasher.*) { |
| 64 | inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), |
| 65 | }; |
| 66 | } |
| 67 | }; |
| 68 | |
| 69 | pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { |
| 70 | assert(bytes.len == oid_format.byteLength()); |
| 71 | return switch (oid_format) { |
| 72 | inline else => |tag| @unionInit(Oid, @tagName(tag), bytes[0..comptime tag.byteLength()].*), |
| 73 | }; |
| 24 | 74 | } |
| 25 | | return oid; |
| 26 | | } |
| 27 | 75 | |
| 28 | | test parseOid { |
| 29 | | try testing.expectEqualSlices( |
| 30 | | u8, |
| 31 | | &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, |
| 32 | | &try parseOid("ce919ccf45951856a762ffdb8ef850301cd8c588"), |
| 33 | | ); |
| 34 | | try testing.expectError(error.InvalidOid, parseOid("ce919ccf")); |
| 35 | | try testing.expectError(error.InvalidOid, parseOid("master")); |
| 36 | | try testing.expectError(error.InvalidOid, parseOid("HEAD")); |
| 37 | | } |
| 76 | pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { |
| 77 | return switch (oid_format) { |
| 78 | inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), |
| 79 | }; |
| 80 | } |
| 81 | |
| 82 | pub fn parse(oid_format: Format, s: []const u8) error{InvalidOid}!Oid { |
| 83 | switch (oid_format) { |
| 84 | inline else => |tag| { |
| 85 | if (s.len != tag.formattedLength()) return error.InvalidOid; |
| 86 | var bytes: [tag.byteLength()]u8 = undefined; |
| 87 | for (&bytes, 0..) |*b, i| { |
| 88 | b.* = std.fmt.parseUnsigned(u8, s[2 * i ..][0..2], 16) catch return error.InvalidOid; |
| 89 | } |
| 90 | return @unionInit(Oid, @tagName(tag), bytes); |
| 91 | }, |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | test parse { |
| 96 | try testing.expectEqualSlices( |
| 97 | u8, |
| 98 | &.{ 0xCE, 0x91, 0x9C, 0xCF, 0x45, 0x95, 0x18, 0x56, 0xA7, 0x62, 0xFF, 0xDB, 0x8E, 0xF8, 0x50, 0x30, 0x1C, 0xD8, 0xC5, 0x88 }, |
| 99 | &(try parse(.sha1, "ce919ccf45951856a762ffdb8ef850301cd8c588")).sha1, |
| 100 | ); |
| 101 | try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf45951856a762ffdb8ef850301cd8c588")); |
| 102 | try testing.expectError(error.InvalidOid, parse(.sha1, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")); |
| 103 | try testing.expectEqualSlices( |
| 104 | u8, |
| 105 | &.{ 0x7F, 0x44, 0x4A, 0x92, 0xBD, 0x45, 0x72, 0xEE, 0x4A, 0x28, 0xB2, 0xC6, 0x30, 0x59, 0x92, 0x4A, 0x9C, 0xA1, 0x82, 0x91, 0x38, 0x55, 0x3E, 0xF3, 0xE7, 0xC4, 0x1E, 0xE1, 0x59, 0xAF, 0xAE, 0x7A }, |
| 106 | &(try parse(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a")).sha256, |
| 107 | ); |
| 108 | try testing.expectError(error.InvalidOid, parse(.sha1, "ce919ccf")); |
| 109 | try testing.expectError(error.InvalidOid, parse(.sha256, "ce919ccf")); |
| 110 | try testing.expectError(error.InvalidOid, parse(.sha1, "master")); |
| 111 | try testing.expectError(error.InvalidOid, parse(.sha256, "master")); |
| 112 | try testing.expectError(error.InvalidOid, parse(.sha1, "HEAD")); |
| 113 | try testing.expectError(error.InvalidOid, parse(.sha256, "HEAD")); |
| 114 | } |
| 115 | |
| 116 | pub fn parseAny(s: []const u8) error{InvalidOid}!Oid { |
| 117 | return for (std.enums.values(Format)) |f| { |
| 118 | if (s.len == f.formattedLength()) break parse(f, s); |
| 119 | } else error.InvalidOid; |
| 120 | } |
| 121 | |
| 122 | pub fn format( |
| 123 | oid: Oid, |
| 124 | comptime fmt: []const u8, |
| 125 | options: std.fmt.FormatOptions, |
| 126 | writer: anytype, |
| 127 | ) @TypeOf(writer).Error!void { |
| 128 | _ = fmt; |
| 129 | _ = options; |
| 130 | try writer.print("{}", .{std.fmt.fmtSliceHexLower(oid.slice())}); |
| 131 | } |
| 132 | |
| 133 | pub fn slice(oid: *const Oid) []const u8 { |
| 134 | return switch (oid.*) { |
| 135 | inline else => |*bytes| bytes, |
| 136 | }; |
| 137 | } |
| 138 | }; |
| 38 | 139 | |
| 39 | 140 | pub const Diagnostics = struct { |
| 40 | 141 | allocator: Allocator, |
| ... | ... | @@ -72,8 +173,8 @@ pub const Diagnostics = struct { |
| 72 | 173 | pub const Repository = struct { |
| 73 | 174 | odb: Odb, |
| 74 | 175 | |
| 75 | | pub fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Repository { |
| 76 | | return .{ .odb = try Odb.init(allocator, pack_file, index_file) }; |
| 176 | pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { |
| 177 | return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; |
| 77 | 178 | } |
| 78 | 179 | |
| 79 | 180 | pub fn deinit(repository: *Repository) void { |
| ... | ... | @@ -92,7 +193,7 @@ pub const Repository = struct { |
| 92 | 193 | const tree_oid = tree_oid: { |
| 93 | 194 | const commit_object = try repository.odb.readObject(); |
| 94 | 195 | if (commit_object.type != .commit) return error.NotACommit; |
| 95 | | break :tree_oid try getCommitTree(commit_object.data); |
| 196 | break :tree_oid try getCommitTree(repository.odb.format, commit_object.data); |
| 96 | 197 | }; |
| 97 | 198 | try repository.checkoutTree(worktree, tree_oid, "", diagnostics); |
| 98 | 199 | } |
| ... | ... | @@ -114,7 +215,11 @@ pub const Repository = struct { |
| 114 | 215 | const tree_data = try repository.odb.allocator.dupe(u8, tree_object.data); |
| 115 | 216 | defer repository.odb.allocator.free(tree_data); |
| 116 | 217 | |
| 117 | | var tree_iter: TreeIterator = .{ .data = tree_data }; |
| 218 | var tree_iter: TreeIterator = .{ |
| 219 | .format = repository.odb.format, |
| 220 | .data = tree_data, |
| 221 | .pos = 0, |
| 222 | }; |
| 118 | 223 | while (try tree_iter.next()) |entry| { |
| 119 | 224 | switch (entry.type) { |
| 120 | 225 | .directory => { |
| ... | ... | @@ -170,19 +275,20 @@ pub const Repository = struct { |
| 170 | 275 | |
| 171 | 276 | /// Returns the ID of the tree associated with the given commit (provided as |
| 172 | 277 | /// raw object data). |
| 173 | | fn getCommitTree(commit_data: []const u8) !Oid { |
| 278 | fn getCommitTree(format: Oid.Format, commit_data: []const u8) !Oid { |
| 174 | 279 | if (!mem.startsWith(u8, commit_data, "tree ") or |
| 175 | | commit_data.len < "tree ".len + fmt_oid_length + "\n".len or |
| 176 | | commit_data["tree ".len + fmt_oid_length] != '\n') |
| 280 | commit_data.len < "tree ".len + format.formattedLength() + "\n".len or |
| 281 | commit_data["tree ".len + format.formattedLength()] != '\n') |
| 177 | 282 | { |
| 178 | 283 | return error.InvalidCommit; |
| 179 | 284 | } |
| 180 | | return try parseOid(commit_data["tree ".len..][0..fmt_oid_length]); |
| 285 | return try .parse(format, commit_data["tree ".len..][0..format.formattedLength()]); |
| 181 | 286 | } |
| 182 | 287 | |
| 183 | 288 | const TreeIterator = struct { |
| 289 | format: Oid.Format, |
| 184 | 290 | data: []const u8, |
| 185 | | pos: usize = 0, |
| 291 | pos: usize, |
| 186 | 292 | |
| 187 | 293 | const Entry = struct { |
| 188 | 294 | type: Type, |
| ... | ... | @@ -220,8 +326,9 @@ pub const Repository = struct { |
| 220 | 326 | const name = iterator.data[iterator.pos..name_end :0]; |
| 221 | 327 | iterator.pos = name_end + 1; |
| 222 | 328 | |
| 329 | const oid_length = iterator.format.byteLength(); |
| 223 | 330 | if (iterator.pos + oid_length > iterator.data.len) return error.InvalidTree; |
| 224 | | const oid = iterator.data[iterator.pos..][0..oid_length].*; |
| 331 | const oid: Oid = .fromBytes(iterator.format, iterator.data[iterator.pos..][0..oid_length]); |
| 225 | 332 | iterator.pos += oid_length; |
| 226 | 333 | |
| 227 | 334 | return .{ .type = @"type", .executable = executable, .name = name, .oid = oid }; |
| ... | ... | @@ -235,6 +342,7 @@ pub const Repository = struct { |
| 235 | 342 | /// The format of the packfile and its associated index are documented in |
| 236 | 343 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 237 | 344 | const Odb = struct { |
| 345 | format: Oid.Format, |
| 238 | 346 | pack_file: std.fs.File, |
| 239 | 347 | index_header: IndexHeader, |
| 240 | 348 | index_file: std.fs.File, |
| ... | ... | @@ -242,11 +350,12 @@ const Odb = struct { |
| 242 | 350 | allocator: Allocator, |
| 243 | 351 | |
| 244 | 352 | /// Initializes the database from open pack and index files. |
| 245 | | fn init(allocator: Allocator, pack_file: std.fs.File, index_file: std.fs.File) !Odb { |
| 353 | fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { |
| 246 | 354 | try pack_file.seekTo(0); |
| 247 | 355 | try index_file.seekTo(0); |
| 248 | 356 | const index_header = try IndexHeader.read(index_file.reader()); |
| 249 | 357 | return .{ |
| 358 | .format = format, |
| 250 | 359 | .pack_file = pack_file, |
| 251 | 360 | .index_header = index_header, |
| 252 | 361 | .index_file = index_file, |
| ... | ... | @@ -268,7 +377,7 @@ const Odb = struct { |
| 268 | 377 | const base_object = while (true) { |
| 269 | 378 | if (odb.cache.get(base_offset)) |base_object| break base_object; |
| 270 | 379 | |
| 271 | | base_header = try EntryHeader.read(odb.pack_file.reader()); |
| 380 | base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); |
| 272 | 381 | switch (base_header) { |
| 273 | 382 | .ofs_delta => |ofs_delta| { |
| 274 | 383 | try delta_offsets.append(odb.allocator, base_offset); |
| ... | ... | @@ -292,6 +401,7 @@ const Odb = struct { |
| 292 | 401 | |
| 293 | 402 | const base_data = try resolveDeltaChain( |
| 294 | 403 | odb.allocator, |
| 404 | odb.format, |
| 295 | 405 | odb.pack_file, |
| 296 | 406 | base_object, |
| 297 | 407 | delta_offsets.items, |
| ... | ... | @@ -303,14 +413,15 @@ const Odb = struct { |
| 303 | 413 | |
| 304 | 414 | /// Seeks to the beginning of the object with the given ID. |
| 305 | 415 | fn seekOid(odb: *Odb, oid: Oid) !void { |
| 306 | | const key = oid[0]; |
| 416 | const oid_length = odb.format.byteLength(); |
| 417 | const key = oid.slice()[0]; |
| 307 | 418 | var start_index = if (key > 0) odb.index_header.fan_out_table[key - 1] else 0; |
| 308 | 419 | var end_index = odb.index_header.fan_out_table[key]; |
| 309 | 420 | const found_index = while (start_index < end_index) { |
| 310 | 421 | const mid_index = start_index + (end_index - start_index) / 2; |
| 311 | 422 | try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); |
| 312 | | const mid_oid = try odb.index_file.reader().readBytesNoEof(oid_length); |
| 313 | | switch (mem.order(u8, &mid_oid, &oid)) { |
| 423 | const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); |
| 424 | switch (mem.order(u8, mid_oid.slice(), oid.slice())) { |
| 314 | 425 | .lt => start_index = mid_index + 1, |
| 315 | 426 | .gt => end_index = mid_index, |
| 316 | 427 | .eq => break mid_index, |
| ... | ... | @@ -495,6 +606,7 @@ pub const Session = struct { |
| 495 | 606 | location: Location, |
| 496 | 607 | supports_agent: bool, |
| 497 | 608 | supports_shallow: bool, |
| 609 | object_format: Oid.Format, |
| 498 | 610 | allocator: Allocator, |
| 499 | 611 | |
| 500 | 612 | const agent = "zig/" ++ @import("builtin").zig_version_string; |
| ... | ... | @@ -513,6 +625,7 @@ pub const Session = struct { |
| 513 | 625 | .location = try .init(allocator, uri), |
| 514 | 626 | .supports_agent = false, |
| 515 | 627 | .supports_shallow = false, |
| 628 | .object_format = .sha1, |
| 516 | 629 | .allocator = allocator, |
| 517 | 630 | }; |
| 518 | 631 | errdefer session.deinit(); |
| ... | ... | @@ -528,6 +641,10 @@ pub const Session = struct { |
| 528 | 641 | session.supports_shallow = true; |
| 529 | 642 | } |
| 530 | 643 | } |
| 644 | } else if (mem.eql(u8, capability.key, "object-format")) { |
| 645 | if (std.meta.stringToEnum(Oid.Format, capability.value orelse continue)) |format| { |
| 646 | session.object_format = format; |
| 647 | } |
| 531 | 648 | } |
| 532 | 649 | } |
| 533 | 650 | return session; |
| ... | ... | @@ -708,6 +825,11 @@ pub const Session = struct { |
| 708 | 825 | if (session.supports_agent) { |
| 709 | 826 | try Packet.write(.{ .data = agent_capability }, body_writer); |
| 710 | 827 | } |
| 828 | { |
| 829 | const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); |
| 830 | defer session.allocator.free(object_format_packet); |
| 831 | try Packet.write(.{ .data = object_format_packet }, body_writer); |
| 832 | } |
| 711 | 833 | try Packet.write(.delimiter, body_writer); |
| 712 | 834 | for (options.ref_prefixes) |ref_prefix| { |
| 713 | 835 | const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix}); |
| ... | ... | @@ -739,10 +861,14 @@ pub const Session = struct { |
| 739 | 861 | try request.wait(); |
| 740 | 862 | if (request.response.status != .ok) return error.ProtocolError; |
| 741 | 863 | |
| 742 | | return .{ .request = request }; |
| 864 | return .{ |
| 865 | .format = session.object_format, |
| 866 | .request = request, |
| 867 | }; |
| 743 | 868 | } |
| 744 | 869 | |
| 745 | 870 | pub const RefIterator = struct { |
| 871 | format: Oid.Format, |
| 746 | 872 | request: std.http.Client.Request, |
| 747 | 873 | buf: [Packet.max_data_length]u8 = undefined, |
| 748 | 874 | |
| ... | ... | @@ -764,7 +890,7 @@ pub const Session = struct { |
| 764 | 890 | .data => |data| { |
| 765 | 891 | const ref_data = Packet.normalizeText(data); |
| 766 | 892 | const oid_sep_pos = mem.indexOfScalar(u8, ref_data, ' ') orelse return error.InvalidRefPacket; |
| 767 | | const oid = parseOid(data[0..oid_sep_pos]) catch return error.InvalidRefPacket; |
| 893 | const oid = Oid.parse(iterator.format, data[0..oid_sep_pos]) catch return error.InvalidRefPacket; |
| 768 | 894 | |
| 769 | 895 | const name_sep_pos = mem.indexOfScalarPos(u8, ref_data, oid_sep_pos + 1, ' ') orelse ref_data.len; |
| 770 | 896 | const name = ref_data[oid_sep_pos + 1 .. name_sep_pos]; |
| ... | ... | @@ -778,7 +904,7 @@ pub const Session = struct { |
| 778 | 904 | if (mem.startsWith(u8, attribute, "symref-target:")) { |
| 779 | 905 | symref_target = attribute["symref-target:".len..]; |
| 780 | 906 | } else if (mem.startsWith(u8, attribute, "peeled:")) { |
| 781 | | peeled = parseOid(attribute["peeled:".len..]) catch return error.InvalidRefPacket; |
| 907 | peeled = Oid.parse(iterator.format, attribute["peeled:".len..]) catch return error.InvalidRefPacket; |
| 782 | 908 | } |
| 783 | 909 | last_sep_pos = next_sep_pos; |
| 784 | 910 | } |
| ... | ... | @@ -814,6 +940,11 @@ pub const Session = struct { |
| 814 | 940 | if (session.supports_agent) { |
| 815 | 941 | try Packet.write(.{ .data = agent_capability }, body_writer); |
| 816 | 942 | } |
| 943 | { |
| 944 | const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)}); |
| 945 | defer session.allocator.free(object_format_packet); |
| 946 | try Packet.write(.{ .data = object_format_packet }, body_writer); |
| 947 | } |
| 817 | 948 | try Packet.write(.delimiter, body_writer); |
| 818 | 949 | // Our packfile parser supports the OFS_DELTA object type |
| 819 | 950 | try Packet.write(.{ .data = "ofs-delta\n" }, body_writer); |
| ... | ... | @@ -997,7 +1128,7 @@ const EntryHeader = union(Type) { |
| 997 | 1128 | }; |
| 998 | 1129 | } |
| 999 | 1130 | |
| 1000 | | fn read(reader: anytype) !EntryHeader { |
| 1131 | fn read(format: Oid.Format, reader: anytype) !EntryHeader { |
| 1001 | 1132 | const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; |
| 1002 | 1133 | const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { |
| 1003 | 1134 | error.EndOfStream => return error.InvalidFormat, |
| ... | ... | @@ -1016,7 +1147,7 @@ const EntryHeader = union(Type) { |
| 1016 | 1147 | .uncompressed_length = uncompressed_length, |
| 1017 | 1148 | } }, |
| 1018 | 1149 | .ref_delta => .{ .ref_delta = .{ |
| 1019 | | .base_object = reader.readBytesNoEof(oid_length) catch |e| switch (e) { |
| 1150 | .base_object = Oid.readBytes(format, reader) catch |e| switch (e) { |
| 1020 | 1151 | error.EndOfStream => return error.InvalidFormat, |
| 1021 | 1152 | else => |other| return other, |
| 1022 | 1153 | }, |
| ... | ... | @@ -1081,7 +1212,7 @@ const IndexEntry = struct { |
| 1081 | 1212 | |
| 1082 | 1213 | /// Writes out a version 2 index for the given packfile, as documented in |
| 1083 | 1214 | /// [pack-format](https://git-scm.com/docs/pack-format). |
| 1084 | | pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) !void { |
| 1215 | pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { |
| 1085 | 1216 | try pack.seekTo(0); |
| 1086 | 1217 | |
| 1087 | 1218 | var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; |
| ... | ... | @@ -1089,7 +1220,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1089 | 1220 | var pending_deltas: std.ArrayListUnmanaged(IndexEntry) = .empty; |
| 1090 | 1221 | defer pending_deltas.deinit(allocator); |
| 1091 | 1222 | |
| 1092 | | const pack_checksum = try indexPackFirstPass(allocator, pack, &index_entries, &pending_deltas); |
| 1223 | const pack_checksum = try indexPackFirstPass(allocator, format, pack, &index_entries, &pending_deltas); |
| 1093 | 1224 | |
| 1094 | 1225 | var cache: ObjectCache = .{}; |
| 1095 | 1226 | defer cache.deinit(allocator); |
| ... | ... | @@ -1099,7 +1230,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1099 | 1230 | while (i > 0) { |
| 1100 | 1231 | i -= 1; |
| 1101 | 1232 | const delta = pending_deltas.items[i]; |
| 1102 | | if (try indexPackHashDelta(allocator, pack, delta, index_entries, &cache)) |oid| { |
| 1233 | if (try indexPackHashDelta(allocator, format, pack, delta, index_entries, &cache)) |oid| { |
| 1103 | 1234 | try index_entries.put(allocator, oid, delta); |
| 1104 | 1235 | _ = pending_deltas.swapRemove(i); |
| 1105 | 1236 | } |
| ... | ... | @@ -1117,7 +1248,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1117 | 1248 | } |
| 1118 | 1249 | mem.sortUnstable(Oid, oids.items, {}, struct { |
| 1119 | 1250 | fn lessThan(_: void, o1: Oid, o2: Oid) bool { |
| 1120 | | return mem.lessThan(u8, &o1, &o2); |
| 1251 | return mem.lessThan(u8, o1.slice(), o2.slice()); |
| 1121 | 1252 | } |
| 1122 | 1253 | }.lessThan); |
| 1123 | 1254 | |
| ... | ... | @@ -1125,15 +1256,16 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1125 | 1256 | var count: u32 = 0; |
| 1126 | 1257 | var fan_out_index: u8 = 0; |
| 1127 | 1258 | for (oids.items) |oid| { |
| 1128 | | if (oid[0] > fan_out_index) { |
| 1129 | | @memset(fan_out_table[fan_out_index..oid[0]], count); |
| 1130 | | fan_out_index = oid[0]; |
| 1259 | const key = oid.slice()[0]; |
| 1260 | if (key > fan_out_index) { |
| 1261 | @memset(fan_out_table[fan_out_index..key], count); |
| 1262 | fan_out_index = key; |
| 1131 | 1263 | } |
| 1132 | 1264 | count += 1; |
| 1133 | 1265 | } |
| 1134 | 1266 | @memset(fan_out_table[fan_out_index..], count); |
| 1135 | 1267 | |
| 1136 | | var index_hashed_writer = hashedWriter(index_writer, Sha1.init(.{})); |
| 1268 | var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); |
| 1137 | 1269 | const writer = index_hashed_writer.writer(); |
| 1138 | 1270 | try writer.writeAll(IndexHeader.signature); |
| 1139 | 1271 | try writer.writeInt(u32, IndexHeader.supported_version, .big); |
| ... | ... | @@ -1142,7 +1274,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1142 | 1274 | } |
| 1143 | 1275 | |
| 1144 | 1276 | for (oids.items) |oid| { |
| 1145 | | try writer.writeAll(&oid); |
| 1277 | try writer.writeAll(oid.slice()); |
| 1146 | 1278 | } |
| 1147 | 1279 | |
| 1148 | 1280 | for (oids.items) |oid| { |
| ... | ... | @@ -1165,9 +1297,9 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1165 | 1297 | try writer.writeInt(u64, offset, .big); |
| 1166 | 1298 | } |
| 1167 | 1299 | |
| 1168 | | try writer.writeAll(&pack_checksum); |
| 1300 | try writer.writeAll(pack_checksum.slice()); |
| 1169 | 1301 | const index_checksum = index_hashed_writer.hasher.finalResult(); |
| 1170 | | try index_writer.writeAll(&index_checksum); |
| 1302 | try index_writer.writeAll(index_checksum.slice()); |
| 1171 | 1303 | } |
| 1172 | 1304 | |
| 1173 | 1305 | /// Performs the first pass over the packfile data for index construction. |
| ... | ... | @@ -1176,13 +1308,14 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1176 | 1308 | /// format). |
| 1177 | 1309 | fn indexPackFirstPass( |
| 1178 | 1310 | allocator: Allocator, |
| 1311 | format: Oid.Format, |
| 1179 | 1312 | pack: std.fs.File, |
| 1180 | 1313 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), |
| 1181 | 1314 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), |
| 1182 | | ) ![Sha1.digest_length]u8 { |
| 1315 | ) !Oid { |
| 1183 | 1316 | var pack_buffered_reader = std.io.bufferedReader(pack.reader()); |
| 1184 | 1317 | var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); |
| 1185 | | var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Sha1.init(.{})); |
| 1318 | var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); |
| 1186 | 1319 | const pack_reader = pack_hashed_reader.reader(); |
| 1187 | 1320 | |
| 1188 | 1321 | const pack_header = try PackHeader.read(pack_reader); |
| ... | ... | @@ -1191,12 +1324,12 @@ fn indexPackFirstPass( |
| 1191 | 1324 | while (current_entry < pack_header.total_objects) : (current_entry += 1) { |
| 1192 | 1325 | const entry_offset = pack_counting_reader.bytes_read; |
| 1193 | 1326 | var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); |
| 1194 | | const entry_header = try EntryHeader.read(entry_crc32_reader.reader()); |
| 1327 | const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); |
| 1195 | 1328 | switch (entry_header) { |
| 1196 | 1329 | .commit, .tree, .blob, .tag => |object| { |
| 1197 | 1330 | var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); |
| 1198 | 1331 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); |
| 1199 | | var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); |
| 1332 | var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); |
| 1200 | 1333 | const entry_writer = entry_hashed_writer.writer(); |
| 1201 | 1334 | // The object header is not included in the pack data but is |
| 1202 | 1335 | // part of the object's ID |
| ... | ... | @@ -1229,8 +1362,8 @@ fn indexPackFirstPass( |
| 1229 | 1362 | } |
| 1230 | 1363 | |
| 1231 | 1364 | const pack_checksum = pack_hashed_reader.hasher.finalResult(); |
| 1232 | | const recorded_checksum = try pack_buffered_reader.reader().readBytesNoEof(Sha1.digest_length); |
| 1233 | | if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) { |
| 1365 | const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); |
| 1366 | if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { |
| 1234 | 1367 | return error.CorruptedPack; |
| 1235 | 1368 | } |
| 1236 | 1369 | _ = pack_reader.readByte() catch |e| switch (e) { |
| ... | ... | @@ -1245,6 +1378,7 @@ fn indexPackFirstPass( |
| 1245 | 1378 | /// delta and we do not yet know the offset of the base object). |
| 1246 | 1379 | fn indexPackHashDelta( |
| 1247 | 1380 | allocator: Allocator, |
| 1381 | format: Oid.Format, |
| 1248 | 1382 | pack: std.fs.File, |
| 1249 | 1383 | delta: IndexEntry, |
| 1250 | 1384 | index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), |
| ... | ... | @@ -1259,7 +1393,7 @@ fn indexPackHashDelta( |
| 1259 | 1393 | if (cache.get(base_offset)) |base_object| break base_object; |
| 1260 | 1394 | |
| 1261 | 1395 | try pack.seekTo(base_offset); |
| 1262 | | base_header = try EntryHeader.read(pack.reader()); |
| 1396 | base_header = try EntryHeader.read(format, pack.reader()); |
| 1263 | 1397 | switch (base_header) { |
| 1264 | 1398 | .ofs_delta => |ofs_delta| { |
| 1265 | 1399 | try delta_offsets.append(allocator, base_offset); |
| ... | ... | @@ -1279,10 +1413,10 @@ fn indexPackHashDelta( |
| 1279 | 1413 | } |
| 1280 | 1414 | }; |
| 1281 | 1415 | |
| 1282 | | const base_data = try resolveDeltaChain(allocator, pack, base_object, delta_offsets.items, cache); |
| 1416 | const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); |
| 1283 | 1417 | |
| 1284 | | var entry_hasher = Sha1.init(.{}); |
| 1285 | | var entry_hashed_writer = hashedWriter(std.io.null_writer, &entry_hasher); |
| 1418 | var entry_hasher: Oid.Hasher = .init(format); |
| 1419 | var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); |
| 1286 | 1420 | try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); |
| 1287 | 1421 | entry_hasher.update(base_data); |
| 1288 | 1422 | return entry_hasher.finalResult(); |
| ... | ... | @@ -1294,6 +1428,7 @@ fn indexPackHashDelta( |
| 1294 | 1428 | /// to obtain the final object. |
| 1295 | 1429 | fn resolveDeltaChain( |
| 1296 | 1430 | allocator: Allocator, |
| 1431 | format: Oid.Format, |
| 1297 | 1432 | pack: std.fs.File, |
| 1298 | 1433 | base_object: Object, |
| 1299 | 1434 | delta_offsets: []const u64, |
| ... | ... | @@ -1306,7 +1441,7 @@ fn resolveDeltaChain( |
| 1306 | 1441 | |
| 1307 | 1442 | const delta_offset = delta_offsets[i]; |
| 1308 | 1443 | try pack.seekTo(delta_offset); |
| 1309 | | const delta_header = try EntryHeader.read(pack.reader()); |
| 1444 | const delta_header = try EntryHeader.read(format, pack.reader()); |
| 1310 | 1445 | const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); |
| 1311 | 1446 | defer allocator.free(delta_data); |
| 1312 | 1447 | var delta_stream = std.io.fixedBufferStream(delta_data); |
| ... | ... | @@ -1394,46 +1529,22 @@ fn expandDelta(base_object: anytype, delta_reader: anytype, writer: anytype) !vo |
| 1394 | 1529 | } |
| 1395 | 1530 | } |
| 1396 | 1531 | |
| 1397 | | fn HashedWriter( |
| 1398 | | comptime WriterType: anytype, |
| 1399 | | comptime HasherType: anytype, |
| 1400 | | ) type { |
| 1401 | | return struct { |
| 1402 | | child_writer: WriterType, |
| 1403 | | hasher: HasherType, |
| 1404 | | |
| 1405 | | const Error = WriterType.Error; |
| 1406 | | const Writer = std.io.Writer(*@This(), Error, write); |
| 1407 | | |
| 1408 | | fn write(hashed_writer: *@This(), buf: []const u8) Error!usize { |
| 1409 | | const amt = try hashed_writer.child_writer.write(buf); |
| 1410 | | hashed_writer.hasher.update(buf); |
| 1411 | | return amt; |
| 1412 | | } |
| 1413 | | |
| 1414 | | fn writer(hashed_writer: *@This()) Writer { |
| 1415 | | return .{ .context = hashed_writer }; |
| 1416 | | } |
| 1417 | | }; |
| 1418 | | } |
| 1419 | | |
| 1420 | | fn hashedWriter( |
| 1421 | | writer: anytype, |
| 1422 | | hasher: anytype, |
| 1423 | | ) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) { |
| 1424 | | return .{ .child_writer = writer, .hasher = hasher }; |
| 1425 | | } |
| 1426 | | |
| 1427 | | test "packfile indexing and checkout" { |
| 1428 | | // To verify the contents of this packfile without using the code in this |
| 1429 | | // file: |
| 1430 | | // |
| 1431 | | // 1. Create a new empty Git repository (`git init`) |
| 1432 | | // 2. `git unpack-objects <path/to/testdata.pack` |
| 1433 | | // 3. `git fsck` -> note the "dangling commit" ID (which matches the commit |
| 1434 | | // checked out below) |
| 1435 | | // 4. `git checkout dd582c0720819ab7130b103635bd7271b9fd4feb` |
| 1436 | | const testrepo_pack = @embedFile("git/testdata/testrepo.pack"); |
| 1532 | /// Runs the packfile indexing and checkout test. |
| 1533 | /// |
| 1534 | /// The two testrepo repositories under testdata contain identical commit |
| 1535 | /// histories and contents. |
| 1536 | /// |
| 1537 | /// To verify the contents of the packfiles using Git alone, run the |
| 1538 | /// following commands in an empty directory: |
| 1539 | /// |
| 1540 | /// 1. `git init --object-format=(sha1|sha256)` |
| 1541 | /// 2. `git unpack-objects <path/to/testrepo.pack` |
| 1542 | /// 3. `git fsck` - will print one "dangling commit": |
| 1543 | /// - SHA-1: `dd582c0720819ab7130b103635bd7271b9fd4feb` |
| 1544 | /// - SHA-256: `7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a` |
| 1545 | /// 4. `git checkout $commit` |
| 1546 | fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void { |
| 1547 | const testrepo_pack = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".pack"); |
| 1437 | 1548 | |
| 1438 | 1549 | var git_dir = testing.tmpDir(.{}); |
| 1439 | 1550 | defer git_dir.cleanup(); |
| ... | ... | @@ -1443,27 +1554,27 @@ test "packfile indexing and checkout" { |
| 1443 | 1554 | |
| 1444 | 1555 | var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); |
| 1445 | 1556 | defer index_file.close(); |
| 1446 | | try indexPack(testing.allocator, pack_file, index_file.writer()); |
| 1557 | try indexPack(testing.allocator, format, pack_file, index_file.writer()); |
| 1447 | 1558 | |
| 1448 | 1559 | // Arbitrary size limit on files read while checking the repository contents |
| 1449 | | // (all files in the test repo are known to be much smaller than this) |
| 1450 | | const max_file_size = 4096; |
| 1560 | // (all files in the test repo are known to be smaller than this) |
| 1561 | const max_file_size = 8192; |
| 1451 | 1562 | |
| 1452 | 1563 | const index_file_data = try git_dir.dir.readFileAlloc(testing.allocator, "testrepo.idx", max_file_size); |
| 1453 | 1564 | defer testing.allocator.free(index_file_data); |
| 1454 | 1565 | // testrepo.idx is generated by Git. The index created by this file should |
| 1455 | 1566 | // match it exactly. Running `git verify-pack -v testrepo.pack` can verify |
| 1456 | 1567 | // this. |
| 1457 | | const testrepo_idx = @embedFile("git/testdata/testrepo.idx"); |
| 1568 | const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); |
| 1458 | 1569 | try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); |
| 1459 | 1570 | |
| 1460 | | var repository = try Repository.init(testing.allocator, pack_file, index_file); |
| 1571 | var repository = try Repository.init(testing.allocator, format, pack_file, index_file); |
| 1461 | 1572 | defer repository.deinit(); |
| 1462 | 1573 | |
| 1463 | 1574 | var worktree = testing.tmpDir(.{ .iterate = true }); |
| 1464 | 1575 | defer worktree.cleanup(); |
| 1465 | 1576 | |
| 1466 | | const commit_id = try parseOid("dd582c0720819ab7130b103635bd7271b9fd4feb"); |
| 1577 | const commit_id = try Oid.parse(format, head_commit); |
| 1467 | 1578 | |
| 1468 | 1579 | var diagnostics: Diagnostics = .{ .allocator = testing.allocator }; |
| 1469 | 1580 | defer diagnostics.deinit(); |
| ... | ... | @@ -1527,6 +1638,14 @@ test "packfile indexing and checkout" { |
| 1527 | 1638 | try testing.expectEqualStrings(expected_file_contents, actual_file_contents); |
| 1528 | 1639 | } |
| 1529 | 1640 | |
| 1641 | test "SHA-1 packfile indexing and checkout" { |
| 1642 | try runRepositoryTest(.sha1, "dd582c0720819ab7130b103635bd7271b9fd4feb"); |
| 1643 | } |
| 1644 | |
| 1645 | test "SHA-256 packfile indexing and checkout" { |
| 1646 | try runRepositoryTest(.sha256, "7f444a92bd4572ee4a28b2c63059924a9ca1829138553ef3e7c41ee159afae7a"); |
| 1647 | } |
| 1648 | |
| 1530 | 1649 | /// Checks out a commit of a packfile. Intended for experimenting with and |
| 1531 | 1650 | /// benchmarking possible optimizations to the indexing and checkout behavior. |
| 1532 | 1651 | pub fn main() !void { |
| ... | ... | @@ -1534,14 +1653,16 @@ pub fn main() !void { |
| 1534 | 1653 | |
| 1535 | 1654 | const args = try std.process.argsAlloc(allocator); |
| 1536 | 1655 | defer std.process.argsFree(allocator, args); |
| 1537 | | if (args.len != 4) { |
| 1538 | | return error.InvalidArguments; // Arguments: packfile commit worktree |
| 1656 | if (args.len != 5) { |
| 1657 | return error.InvalidArguments; // Arguments: format packfile commit worktree |
| 1539 | 1658 | } |
| 1540 | 1659 | |
| 1541 | | var pack_file = try std.fs.cwd().openFile(args[1], .{}); |
| 1660 | const format = std.meta.stringToEnum(Oid.Format, args[1]) orelse return error.InvalidFormat; |
| 1661 | |
| 1662 | var pack_file = try std.fs.cwd().openFile(args[2], .{}); |
| 1542 | 1663 | defer pack_file.close(); |
| 1543 | | const commit = try parseOid(args[2]); |
| 1544 | | var worktree = try std.fs.cwd().makeOpenPath(args[3], .{}); |
| 1664 | const commit = try Oid.parse(format, args[3]); |
| 1665 | var worktree = try std.fs.cwd().makeOpenPath(args[4], .{}); |
| 1545 | 1666 | defer worktree.close(); |
| 1546 | 1667 | |
| 1547 | 1668 | var git_dir = try worktree.makeOpenPath(".git", .{}); |
| ... | ... | @@ -1551,12 +1672,12 @@ pub fn main() !void { |
| 1551 | 1672 | var index_file = try git_dir.createFile("idx", .{ .read = true }); |
| 1552 | 1673 | defer index_file.close(); |
| 1553 | 1674 | var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); |
| 1554 | | try indexPack(allocator, pack_file, index_buffered_writer.writer()); |
| 1675 | try indexPack(allocator, format, pack_file, index_buffered_writer.writer()); |
| 1555 | 1676 | try index_buffered_writer.flush(); |
| 1556 | 1677 | try index_file.sync(); |
| 1557 | 1678 | |
| 1558 | 1679 | std.debug.print("Starting checkout...\n", .{}); |
| 1559 | | var repository = try Repository.init(allocator, pack_file, index_file); |
| 1680 | var repository = try Repository.init(allocator, format, pack_file, index_file); |
| 1560 | 1681 | defer repository.deinit(); |
| 1561 | 1682 | var diagnostics: Diagnostics = .{ .allocator = allocator }; |
| 1562 | 1683 | defer diagnostics.deinit(); |
Andrew Kelley <andrew@ziglang.org> 2024-12-14 16:34:47-05:00
GitHub <noreply@github.com> 2024-12-14 16:34:47-05:00
Signed by PGP key B5690EEEBB952194
src