| ... | ... | @@ -1091,87 +1091,7 @@ pub fn indexPack(allocator: Allocator, pack: std.fs.File, index_writer: anytype) |
| 1091 | 1091 | try index_writer.writeAll(&index_checksum); |
| 1092 | 1092 | } |
| 1093 | 1093 | |
| 1094 | | /// A reader that stores read data in a growable internal buffer. The read |
| 1095 | | /// position can be rewound to allow previously read data to be read again. |
| 1096 | | fn AccumulatingReader(comptime ReaderType: type) type { |
| 1097 | | return struct { |
| 1098 | | child_reader: ReaderType, |
| 1099 | | buffer: std.ArrayListUnmanaged(u8) = .{}, |
| 1100 | | /// The position in `buffer` from which reads should start, returning |
| 1101 | | /// buffered data. If this is `buffer.items.len`, data will be read from |
| 1102 | | /// `child_reader` instead. |
| 1103 | | read_start: usize = 0, |
| 1104 | | allocator: Allocator, |
| 1105 | | |
| 1106 | | const Self = @This(); |
| 1107 | | |
| 1108 | | fn deinit(self: *Self) void { |
| 1109 | | self.buffer.deinit(self.allocator); |
| 1110 | | self.* = undefined; |
| 1111 | | } |
| 1112 | | |
| 1113 | | const ReadError = ReaderType.Error || Allocator.Error; |
| 1114 | | const Reader = std.io.Reader(*Self, ReadError, read); |
| 1115 | | |
| 1116 | | fn read(self: *Self, buf: []u8) ReadError!usize { |
| 1117 | | if (self.read_start < self.buffer.items.len) { |
| 1118 | | // Previously buffered data is available and should be used |
| 1119 | | // before reading more data from the underlying reader. |
| 1120 | | const available = self.buffer.items.len - self.read_start; |
| 1121 | | const count = @min(available, buf.len); |
| 1122 | | @memcpy(buf[0..count], self.buffer.items[self.read_start..][0..count]); |
| 1123 | | self.read_start += count; |
| 1124 | | return count; |
| 1125 | | } |
| 1126 | | |
| 1127 | | try self.buffer.ensureUnusedCapacity(self.allocator, buf.len); |
| 1128 | | const read_buffer = self.buffer.unusedCapacitySlice(); |
| 1129 | | const count = try self.child_reader.read(read_buffer[0..buf.len]); |
| 1130 | | @memcpy(buf[0..count], read_buffer[0..count]); |
| 1131 | | self.buffer.items.len += count; |
| 1132 | | self.read_start += count; |
| 1133 | | return count; |
| 1134 | | } |
| 1135 | | |
| 1136 | | fn reader(self: *Self) Reader { |
| 1137 | | return .{ .context = self }; |
| 1138 | | } |
| 1139 | | |
| 1140 | | /// Returns a slice of the buffered data that has already been read, |
| 1141 | | /// except the last `count_before_end` bytes. |
| 1142 | | fn readDataExcept(self: Self, count_before_end: usize) []const u8 { |
| 1143 | | assert(count_before_end <= self.read_start); |
| 1144 | | return self.buffer.items[0 .. self.read_start - count_before_end]; |
| 1145 | | } |
| 1146 | | |
| 1147 | | /// Discards the first `count` bytes of buffered data. |
| 1148 | | fn discard(self: *Self, count: usize) void { |
| 1149 | | assert(count <= self.buffer.items.len); |
| 1150 | | const retain = self.buffer.items.len - count; |
| 1151 | | mem.copyForwards( |
| 1152 | | u8, |
| 1153 | | self.buffer.items[0..retain], |
| 1154 | | self.buffer.items[count..][0..retain], |
| 1155 | | ); |
| 1156 | | self.buffer.items.len = retain; |
| 1157 | | self.read_start -= @min(self.read_start, count); |
| 1158 | | } |
| 1159 | | |
| 1160 | | /// Rewinds the read position to the beginning of buffered data. |
| 1161 | | fn rewind(self: *Self) void { |
| 1162 | | self.read_start = 0; |
| 1163 | | } |
| 1164 | | }; |
| 1165 | | } |
| 1166 | | |
| 1167 | | fn accumulatingReader( |
| 1168 | | allocator: Allocator, |
| 1169 | | reader: anytype, |
| 1170 | | ) AccumulatingReader(@TypeOf(reader)) { |
| 1171 | | return .{ .child_reader = reader, .allocator = allocator }; |
| 1172 | | } |
| 1173 | | |
| 1174 | | /// Performs the first pass over the packfile data for index construction. |
| 1094 | // Performs the first pass over the packfile data for index construction. |
| 1175 | 1095 | /// This will index all non-delta objects, queue delta objects for further |
| 1176 | 1096 | /// processing, and return the pack checksum (which is part of the index |
| 1177 | 1097 | /// format). |
| ... | ... | @@ -1181,102 +1101,62 @@ fn indexPackFirstPass( |
| 1181 | 1101 | index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), |
| 1182 | 1102 | pending_deltas: *std.ArrayListUnmanaged(IndexEntry), |
| 1183 | 1103 | ) ![Sha1.digest_length]u8 { |
| 1184 | | var pack_buffered_reader = std.io.bufferedReader(pack.reader()); |
| 1185 | | var pack_accumulating_reader = accumulatingReader(allocator, pack_buffered_reader.reader()); |
| 1186 | | defer pack_accumulating_reader.deinit(); |
| 1187 | | var pack_position: usize = 0; |
| 1188 | | var pack_hash = Sha1.init(.{}); |
| 1189 | | const pack_reader = pack_accumulating_reader.reader(); |
| 1104 | var pack_counting_writer = std.io.countingWriter(std.io.null_writer); |
| 1105 | var pack_hashed_writer = std.compress.hashedWriter(pack_counting_writer.writer(), Sha1.init(.{})); |
| 1106 | var entry_crc32_writer = std.compress.hashedWriter(pack_hashed_writer.writer(), std.hash.Crc32.init()); |
| 1107 | var pack_buffered_reader = std.io.bufferedTee(4096, 8, pack.reader(), entry_crc32_writer.writer()); |
| 1108 | const pack_reader = pack_buffered_reader.reader(); |
| 1190 | 1109 | |
| 1191 | 1110 | const pack_header = try PackHeader.read(pack_reader); |
| 1192 | | const pack_header_bytes = pack_accumulating_reader.readDataExcept(0); |
| 1193 | | pack_position += pack_header_bytes.len; |
| 1194 | | pack_hash.update(pack_header_bytes); |
| 1195 | | pack_accumulating_reader.discard(pack_header_bytes.len); |
| 1111 | try pack_buffered_reader.flush(); |
| 1196 | 1112 | |
| 1197 | 1113 | var current_entry: u32 = 0; |
| 1198 | 1114 | while (current_entry < pack_header.total_objects) : (current_entry += 1) { |
| 1199 | | const entry_offset = pack_position; |
| 1200 | | var entry_crc32 = std.hash.Crc32.init(); |
| 1201 | | |
| 1115 | const entry_offset = pack_counting_writer.bytes_written; |
| 1116 | entry_crc32_writer.hasher = std.hash.Crc32.init(); // reset hasher |
| 1202 | 1117 | const entry_header = try EntryHeader.read(pack_reader); |
| 1203 | | const entry_header_bytes = pack_accumulating_reader.readDataExcept(0); |
| 1204 | | pack_position += entry_header_bytes.len; |
| 1205 | | pack_hash.update(entry_header_bytes); |
| 1206 | | entry_crc32.update(entry_header_bytes); |
| 1207 | | pack_accumulating_reader.discard(entry_header_bytes.len); |
| 1208 | 1118 | |
| 1209 | 1119 | switch (entry_header) { |
| 1210 | | .commit, .tree, .blob, .tag => |object| { |
| 1120 | inline .commit, .tree, .blob, .tag => |object, tag| { |
| 1211 | 1121 | var entry_decompress_stream = std.compress.zlib.decompressor(pack_reader); |
| 1212 | | var entry_data_size: usize = 0; |
| 1122 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); |
| 1213 | 1123 | var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{})); |
| 1214 | 1124 | const entry_writer = entry_hashed_writer.writer(); |
| 1215 | | |
| 1216 | 1125 | // The object header is not included in the pack data but is |
| 1217 | 1126 | // part of the object's ID |
| 1218 | | try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); |
| 1219 | | |
| 1220 | | while (try entry_decompress_stream.next()) |decompressed_data| { |
| 1221 | | entry_data_size += decompressed_data.len; |
| 1222 | | try entry_writer.writeAll(decompressed_data); |
| 1223 | | |
| 1224 | | const compressed_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes()); |
| 1225 | | pack_position += compressed_bytes.len; |
| 1226 | | pack_hash.update(compressed_bytes); |
| 1227 | | entry_crc32.update(compressed_bytes); |
| 1228 | | pack_accumulating_reader.discard(compressed_bytes.len); |
| 1229 | | } |
| 1230 | | const footer_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes()); |
| 1231 | | pack_position += footer_bytes.len; |
| 1232 | | pack_hash.update(footer_bytes); |
| 1233 | | entry_crc32.update(footer_bytes); |
| 1234 | | pack_accumulating_reader.discard(footer_bytes.len); |
| 1235 | | pack_accumulating_reader.rewind(); |
| 1236 | | |
| 1237 | | if (entry_data_size != object.uncompressed_length) { |
| 1127 | try entry_writer.print("{s} {}\x00", .{ @tagName(tag), object.uncompressed_length }); |
| 1128 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); |
| 1129 | try fifo.pump(entry_counting_reader.reader(), entry_writer); |
| 1130 | if (entry_counting_reader.bytes_read != object.uncompressed_length) { |
| 1238 | 1131 | return error.InvalidObject; |
| 1239 | 1132 | } |
| 1240 | | |
| 1241 | 1133 | const oid = entry_hashed_writer.hasher.finalResult(); |
| 1134 | pack_buffered_reader.putBack(entry_decompress_stream.unreadBytes()); |
| 1135 | try pack_buffered_reader.flush(); |
| 1242 | 1136 | try index_entries.put(allocator, oid, .{ |
| 1243 | 1137 | .offset = entry_offset, |
| 1244 | | .crc32 = entry_crc32.final(), |
| 1138 | .crc32 = entry_crc32_writer.hasher.final(), |
| 1245 | 1139 | }); |
| 1246 | 1140 | }, |
| 1247 | 1141 | inline .ofs_delta, .ref_delta => |delta| { |
| 1248 | 1142 | var entry_decompress_stream = std.compress.zlib.decompressor(pack_reader); |
| 1249 | | var entry_data_size: usize = 0; |
| 1250 | | |
| 1251 | | while (try entry_decompress_stream.next()) |decompressed_data| { |
| 1252 | | entry_data_size += decompressed_data.len; |
| 1253 | | |
| 1254 | | const compressed_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes()); |
| 1255 | | pack_position += compressed_bytes.len; |
| 1256 | | pack_hash.update(compressed_bytes); |
| 1257 | | entry_crc32.update(compressed_bytes); |
| 1258 | | pack_accumulating_reader.discard(compressed_bytes.len); |
| 1259 | | } |
| 1260 | | const footer_bytes = pack_accumulating_reader.readDataExcept(entry_decompress_stream.unreadBytes()); |
| 1261 | | pack_position += footer_bytes.len; |
| 1262 | | pack_hash.update(footer_bytes); |
| 1263 | | entry_crc32.update(footer_bytes); |
| 1264 | | pack_accumulating_reader.discard(footer_bytes.len); |
| 1265 | | pack_accumulating_reader.rewind(); |
| 1266 | | |
| 1267 | | if (entry_data_size != delta.uncompressed_length) { |
| 1143 | var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); |
| 1144 | var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); |
| 1145 | try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); |
| 1146 | if (entry_counting_reader.bytes_read != delta.uncompressed_length) { |
| 1268 | 1147 | return error.InvalidObject; |
| 1269 | 1148 | } |
| 1270 | | |
| 1149 | pack_buffered_reader.putBack(entry_decompress_stream.unreadBytes()); |
| 1150 | try pack_buffered_reader.flush(); |
| 1271 | 1151 | try pending_deltas.append(allocator, .{ |
| 1272 | 1152 | .offset = entry_offset, |
| 1273 | | .crc32 = entry_crc32.final(), |
| 1153 | .crc32 = entry_crc32_writer.hasher.final(), |
| 1274 | 1154 | }); |
| 1275 | 1155 | }, |
| 1276 | 1156 | } |
| 1277 | 1157 | } |
| 1278 | 1158 | |
| 1279 | | const pack_checksum = pack_hash.finalResult(); |
| 1159 | const pack_checksum = pack_hashed_writer.hasher.finalResult(); |
| 1280 | 1160 | const recorded_checksum = try pack_reader.readBytesNoEof(Sha1.digest_length); |
| 1281 | 1161 | if (!mem.eql(u8, &pack_checksum, &recorded_checksum)) { |
| 1282 | 1162 | return error.CorruptedPack; |
Igor Anić <igor.anic@gmail.com> 2024-02-22 15:18:03+01:00
src