| ... | @@ -183,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel | ... | @@ -183,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel |
| 183 | .fallocate_insert_range_unsupported = false, | 183 | .fallocate_insert_range_unsupported = false, |
| 184 | .fallocate_punch_hole_unsupported = false, | 184 | .fallocate_punch_hole_unsupported = false, |
| 185 | }; | 185 | }; |
| 186 | try mf.nodes.ensureUnusedCapacity(gpa, 1); | 186 | |
| 187 | const root_ni = try mf.addNode(gpa, .{ .add_node = .{ | 187 | const root_location: Node.Location = l: { |
| 188 | .size = size, | 188 | if (std.math.cast(u32, size)) |small_size| { |
| 189 | .alignment = mf.flags.block_size, | 189 | break :l .{ .small = .{ .offset = 0, .size = small_size } }; |
| 190 | .fixed = true, | 190 | } |
| 191 | } }); | 191 | try mf.large.appendSlice(gpa, &.{ 0, size }); |
| 192 | assert(root_ni == .root); | 192 | break :l .{ .large = .{ .index = 0 } }; |
| 193 | try mf.ensureTotalCapacityInner(@intCast(size)); | 193 | }; |
| | 194 | try mf.nodes.append(gpa, .{ |
| | 195 | .parent = .none, |
| | 196 | .prev = .none, |
| | 197 | .next = .none, |
| | 198 | .first = .none, |
| | 199 | .last = .none, |
| | 200 | .flags = .{ |
| | 201 | .alignment = mf.flags.block_size, |
| | 202 | .position = .floating, |
| | 203 | .bubbles_moved = true, |
| | 204 | .enable_next_moved = false, |
| | 205 | .location_tag = root_location, |
| | 206 | .moved = false, |
| | 207 | .resized = false, |
| | 208 | .next_moved = false, |
| | 209 | .has_content = false, |
| | 210 | }, |
| | 211 | .location_payload = switch (root_location) { |
| | 212 | .small => |small| .{ .small = small }, |
| | 213 | .large => |large| .{ .large = large }, |
| | 214 | }, |
| | 215 | }); |
| | 216 | |
| | 217 | mf.ensureTotalCapacity(@intCast(size)) catch |err| switch (err) { |
| | 218 | error.MappedFileIo => return mf.io_err.?, |
| | 219 | else => |e| return e, |
| | 220 | }; |
| | 221 | |
| 194 | return mf; | 222 | return mf; |
| 195 | } | 223 | } |
| 196 | | 224 | |
| ... | @@ -213,24 +241,53 @@ pub const Node = extern struct { | ... | @@ -213,24 +241,53 @@ pub const Node = extern struct { |
| 213 | flags: Flags, | 241 | flags: Flags, |
| 214 | location_payload: Location.Payload, | 242 | location_payload: Location.Payload, |
| 215 | | 243 | |
| | 244 | /// Any non-leaf node may designate its first N children as "header" nodes. This means that its |
| | 245 | /// first N children must be densely packed together and positioned at the start of the parent. |
| | 246 | /// The implementation guarantees that it will never re-order these nodes, nor will it introduce |
| | 247 | /// padding between them. |
| | 248 | /// |
| | 249 | /// Likewise, any non-leaf node may designate its *last* M children as "footer" nodes, which are |
| | 250 | /// like header nodes except they are positioned at the *end* of the parent rather than the |
| | 251 | /// start. |
| | 252 | /// |
| | 253 | /// Nodes which are neither headers nor footers are called "floating". The implementation is |
| | 254 | /// always free to re-order floating nodes relative to one another, and to add or remove padding |
| | 255 | /// between them. |
| | 256 | pub const Position = enum(u2) { |
| | 257 | header, |
| | 258 | footer, |
| | 259 | floating, |
| | 260 | }; |
| | 261 | |
| 216 | pub const Flags = packed struct(u32) { | 262 | pub const Flags = packed struct(u32) { |
| 217 | location_tag: Location.Tag, | 263 | /// While the number of header and footer nodes within a parent node is logically a part of |
| | 264 | /// that parent, we actually store this information on the child nodes for efficiency: this |
| | 265 | /// field indicates whether each child is a header node, a footer node, or a floating node. |
| | 266 | /// |
| | 267 | /// This value is meaningless for the root node, so is arbitrarily set to `.floating`. |
| | 268 | position: Position, |
| | 269 | /// For floating nodes, this node's offset into its parent will always be aligned to this |
| | 270 | /// boundary. (This is not the case for header and footer nodes due to the requirement that |
| | 271 | /// they be densely packed against the start/end of the parent node.) |
| | 272 | /// |
| | 273 | /// This node's size will also always be aligned to this boundary. (This applies regardless |
| | 274 | /// of whether this is a floating node, a header node, or a footer node.) |
| 218 | alignment: Alignment, | 275 | alignment: Alignment, |
| 219 | /// Whether this node can be moved. | 276 | /// Whether `moved` events on this node bubble down to children. |
| 220 | fixed: bool, | 277 | bubbles_moved: bool, |
| | 278 | /// Whether `next_moved` events are reported in `updates`. |
| | 279 | enable_next_moved: bool, |
| | 280 | |
| | 281 | location_tag: Location.Tag, |
| 221 | /// Whether this node has been moved. | 282 | /// Whether this node has been moved. |
| 222 | moved: bool, | 283 | moved: bool, |
| 223 | /// Whether this node has been resized. | 284 | /// Whether this node has been resized. |
| 224 | resized: bool, | 285 | resized: bool, |
| 225 | /// Whether the next sibling has moved or is a different node. | 286 | /// Whether the next sibling has moved or is a different node. |
| 226 | next_moved: bool, | 287 | next_moved: bool, |
| 227 | /// Whether this node might contain non-zero bytes. | 288 | /// Whether this node might contain initialized bytes. |
| 228 | has_content: bool, | 289 | has_content: bool, |
| 229 | /// Whether `moved` events on this node bubble down to children. | 290 | unused: u17 = 0, |
| 230 | bubbles_moved: bool, | | |
| 231 | /// Whether `next_moved` events are reported in `updates`. | | |
| 232 | enable_next_moved: bool, | | |
| 233 | unused: u18 = 0, | | |
| 234 | }; | 291 | }; |
| 235 | | 292 | |
| 236 | pub const Location = union(enum(u1)) { | 293 | pub const Location = union(enum(u1)) { |
| ... | @@ -267,6 +324,18 @@ pub const Node = extern struct { | ... | @@ -267,6 +324,18 @@ pub const Node = extern struct { |
| 267 | } | 324 | } |
| 268 | }; | 325 | }; |
| 269 | | 326 | |
| | 327 | pub const AddOptions = struct { |
| | 328 | /// Must be aligned to the given `alignment`. |
| | 329 | size: u64 = 0, |
| | 330 | alignment: Alignment = .@"1", |
| | 331 | bubbles_moved: bool = true, |
| | 332 | enable_next_moved: bool = false, |
| | 333 | |
| | 334 | moved: bool = false, |
| | 335 | resized: bool = false, |
| | 336 | next_moved: bool = false, |
| | 337 | }; |
| | 338 | |
| 270 | pub const Index = enum(u32) { | 339 | pub const Index = enum(u32) { |
| 271 | root, | 340 | root, |
| 272 | _, | 341 | _, |
| ... | @@ -292,6 +361,70 @@ pub const Node = extern struct { | ... | @@ -292,6 +361,70 @@ pub const Node = extern struct { |
| 292 | return &mf.nodes.items[@backingInt(ni)]; | 361 | return &mf.nodes.items[@backingInt(ni)]; |
| 293 | } | 362 | } |
| 294 | | 363 | |
| | 364 | /// Adds a floating child node to `parent_ni`. Returns the index of the new child. |
| | 365 | pub fn addFloatingChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { |
| | 366 | return mf.addNode(gpa, .{ |
| | 367 | .add_options = opts, |
| | 368 | .position = .floating, |
| | 369 | .parent = parent_ni, |
| | 370 | .prev = parent_ni.lastHeader(mf), |
| | 371 | }); |
| | 372 | } |
| | 373 | /// Adds a header child node to `parent_ni`. Returns the index of the new child. |
| | 374 | /// |
| | 375 | /// Asserts that `parent_ni` has no existing header children. |
| | 376 | pub fn addOnlyHeaderChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { |
| | 377 | if (parent_ni.first(mf).unwrap()) |first_ni| { |
| | 378 | assert(first_ni.position(mf) != .header); // `parent_ni` already has a header child |
| | 379 | } |
| | 380 | return parent_ni.addHeaderChildAfter(mf, gpa, .none, opts); |
| | 381 | } |
| | 382 | /// Adds a header child node to `parent_ni`. Returns the index of the new child. |
| | 383 | /// |
| | 384 | /// If `prev_oni` is `.none`, the new child is placed at the very start of the parent, |
| | 385 | /// before any existing header nodes. |
| | 386 | /// |
| | 387 | /// Otherwise, asserts that `prev_oni` is a header node and a child of `parent_ni`, and |
| | 388 | /// places the new child node immediately after `prev_oni`. |
| | 389 | pub fn addHeaderChildAfter(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, prev_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { |
| | 390 | return mf.addNode(gpa, .{ |
| | 391 | .add_options = opts, |
| | 392 | .position = .header, |
| | 393 | .parent = parent_ni, |
| | 394 | .prev = prev_oni, |
| | 395 | }); |
| | 396 | } |
| | 397 | /// Adds a footer child node to `parent_ni`. Returns the index of the new child. |
| | 398 | /// |
| | 399 | /// Asserts that `parent_ni` has no existing footer children. |
| | 400 | pub fn addOnlyFooterChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { |
| | 401 | if (parent_ni.last(mf).unwrap()) |last_ni| { |
| | 402 | assert(last_ni.position(mf) != .footer); // `parent_ni` already has a footer child |
| | 403 | } |
| | 404 | return parent_ni.addFooterChildBefore(mf, gpa, .none, opts); |
| | 405 | } |
| | 406 | /// Adds a footer child node to `parent_ni`. Returns the index of the new child. |
| | 407 | /// |
| | 408 | /// If `next_oni` is `.none`, the new child is placed at the very end of the parent, after |
| | 409 | /// any existing footer nodes. |
| | 410 | /// |
| | 411 | /// Otherwise, asserts that `next_oni` is a footer node and a child of `parent_ni`, and |
| | 412 | /// places the new child node immediately before `next_oni`. |
| | 413 | pub fn addFooterChildBefore(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, next_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { |
| | 414 | const prev_oni: Node.Index.Optional = prev: { |
| | 415 | const next_ni = next_oni.unwrap() orelse { |
| | 416 | break :prev parent_ni.last(mf); |
| | 417 | }; |
| | 418 | break :prev next_ni.prev(mf); |
| | 419 | }; |
| | 420 | return mf.addNode(gpa, .{ |
| | 421 | .add_options = opts, |
| | 422 | .position = .footer, |
| | 423 | .parent = parent_ni, |
| | 424 | .prev = prev_oni, |
| | 425 | }); |
| | 426 | } |
| | 427 | |
| 295 | /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. | 428 | /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. |
| 296 | pub const toOptional = Optional.wrap; | 429 | pub const toOptional = Optional.wrap; |
| 297 | | 430 | |
| ... | @@ -299,19 +432,54 @@ pub const Node = extern struct { | ... | @@ -299,19 +432,54 @@ pub const Node = extern struct { |
| 299 | return ni.get(mf).parent; | 432 | return ni.get(mf).parent; |
| 300 | } | 433 | } |
| 301 | | 434 | |
| | 435 | pub fn first(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| | 436 | return ni.get(mf).first; |
| | 437 | } |
| | 438 | |
| | 439 | pub fn last(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| | 440 | return ni.get(mf).last; |
| | 441 | } |
| | 442 | |
| | 443 | fn lastHeader(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| | 444 | var header_ni = ni.first(mf).unwrap() orelse return .none; |
| | 445 | if (header_ni.position(mf) != .header) return .none; |
| | 446 | while (true) { |
| | 447 | const next_ni = header_ni.next(mf).unwrap() orelse break; |
| | 448 | if (next_ni.position(mf) != .header) break; |
| | 449 | header_ni = next_ni; |
| | 450 | } |
| | 451 | return .wrap(header_ni); |
| | 452 | } |
| | 453 | fn firstFooter(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| | 454 | var footer_ni = ni.last(mf).unwrap() orelse return .none; |
| | 455 | if (footer_ni.position(mf) != .footer) return .none; |
| | 456 | while (true) { |
| | 457 | const prev_ni = footer_ni.prev(mf).unwrap() orelse break; |
| | 458 | if (prev_ni.position(mf) != .footer) break; |
| | 459 | footer_ni = prev_ni; |
| | 460 | } |
| | 461 | return .wrap(footer_ni); |
| | 462 | } |
| | 463 | |
| | 464 | /// Asserts that `ni` is not `.root`, because `Position` is meaningless for the root node. |
| | 465 | pub fn position(ni: Node.Index, mf: *const MappedFile) Node.Position { |
| | 466 | assert(ni != .root); |
| | 467 | return ni.get(mf).flags.position; |
| | 468 | } |
| | 469 | |
| 302 | pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { | 470 | pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| 303 | return ni.get(mf).next; | 471 | return ni.get(mf).next; |
| 304 | } | 472 | } |
| 305 | fn setNext( | 473 | fn setNext( |
| 306 | prev_ni: Node.Index, | 474 | ni: Node.Index, |
| 307 | gpa: Allocator, | 475 | gpa: Allocator, |
| 308 | next_ni: Node.Index.Optional, | 476 | next_ni: Node.Index.Optional, |
| 309 | mf: *MappedFile, | 477 | mf: *MappedFile, |
| 310 | ) Allocator.Error!void { | 478 | ) Allocator.Error!void { |
| 311 | const prev_next = &prev_ni.get(mf).next; | 479 | const next_ptr = &ni.get(mf).next; |
| 312 | if (prev_next.* == next_ni) return; | 480 | if (next_ptr.* == next_ni) return; |
| 313 | prev_next.* = next_ni; | 481 | next_ptr.* = next_ni; |
| 314 | try prev_ni.nextMoved(gpa, mf); | 482 | try ni.nextMoved(gpa, mf); |
| 315 | } | 483 | } |
| 316 | | 484 | |
| 317 | pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { | 485 | pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| ... | @@ -421,8 +589,14 @@ pub const Node = extern struct { | ... | @@ -421,8 +589,14 @@ pub const Node = extern struct { |
| 421 | return ni.get(mf).flags.alignment; | 589 | return ni.get(mf).flags.alignment; |
| 422 | } | 590 | } |
| 423 | | 591 | |
| 424 | fn setLocationAssumeCapacity(ni: Node.Index, mf: *MappedFile, offset: u64, size: u64) void { | 592 | fn setLocation(ni: Node.Index, mf: *MappedFile, gpa: Allocator, offset: u64, size: u64) Allocator.Error!void { |
| | 593 | try mf.large.ensureUnusedCapacity(gpa, 2); |
| | 594 | try mf.updates.ensureUnusedCapacity(gpa, 2); |
| 425 | const node = ni.get(mf); | 595 | const node = ni.get(mf); |
| | 596 | if (node.flags.position == .floating) { |
| | 597 | assert(node.flags.alignment.check(offset)); |
| | 598 | } |
| | 599 | assert(node.flags.alignment.check(size)); |
| 426 | if (size == 0) node.flags.has_content = false; | 600 | if (size == 0) node.flags.has_content = false; |
| 427 | switch (node.location()) { | 601 | switch (node.location()) { |
| 428 | .small => |small| { | 602 | .small => |small| { |
| ... | @@ -485,62 +659,46 @@ pub const Node = extern struct { | ... | @@ -485,62 +659,46 @@ pub const Node = extern struct { |
| 485 | return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; | 659 | return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; |
| 486 | } | 660 | } |
| 487 | | 661 | |
| 488 | pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { | 662 | /// Ensures that the size of `ni` is at least `min_size`. Valid for any node. |
| 489 | mf.resizeNode(gpa, ni, size) catch |err| switch (err) { | 663 | /// |
| 490 | error.OutOfMemory, | 664 | /// Applies `growth_factor` if necessary (so the caller should *not* apply `growth_factor`). |
| 491 | error.Canceled, | 665 | pub fn ensureMinimumSize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, min_size: u64) Error!void { |
| 492 | => |e| return e, | 666 | _, const current_size = ni.location(mf).resolve(mf); |
| 493 | else => |e| { | 667 | if (current_size >= min_size) return; |
| 494 | mf.io_err = e; | 668 | const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); |
| 495 | return error.MappedFileIo; | 669 | try mf.growNode(gpa, ni, new_size, .minimum); |
| 496 | }, | 670 | mf.updateWriters(); |
| 497 | }; | | |
| 498 | var writers_it = mf.writers.first; | | |
| 499 | while (writers_it) |writer_node| : (writers_it = writer_node.next) { | | |
| 500 | const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); | | |
| 501 | w.interface.buffer = w.ni.slice(mf); | | |
| 502 | } | | |
| 503 | } | 671 | } |
| 504 | | 672 | |
| 505 | pub const RealignNodeOptions = struct { | 673 | /// Sets the size of `ni` to exactly `size`. |
| 506 | /// Shift the node backwards if possible | 674 | /// |
| 507 | try_backwards: bool = false, | 675 | /// Asserts that `ni` is a leaf node, i.e. has no children. |
| 508 | }; | 676 | /// |
| 509 | | 677 | /// Asserts that `size` is aligned to `ni.alignment(mf)`. |
| 510 | /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. | 678 | pub fn resizeLeaf(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { |
| 511 | /// Asserts that `ni` is not `.root`. | 679 | assert(ni.first(mf) == .none); |
| 512 | pub fn realign( | 680 | // The alignment of `size` is asserted by `shrinkLeafNode` and `growNode`. |
| 513 | ni: Node.Index, | 681 | _, const old_size = ni.location(mf).resolve(mf); |
| 514 | mf: *MappedFile, | 682 | switch (std.math.order(size, old_size)) { |
| 515 | gpa: Allocator, | 683 | .lt => try mf.shrinkLeafNode(gpa, ni, size), |
| 516 | new_alignment: Alignment, | 684 | .eq => {}, // `old_size` must be well-aligned, so `size` is too |
| 517 | opts: RealignNodeOptions, | 685 | .gt => try mf.growNode(gpa, ni, size, .exact), |
| 518 | ) Error!void { | 686 | } |
| 519 | mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { | | |
| 520 | error.OutOfMemory, | | |
| 521 | error.Canceled, | | |
| 522 | => |e| return e, | | |
| 523 | else => |e| { | | |
| 524 | mf.io_err = e; | | |
| 525 | return error.MappedFileIo; | | |
| 526 | }, | | |
| 527 | }; | | |
| 528 | mf.updateWriters(); | 687 | mf.updateWriters(); |
| 529 | } | 688 | } |
| 530 | | 689 | |
| 531 | /// Shrink a node to `size`, exactly. | 690 | /// Updates a node's alignment to exactly `new_alignment`. Valid for any node. |
| 532 | /// Asserts that the new size can contain all the children. | 691 | /// |
| 533 | /// If `shift_next` is set, then the following node is shifted backwards into | 692 | /// If the node's current offset or size is not sufficiently aligned, it will be moved |
| 534 | /// the free space as much as alignment allows. | 693 | /// and/or resized to match the new alignment. The node's size may be increased by any |
| 535 | /// Asserts that `size` is >= the end of the last child node. | 694 | /// amount, as if `ensureMinimumSize` were used. |
| 536 | pub fn shrink( | 695 | pub fn realign( |
| 537 | ni: Node.Index, | 696 | ni: Node.Index, |
| 538 | mf: *MappedFile, | 697 | mf: *MappedFile, |
| 539 | gpa: Allocator, | 698 | gpa: Allocator, |
| 540 | size: u64, | 699 | new_alignment: Alignment, |
| 541 | shift_next: bool, | | |
| 542 | ) Error!void { | 700 | ) Error!void { |
| 543 | try mf.shrinkNode(gpa, ni, size, shift_next); | 701 | try mf.realignNode(gpa, ni, new_alignment); |
| 544 | mf.updateWriters(); | 702 | mf.updateWriters(); |
| 545 | } | 703 | } |
| 546 | | 704 | |
| ... | @@ -644,16 +802,9 @@ pub const Node = extern struct { | ... | @@ -644,16 +802,9 @@ pub const Node = extern struct { |
| 644 | file_reader.pos, | 802 | file_reader.pos, |
| 645 | w.ni.fileLocation(w.mf, true).offset + interface.end, | 803 | w.ni.fileLocation(w.mf, true).offset + interface.end, |
| 646 | limit.minInt(interface.unusedCapacityLen()), | 804 | limit.minInt(interface.unusedCapacityLen()), |
| 647 | ) catch |err| switch (err) { | 805 | ) catch |err| { |
| 648 | error.Canceled => |e| { | 806 | w.err = err; |
| 649 | w.err = e; | 807 | return error.WriteFailed; |
| 650 | return error.WriteFailed; | | |
| 651 | }, | | |
| 652 | else => |e| { | | |
| 653 | w.mf.io_err = e; | | |
| 654 | w.err = error.MappedFileIo; | | |
| 655 | return error.WriteFailed; | | |
| 656 | }, | | |
| 657 | }); | 808 | }); |
| 658 | if (n == 0) return error.Unimplemented; | 809 | if (n == 0) return error.Unimplemented; |
| 659 | file_reader.pos += n; | 810 | file_reader.pos += n; |
| ... | @@ -680,10 +831,8 @@ pub const Node = extern struct { | ... | @@ -680,10 +831,8 @@ pub const Node = extern struct { |
| 680 | unused_capacity: usize, | 831 | unused_capacity: usize, |
| 681 | ) Io.Writer.Error!void { | 832 | ) Io.Writer.Error!void { |
| 682 | _ = preserve; | 833 | _ = preserve; |
| 683 | const total_capacity = interface.end + unused_capacity; | | |
| 684 | if (interface.buffer.len >= total_capacity) return; | | |
| 685 | const w: *Writer = @fieldParentPtr("interface", interface); | 834 | const w: *Writer = @fieldParentPtr("interface", interface); |
| 686 | w.ni.resize(w.mf, w.gpa, total_capacity +| total_capacity / growth_factor) catch |err| { | 835 | w.ni.ensureMinimumSize(w.mf, w.gpa, interface.end + unused_capacity) catch |err| { |
| 687 | w.err = err; | 836 | w.err = err; |
| 688 | return error.WriteFailed; | 837 | return error.WriteFailed; |
| 689 | }; | 838 | }; |
| ... | @@ -691,624 +840,1267 @@ pub const Node = extern struct { | ... | @@ -691,624 +840,1267 @@ pub const Node = extern struct { |
| 691 | }; | 840 | }; |
| 692 | | 841 | |
| 693 | comptime { | 842 | comptime { |
| 694 | if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 32); | 843 | if (!std.debug.runtime_safety) assert(@sizeOf(Node) == 32); |
| 695 | } | 844 | } |
| 696 | }; | 845 | }; |
| 697 | | 846 | |
| | 847 | /// Asserts that `opts.position` is compatible with `opts.prev` (i.e. that this addition will not |
| | 848 | /// violate the requirement that header nodes come before floating nodes come before footer nodes). |
| 698 | fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { | 849 | fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { |
| 699 | parent: Node.Index.Optional = .none, | 850 | add_options: Node.AddOptions, |
| 700 | prev: Node.Index.Optional = .none, | 851 | position: Node.Position, |
| 701 | next: Node.Index.Optional = .none, | 852 | parent: Node.Index, |
| 702 | offset: u64 = 0, | 853 | /// If `position == .floating`, this is just used as an initial value, and may be immediately |
| 703 | add_node: AddNodeOptions, | 854 | /// replaced when finding a location for this node. In this case, it is still necessary that |
| 704 | }) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { | 855 | /// `prev` be compatible with `position` (so `prev` must be either a floating node or the last |
| | 856 | /// header node in `parent`). |
| | 857 | prev: Node.Index.Optional, |
| | 858 | }) Error!Node.Index { |
| 705 | mf.nodes_lock.assertUnlocked(); | 859 | mf.nodes_lock.assertUnlocked(); |
| 706 | const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: { | | |
| 707 | if (std.math.cast(u32, opts.offset)) |small_offset| break :location .{ .small, .{ | | |
| 708 | .small = .{ .offset = small_offset, .size = 0 }, | | |
| 709 | } }; | | |
| 710 | try mf.large.ensureUnusedCapacity(gpa, 2); | | |
| 711 | defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); | | |
| 712 | break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; | | |
| 713 | }; | | |
| 714 | | 860 | |
| 715 | const free_ni: Node.Index, const free_node: *Node = if (mf.free_ni.unwrap()) |free_ni| free: { | 861 | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 716 | const free_node = free_ni.get(mf); | 862 | try mf.large.ensureUnusedCapacity(gpa, 2); |
| 717 | mf.free_ni = free_node.next; | 863 | |
| 718 | break :free .{ free_ni, free_node }; | 864 | const new_ni: Node.Index = new: { |
| 719 | } else .{ | 865 | if (mf.free_ni.unwrap()) |free_ni| { |
| 720 | @fromBackingInt(@intCast(mf.nodes.items.len)), | 866 | mf.free_ni = free_ni.get(mf).next; |
| 721 | mf.nodes.addOneAssumeCapacity(), | 867 | break :new free_ni; |
| | 868 | } |
| | 869 | const new_ni: Node.Index = @fromBackingInt(@intCast(mf.nodes.items.len)); |
| | 870 | _ = mf.nodes.addOneAssumeCapacity(); |
| | 871 | break :new new_ni; |
| 722 | }; | 872 | }; |
| 723 | | 873 | |
| 724 | if (opts.prev.unwrap()) |prev_ni| { | 874 | const next_oni: Node.Index.Optional = if (opts.prev.unwrap()) |prev_ni| next: { |
| 725 | try prev_ni.setNext(gpa, .wrap(free_ni), mf); | 875 | assert(prev_ni.parent(mf) == opts.parent.toOptional()); // `prev` is not a child of `parent` |
| 726 | } else if (opts.parent.unwrap()) |parent_ni| { | 876 | break :next prev_ni.get(mf).next; |
| 727 | parent_ni.get(mf).first = .wrap(free_ni); | 877 | } else opts.parent.first(mf); |
| 728 | } else { | | |
| 729 | assert(free_ni == .root); | | |
| 730 | } | | |
| 731 | | 878 | |
| 732 | if (opts.next.unwrap()) |next_ni| { | 879 | // Validate node ordering |
| 733 | next_ni.get(mf).prev = .wrap(free_ni); | 880 | switch (opts.position) { |
| 734 | } else if (opts.parent.unwrap()) |parent_ni| { | 881 | .floating => { |
| 735 | parent_ni.get(mf).last = .wrap(free_ni); | 882 | if (opts.prev.unwrap()) |prev_ni| { |
| 736 | } else { | 883 | assert(prev_ni.position(mf) != .footer); // tried to add floating node after footer node |
| 737 | assert(free_ni == .root); | 884 | } |
| | 885 | if (next_oni.unwrap()) |next_ni| { |
| | 886 | assert(next_ni.position(mf) != .header); // tried to add floating node before header node |
| | 887 | } |
| | 888 | }, |
| | 889 | .header => if (opts.prev.unwrap()) |prev_ni| { |
| | 890 | switch (prev_ni.position(mf)) { |
| | 891 | .header => {}, |
| | 892 | .floating => unreachable, // tried to add header node after floating node |
| | 893 | .footer => unreachable, // tried to add header node after footer node |
| | 894 | } |
| | 895 | }, |
| | 896 | .footer => if (next_oni.unwrap()) |next_ni| { |
| | 897 | switch (next_ni.position(mf)) { |
| | 898 | .header => unreachable, // tried to add footer node before header node |
| | 899 | .floating => unreachable, // tried to add footer node before floating node |
| | 900 | .footer => {}, |
| | 901 | } |
| | 902 | }, |
| 738 | } | 903 | } |
| 739 | | 904 | |
| 740 | free_node.* = .{ | 905 | // Initialize the node as empty with alignment 1 |
| 741 | .parent = opts.parent, | 906 | const location: Node.Location = loc: { |
| 742 | .prev = opts.prev, | 907 | const offset: u64 = switch (opts.position) { |
| 743 | .next = opts.next, | 908 | .header, .floating => offset: { |
| | 909 | const prev_ni = opts.prev.unwrap() orelse break :offset 0; |
| | 910 | const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); |
| | 911 | break :offset prev_offset + prev_size; |
| | 912 | }, |
| | 913 | .footer => offset: { |
| | 914 | const next_ni = next_oni.unwrap() orelse { |
| | 915 | _, const parent_size = opts.parent.location(mf).resolve(mf); |
| | 916 | break :offset parent_size; |
| | 917 | }; |
| | 918 | const next_offset, _ = next_ni.location(mf).resolve(mf); |
| | 919 | break :offset next_offset; |
| | 920 | }, |
| | 921 | }; |
| | 922 | if (std.math.cast(u32, offset)) |small_offset| { |
| | 923 | break :loc .{ .small = .{ .offset = small_offset, .size = 0 } }; |
| | 924 | } |
| | 925 | const large_index = mf.large.items.len; |
| | 926 | mf.large.appendSliceAssumeCapacity(&.{ offset, 0 }); |
| | 927 | break :loc .{ .large = .{ .index = large_index } }; |
| | 928 | }; |
| | 929 | new_ni.get(mf).* = .{ |
| | 930 | .parent = .wrap(opts.parent), |
| | 931 | .prev = .none, |
| | 932 | .next = .none, |
| 744 | .first = .none, | 933 | .first = .none, |
| 745 | .last = .none, | 934 | .last = .none, |
| 746 | .flags = .{ | 935 | .flags = .{ |
| 747 | .location_tag = location_tag, | 936 | .position = opts.position, |
| 748 | .alignment = .@"1", | 937 | .alignment = .@"1", |
| 749 | .fixed = opts.add_node.fixed, | 938 | .bubbles_moved = opts.add_options.bubbles_moved, |
| 750 | .moved = true, | 939 | .enable_next_moved = opts.add_options.enable_next_moved, |
| 751 | .resized = true, | 940 | .location_tag = location, |
| 752 | .next_moved = true, | 941 | .moved = false, |
| | 942 | .resized = false, |
| | 943 | .next_moved = false, |
| 753 | .has_content = false, | 944 | .has_content = false, |
| 754 | .bubbles_moved = opts.add_node.bubbles_moved, | | |
| 755 | .enable_next_moved = opts.add_node.enable_next_moved, | | |
| 756 | }, | 945 | }, |
| 757 | .location_payload = location_payload, | 946 | .location_payload = switch (location) { |
| | 947 | .small => |small| .{ .small = small }, |
| | 948 | .large => |large| .{ .large = large }, |
| | 949 | }, |
| 758 | }; | 950 | }; |
| 759 | | 951 | |
| 760 | { | 952 | try mf.addNodesToChildListBefore(gpa, next_oni, new_ni, new_ni); |
| 761 | defer { | 953 | |
| 762 | free_node.flags.moved = false; | 954 | try mf.realignNode(gpa, new_ni, opts.add_options.alignment); |
| 763 | free_node.flags.resized = false; | 955 | if (opts.add_options.size > 0) { |
| 764 | free_node.flags.next_moved = false; | 956 | try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); |
| 765 | } | | |
| 766 | try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{}); | | |
| 767 | try mf.resizeNode(gpa, free_ni, opts.add_node.size); | | |
| 768 | } | 957 | } |
| 769 | mf.updateWriters(); | 958 | mf.updateWriters(); |
| 770 | if (opts.add_node.moved) try free_ni.moved(gpa, mf); | | |
| 771 | if (opts.add_node.resized) try free_ni.resized(gpa, mf); | | |
| 772 | if (opts.add_node.next_moved) try free_ni.nextMoved(gpa, mf); | | |
| 773 | return free_ni; | | |
| 774 | } | | |
| 775 | | 959 | |
| 776 | pub const AddNodeOptions = struct { | 960 | new_ni.get(mf).flags.moved = false; |
| 777 | size: u64 = 0, | 961 | new_ni.get(mf).flags.resized = false; |
| 778 | alignment: Alignment = .@"1", | 962 | new_ni.get(mf).flags.next_moved = false; |
| 779 | fixed: bool = false, | | |
| 780 | moved: bool = false, | | |
| 781 | resized: bool = false, | | |
| 782 | next_moved: bool = false, | | |
| 783 | bubbles_moved: bool = true, | | |
| 784 | enable_next_moved: bool = false, | | |
| 785 | }; | | |
| 786 | | 963 | |
| 787 | pub fn addOnlyChildNode( | 964 | if (opts.add_options.moved) try new_ni.moved(gpa, mf); |
| 788 | mf: *MappedFile, | 965 | if (opts.add_options.resized) try new_ni.resized(gpa, mf); |
| 789 | gpa: Allocator, | 966 | if (opts.add_options.next_moved) try new_ni.nextMoved(gpa, mf); |
| 790 | parent_ni: Node.Index, | | |
| 791 | opts: AddNodeOptions, | | |
| 792 | ) Error!Node.Index { | | |
| 793 | try mf.nodes.ensureUnusedCapacity(gpa, 1); | | |
| 794 | const parent = parent_ni.get(mf); | | |
| 795 | assert(parent.first == .none and parent.last == .none); | | |
| 796 | return mf.addNode(gpa, .{ | | |
| 797 | .parent = .wrap(parent_ni), | | |
| 798 | .add_node = opts, | | |
| 799 | }) catch |err| switch (err) { | | |
| 800 | error.OutOfMemory, | | |
| 801 | error.Canceled, | | |
| 802 | => |e| return e, | | |
| 803 | else => |e| { | | |
| 804 | mf.io_err = e; | | |
| 805 | return error.MappedFileIo; | | |
| 806 | }, | | |
| 807 | }; | | |
| 808 | } | | |
| 809 | | 967 | |
| 810 | pub fn addFirstChildNode( | 968 | return new_ni; |
| 811 | mf: *MappedFile, | | |
| 812 | gpa: Allocator, | | |
| 813 | parent_ni: Node.Index, | | |
| 814 | opts: AddNodeOptions, | | |
| 815 | ) Error!Node.Index { | | |
| 816 | try mf.nodes.ensureUnusedCapacity(gpa, 1); | | |
| 817 | const parent = parent_ni.get(mf); | | |
| 818 | return mf.addNode(gpa, .{ | | |
| 819 | .parent = .wrap(parent_ni), | | |
| 820 | .next = parent.first, | | |
| 821 | .add_node = opts, | | |
| 822 | }) catch |err| switch (err) { | | |
| 823 | error.OutOfMemory, | | |
| 824 | error.Canceled, | | |
| 825 | => |e| return e, | | |
| 826 | else => |e| { | | |
| 827 | mf.io_err = e; | | |
| 828 | return error.MappedFileIo; | | |
| 829 | }, | | |
| 830 | }; | | |
| 831 | } | 969 | } |
| 832 | | 970 | |
| 833 | pub fn addLastChildNode( | 971 | fn shrinkLeafNode( |
| 834 | mf: *MappedFile, | 972 | mf: *MappedFile, |
| 835 | gpa: Allocator, | 973 | gpa: Allocator, |
| 836 | parent_ni: Node.Index, | 974 | ni: Node.Index, |
| 837 | opts: AddNodeOptions, | 975 | new_size: u64, |
| 838 | ) Error!Node.Index { | 976 | ) Error!void { |
| 839 | try mf.nodes.ensureUnusedCapacity(gpa, 1); | 977 | mf.nodes_lock.assertUnlocked(); |
| 840 | const parent = parent_ni.get(mf); | 978 | |
| 841 | return mf.addNode(gpa, .{ | 979 | const old_offset, const old_size = ni.location(mf).resolve(mf); |
| 842 | .parent = .wrap(parent_ni), | 980 | |
| 843 | .prev = parent.last, | 981 | assert(new_size < old_size); |
| 844 | .offset = offset: { | 982 | assert(ni.alignment(mf).check(new_size)); |
| 845 | const last_ni = parent.last.unwrap() orelse break :offset 0; | 983 | assert(ni.first(mf) == .none); // `ni` must be a leaf node |
| 846 | const last_offset, const last_size = last_ni.location(mf).resolve(mf); | 984 | |
| 847 | break :offset last_offset + last_size; | 985 | const parent_ni = ni.parent(mf).unwrap() orelse { |
| 848 | }, | 986 | assert(ni == .root); |
| 849 | .add_node = opts, | 987 | mf.memory_map.write(mf.io) catch |err| { |
| 850 | }) catch |err| switch (err) { | 988 | mf.io_err = switch (err) { |
| 851 | error.OutOfMemory, | 989 | error.Canceled => |e| return e, |
| 852 | error.Canceled, | 990 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 853 | => |e| return e, | 991 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 854 | else => |e| { | 992 | else => |e| e, |
| 855 | mf.io_err = e; | 993 | }; |
| 856 | return error.MappedFileIo; | 994 | return error.MappedFileIo; |
| 857 | }, | 995 | }; |
| | 996 | mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { |
| | 997 | error.Canceled => |e| return e, |
| | 998 | else => |e| { |
| | 999 | mf.io_err = e; |
| | 1000 | return error.MappedFileIo; |
| | 1001 | }, |
| | 1002 | }; |
| | 1003 | try mf.ensureTotalCapacityPrecise(@intCast(new_size)); |
| | 1004 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| | 1005 | return; |
| 858 | }; | 1006 | }; |
| 859 | } | | |
| 860 | | 1007 | |
| 861 | pub fn addNodeAfter( | 1008 | switch (ni.position(mf)) { |
| 862 | mf: *MappedFile, | 1009 | .header => { |
| 863 | gpa: Allocator, | 1010 | const shift = old_size - new_size; |
| 864 | prev_ni: Node.Index, | 1011 | |
| 865 | opts: AddNodeOptions, | 1012 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 866 | ) Error!Node.Index { | 1013 | |
| 867 | try mf.nodes.ensureUnusedCapacity(gpa, 1); | 1014 | // We need to shift backwards all header nodes following us. |
| 868 | const prev = prev_ni.get(mf); | 1015 | const next_header_ni = ni.next(mf).unwrap() orelse return; |
| 869 | const prev_offset, const prev_size = prev.location().resolve(mf); | 1016 | if (next_header_ni.position(mf) != .header) return; |
| 870 | return mf.addNode(gpa, .{ | 1017 | |
| 871 | .parent = prev.parent, | 1018 | var header_ni = next_header_ni; |
| 872 | .prev = .wrap(prev_ni), | 1019 | while (true) { |
| 873 | .next = prev.next, | 1020 | const old_header_off, const old_header_size = header_ni.location(mf).resolve(mf); |
| 874 | .offset = prev_offset + prev_size, | 1021 | try header_ni.setLocation(mf, gpa, old_header_off - shift, old_header_size); |
| 875 | .add_node = opts, | 1022 | |
| 876 | }) catch |err| switch (err) { | 1023 | const next_ni = header_ni.next(mf).unwrap() orelse break; |
| 877 | error.OutOfMemory, | 1024 | if (next_ni.position(mf) != .header) break; |
| 878 | error.Canceled, | 1025 | header_ni = next_ni; |
| 879 | => |e| return e, | 1026 | } |
| 880 | else => |e| { | 1027 | |
| 881 | mf.io_err = e; | 1028 | // Now we must shift the actual header bytes of those nodes backwards. |
| 882 | return error.MappedFileIo; | 1029 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| | 1030 | const move_src_off = old_offset + old_size; |
| | 1031 | const move_dest_off = old_offset + new_size; |
| | 1032 | assert(next_header_ni.location(mf).resolve(mf)[0] == move_dest_off); // `move_dest_off` because we already updated the location |
| | 1033 | const move_size = size: { |
| | 1034 | // `header_ni` is the last header in the parent. |
| | 1035 | const last_off, const last_size = header_ni.location(mf).resolve(mf); |
| | 1036 | const move_end = last_off + last_size; |
| | 1037 | break :size move_end - move_dest_off; // `move_dest_off` because we already updated the location |
| | 1038 | }; |
| | 1039 | try mf.moveRange( |
| | 1040 | parent_file_off + move_src_off, |
| | 1041 | parent_file_off + move_dest_off, |
| | 1042 | move_size, |
| | 1043 | ); |
| 883 | }, | 1044 | }, |
| 884 | }; | 1045 | .floating => { |
| 885 | } | 1046 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| | 1047 | }, |
| | 1048 | .footer => { |
| | 1049 | const shift = old_size - new_size; |
| 886 | | 1050 | |
| 887 | fn shrinkNode( | 1051 | const new_offset = old_offset + shift; |
| 888 | mf: *MappedFile, | 1052 | try ni.setLocation(mf, gpa, new_offset, new_size); |
| 889 | gpa: Allocator, | | |
| 890 | ni: Node.Index, | | |
| 891 | size: u64, | | |
| 892 | shift_next: bool, | | |
| 893 | ) !void { | | |
| 894 | mf.nodes_lock.assertUnlocked(); | | |
| 895 | const node = ni.get(mf); | | |
| 896 | const old_offset, _ = node.location().resolve(mf); | | |
| 897 | | 1053 | |
| 898 | // This would require unmapping first | 1054 | const prev_footers_size = prev_footers_size: { |
| 899 | assert(ni != .root); | 1055 | // We need to shift forwards all footer nodes preceding us. |
| | 1056 | const prev_footer_ni = ni.prev(mf).unwrap() orelse { |
| | 1057 | break :prev_footers_size 0; |
| | 1058 | }; |
| | 1059 | if (prev_footer_ni.position(mf) != .footer) { |
| | 1060 | break :prev_footers_size 0; |
| | 1061 | } |
| 900 | | 1062 | |
| 901 | if (node.last.unwrap()) |last_ni| { | 1063 | var footer_ni = prev_footer_ni; |
| 902 | const last = last_ni.get(mf); | 1064 | while (true) { |
| 903 | const last_offset, const last_size = last.location().resolve(mf); | 1065 | const old_footer_off, const old_footer_size = footer_ni.location(mf).resolve(mf); |
| 904 | assert(last_offset + last_size > size); | 1066 | try footer_ni.setLocation(mf, gpa, old_footer_off + shift, old_footer_size); |
| 905 | } | | |
| 906 | | 1067 | |
| 907 | try mf.large.ensureUnusedCapacity(gpa, 4); | 1068 | const prev_ni = footer_ni.prev(mf).unwrap() orelse break; |
| 908 | try mf.updates.ensureUnusedCapacity(gpa, 4); | 1069 | if (prev_ni.position(mf) != .footer) break; |
| | 1070 | footer_ni = prev_ni; |
| | 1071 | } |
| 909 | | 1072 | |
| 910 | ni.setLocationAssumeCapacity(mf, old_offset, size); | 1073 | // `footer_ni` is the first footer in the parent. This expression gets its *new* |
| 911 | if (!shift_next) return; | 1074 | // offset because we already did the `setLocation` calls. |
| 912 | const next_ni = node.next.unwrap() orelse return; | 1075 | const first_footer_new_offset = footer_ni.location(mf).resolve(mf)[0]; |
| 913 | | 1076 | |
| 914 | const next = next_ni.get(mf); | 1077 | break :prev_footers_size new_offset - first_footer_new_offset; |
| 915 | const old_next_offset, const next_size = next.location().resolve(mf); | 1078 | }; |
| 916 | const padding = old_next_offset - (old_offset + size); | | |
| 917 | const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); | | |
| 918 | | 1079 | |
| 919 | if (next.flags.has_content and new_next_offset < old_next_offset) { | 1080 | // Now we must shift the actual footer bytes forwards, including our own. |
| 920 | const old_file_offset = next_ni.fileLocation(mf, false).offset; | 1081 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| 921 | const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; | 1082 | try mf.moveRange( |
| 922 | @memmove( | 1083 | parent_file_offset + old_offset - prev_footers_size, |
| 923 | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], | 1084 | parent_file_offset + new_offset - prev_footers_size, |
| 924 | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(next_size)], | 1085 | prev_footers_size + new_size, |
| 925 | ); | 1086 | ); |
| 926 | @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); | 1087 | }, |
| 927 | } | 1088 | } |
| 928 | | | |
| 929 | next_ni.setLocationAssumeCapacity(mf, new_next_offset, next_size); | | |
| 930 | } | 1089 | } |
| 931 | | 1090 | |
| 932 | fn resizeNode( | 1091 | const GrowMode = enum { exact, minimum }; |
| | 1092 | |
| | 1093 | /// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. |
| | 1094 | /// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. |
| | 1095 | /// |
| | 1096 | /// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). |
| | 1097 | /// |
| | 1098 | /// Asserts that `new_size` is greater than the current size of `ni`. |
| | 1099 | fn growNode( |
| 933 | mf: *MappedFile, | 1100 | mf: *MappedFile, |
| 934 | gpa: Allocator, | 1101 | gpa: Allocator, |
| 935 | ni: Node.Index, | 1102 | ni: Node.Index, |
| 936 | requested_size: u64, | 1103 | new_size: u64, |
| 937 | ) (Allocator.Error || Io.Cancelable || IoError)!void { | 1104 | grow_mode: GrowMode, |
| | 1105 | ) Error!void { |
| 938 | mf.nodes_lock.assertUnlocked(); | 1106 | mf.nodes_lock.assertUnlocked(); |
| 939 | const io = mf.io; | 1107 | |
| 940 | const node = ni.get(mf); | 1108 | const node = ni.get(mf); |
| | 1109 | |
| 941 | const old_offset, const old_size = node.location().resolve(mf); | 1110 | const old_offset, const old_size = node.location().resolve(mf); |
| 942 | const new_size = node.flags.alignment.forward(@intCast(requested_size)); | | |
| 943 | | 1111 | |
| 944 | // Resize the entire file | 1112 | assert(node.flags.alignment.check(old_size)); |
| | 1113 | assert(node.flags.alignment.check(new_size)); |
| | 1114 | assert(new_size > old_size); |
| | 1115 | |
| 945 | const parent_ni = node.parent.unwrap() orelse { | 1116 | const parent_ni = node.parent.unwrap() orelse { |
| 946 | assert(ni == .root); | 1117 | assert(ni == .root); |
| 947 | try mf.ensureCapacityForSetLocation(gpa); | 1118 | |
| 948 | mf.memory_map.write(io) catch |err| switch (err) { | 1119 | if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { |
| 949 | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking | 1120 | return; |
| 950 | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing | 1121 | } |
| 951 | else => |e| return e, | 1122 | |
| 952 | }; | 1123 | mf.memory_map.write(mf.io) catch |err| { |
| 953 | try mf.memory_map.file.setLength(io, new_size); | 1124 | mf.io_err = switch (err) { |
| 954 | try mf.ensureTotalCapacityInner(@intCast(new_size)); | 1125 | error.Canceled => |e| return e, |
| 955 | ni.setLocationAssumeCapacity(mf, old_offset, new_size); | 1126 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 956 | return; | 1127 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 957 | }; | 1128 | else => |e| e, |
| 958 | const parent = parent_ni.get(mf); | 1129 | }; |
| 959 | _, var old_parent_size = parent.location().resolve(mf); | 1130 | return error.MappedFileIo; |
| 960 | const trailing_end = trailing_end: { | | |
| 961 | const next_ni = node.next.unwrap() orelse break :trailing_end old_parent_size; | | |
| 962 | const next_offset, _ = next_ni.location(mf).resolve(mf); | | |
| 963 | break :trailing_end next_offset; | | |
| 964 | }; | | |
| 965 | assert(old_offset + old_size <= trailing_end); | | |
| 966 | if (old_offset + new_size <= trailing_end) { | | |
| 967 | // Expand the node into trailing free space | | |
| 968 | try mf.ensureCapacityForSetLocation(gpa); | | |
| 969 | ni.setLocationAssumeCapacity(mf, old_offset, new_size); | | |
| 970 | return; | | |
| 971 | } | | |
| 972 | insert_range: { | | |
| 973 | if (!is_linux) break :insert_range; | | |
| 974 | if (mf.flags.fallocate_insert_range_unsupported) break :insert_range; | | |
| 975 | | | |
| 976 | // We need the node to be aligned to `mf.flags.block_size` in the file in order to use this | | |
| 977 | // fast path. It is not sufficient to check `node.flags.alignment`, because that doesn't | | |
| 978 | // necessarily mean that all *parent* nodes are equally aligned; instead we must compute the | | |
| 979 | // actual file offset. | | |
| 980 | const range_file_offset = ni.fileLocation(mf, false).offset + old_size; | | |
| 981 | const range_size = node.flags.alignment.forward( | | |
| 982 | @intCast(requested_size +| requested_size / growth_factor), | | |
| 983 | ) - old_size; | | |
| 984 | if (!mf.flags.block_size.check(@intCast(range_file_offset))) break :insert_range; | | |
| 985 | if (!mf.flags.block_size.check(@intCast(range_size))) break :insert_range; | | |
| 986 | | | |
| 987 | mf.memory_map.write(io) catch |err| switch (err) { | | |
| 988 | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking | | |
| 989 | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing | | |
| 990 | else => |e| return e, | | |
| 991 | }; | 1131 | }; |
| 992 | // Ask the filesystem driver to insert extents into the file without copying any data | 1132 | mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { |
| 993 | const last_offset, const last_size = parent.last.unwrap().?.location(mf).resolve(mf); | 1133 | error.Canceled => |e| return e, |
| 994 | const last_end = last_offset + last_size; | 1134 | else => |e| { |
| 995 | assert(last_end <= old_parent_size); | 1135 | mf.io_err = e; |
| 996 | _, const file_size = Node.Index.root.location(mf).resolve(mf); | 1136 | return error.MappedFileIo; |
| 997 | while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) { | | |
| 998 | .lt => linux.fallocate( | | |
| 999 | mf.memory_map.file.handle, | | |
| 1000 | linux.FALLOC.FL_INSERT_RANGE, | | |
| 1001 | @intCast(range_file_offset), | | |
| 1002 | @intCast(range_size), | | |
| 1003 | ), | | |
| 1004 | .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)), | | |
| 1005 | .gt => unreachable, | | |
| 1006 | })) { | | |
| 1007 | .SUCCESS => { | | |
| 1008 | var enclosing_ni = ni; | | |
| 1009 | while (true) { | | |
| 1010 | try mf.ensureCapacityForSetLocation(gpa); | | |
| 1011 | const enclosing = enclosing_ni.get(mf); | | |
| 1012 | const enclosing_offset, const old_enclosing_size = | | |
| 1013 | enclosing.location().resolve(mf); | | |
| 1014 | const new_enclosing_size = old_enclosing_size + range_size; | | |
| 1015 | enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); | | |
| 1016 | if (enclosing_ni == .root) { | | |
| 1017 | assert(enclosing_offset == 0); | | |
| 1018 | try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); | | |
| 1019 | break; | | |
| 1020 | } | | |
| 1021 | var after_oni = enclosing.next; | | |
| 1022 | while (after_oni.unwrap()) |after_ni| { | | |
| 1023 | try mf.ensureCapacityForSetLocation(gpa); | | |
| 1024 | const after = after_ni.get(mf); | | |
| 1025 | const after_offset, const after_size = after.location().resolve(mf); | | |
| 1026 | after_ni.setLocationAssumeCapacity( | | |
| 1027 | mf, | | |
| 1028 | range_size + after_offset, | | |
| 1029 | after_size, | | |
| 1030 | ); | | |
| 1031 | after_oni = after.next; | | |
| 1032 | } | | |
| 1033 | enclosing_ni = enclosing.parent.unwrap().?; | | |
| 1034 | } | | |
| 1035 | return; | | |
| 1036 | }, | | |
| 1037 | .INTR => continue, | | |
| 1038 | .BADF, .FBIG, .INVAL => unreachable, | | |
| 1039 | .IO => return error.InputOutput, | | |
| 1040 | .NODEV => return error.NotFile, | | |
| 1041 | .NOSPC => return error.NoSpaceLeft, | | |
| 1042 | .NOSYS, .OPNOTSUPP => { | | |
| 1043 | mf.flags.fallocate_insert_range_unsupported = true; | | |
| 1044 | break :insert_range; | | |
| 1045 | }, | 1137 | }, |
| 1046 | .PERM => return error.PermissionDenied, | | |
| 1047 | .SPIPE => return error.Unseekable, | | |
| 1048 | .TXTBSY => return error.FileBusy, | | |
| 1049 | else => |e| return std.posix.unexpectedErrno(e), | | |
| 1050 | }; | 1138 | }; |
| 1051 | } | 1139 | try mf.ensureTotalCapacityPrecise(@intCast(new_size)); |
| 1052 | if (node.next == .none) { | 1140 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1053 | // As this is the last node, we simply need more space in the parent | 1141 | // We need to move any footers to be at the *new* end of the file. |
| 1054 | const new_parent_size = old_offset + new_size; | 1142 | if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { |
| 1055 | try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); | 1143 | const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1056 | try mf.ensureCapacityForSetLocation(gpa); | 1144 | const footers_size = old_size - old_footers_offset; |
| 1057 | ni.setLocationAssumeCapacity(mf, old_offset, new_size); | | |
| 1058 | return; | | |
| 1059 | } | | |
| 1060 | if (!node.flags.fixed) { | | |
| 1061 | // Make space at the end of the parent for this floating node | | |
| 1062 | const last = parent.last.unwrap().?.get(mf); | | |
| 1063 | const last_offset, const last_size = last.location().resolve(mf); | | |
| 1064 | const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); | | |
| 1065 | const new_parent_size = new_offset + new_size; | | |
| 1066 | if (new_parent_size > old_parent_size) | | |
| 1067 | try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); | | |
| 1068 | try mf.ensureCapacityForSetLocation(gpa); | | |
| 1069 | const next_ni = node.next.unwrap().?; | | |
| 1070 | next_ni.get(mf).prev = node.prev; | | |
| 1071 | if (node.prev.unwrap()) |prev_ni| { | | |
| 1072 | try prev_ni.setNext(gpa, .wrap(next_ni), mf); | | |
| 1073 | } else { | | |
| 1074 | parent.first = .wrap(next_ni); | | |
| 1075 | } | | |
| 1076 | try parent.last.unwrap().?.setNext(gpa, .wrap(ni), mf); | | |
| 1077 | node.prev = parent.last; | | |
| 1078 | try ni.setNext(gpa, .none, mf); | | |
| 1079 | parent.last = .wrap(ni); | | |
| 1080 | if (node.flags.has_content) { | | |
| 1081 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; | | |
| 1082 | try mf.moveRange( | 1145 | try mf.moveRange( |
| 1083 | parent_file_offset + old_offset, | 1146 | old_footers_offset, |
| 1084 | parent_file_offset + new_offset, | 1147 | old_footers_offset + (new_size - old_size), |
| 1085 | old_size, | 1148 | footers_size, |
| 1086 | ); | 1149 | ); |
| | 1150 | // Also update the footers' locations. |
| | 1151 | var cur_ni = first_footer_ni; |
| | 1152 | while (true) { |
| | 1153 | const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); |
| | 1154 | try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); |
| | 1155 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| | 1156 | } |
| 1087 | } | 1157 | } |
| 1088 | ni.setLocationAssumeCapacity(mf, new_offset, new_size); | | |
| 1089 | return; | 1158 | return; |
| 1090 | } | 1159 | }; |
| 1091 | // Search for the first floating node following this fixed node | 1160 | |
| 1092 | var last_fixed_ni = ni; | 1161 | switch (node.flags.position) { |
| 1093 | var first_floating_oni = node.next; | 1162 | .header => { |
| 1094 | var shift = new_size - old_size; | 1163 | if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { |
| 1095 | var max_shift_align: Alignment = .@"1"; | 1164 | return; |
| 1096 | var direction: enum { forward, reverse } = .forward; | | |
| 1097 | while (true) { | | |
| 1098 | const last_fixed = last_fixed_ni.get(mf); | | |
| 1099 | assert(last_fixed.flags.fixed); | | |
| 1100 | const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); | | |
| 1101 | const new_last_fixed_offset = old_last_fixed_offset + shift; | | |
| 1102 | if (first_floating_oni.unwrap()) |first_floating_ni| make_space: { | | |
| 1103 | const first_floating = first_floating_ni.get(mf); | | |
| 1104 | const old_first_floating_offset, const first_floating_size = | | |
| 1105 | first_floating.location().resolve(mf); | | |
| 1106 | assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); | | |
| 1107 | if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) | | |
| 1108 | break :make_space; | | |
| 1109 | assert(direction == .forward); | | |
| 1110 | max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); | | |
| 1111 | if (first_floating.flags.fixed) { | | |
| 1112 | shift = max_shift_align.forward(@intCast( | | |
| 1113 | @max(shift, first_floating_size), | | |
| 1114 | )); | | |
| 1115 | | | |
| 1116 | // Not enough space, try the next node | | |
| 1117 | last_fixed_ni = first_floating_ni; | | |
| 1118 | first_floating_oni = first_floating.next; | | |
| 1119 | continue; | | |
| 1120 | } | | |
| 1121 | // Move the found floating node to make space for preceding fixed nodes | | |
| 1122 | const last = parent.last.unwrap().?.get(mf); | | |
| 1123 | const last_offset, const last_size = last.location().resolve(mf); | | |
| 1124 | const new_first_floating_offset = max_shift_align.forward( | | |
| 1125 | @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), | | |
| 1126 | ); | | |
| 1127 | const new_parent_size = new_first_floating_offset + first_floating_size; | | |
| 1128 | if (new_parent_size > old_parent_size) { | | |
| 1129 | try mf.resizeNode( | | |
| 1130 | gpa, | | |
| 1131 | parent_ni, | | |
| 1132 | new_parent_size +| new_parent_size / growth_factor, | | |
| 1133 | ); | | |
| 1134 | _, old_parent_size = parent.location().resolve(mf); | | |
| 1135 | } | 1165 | } |
| 1136 | try mf.ensureCapacityForSetLocation(gpa); | 1166 | |
| 1137 | if (parent.last.unwrap().? != first_floating_ni) { | 1167 | try mf.ensureAdditionalHeaderCapacity(gpa, parent_ni, new_size - old_size); |
| 1138 | const old_last = parent.last.unwrap().?; | 1168 | |
| 1139 | first_floating.prev = .wrap(old_last); | 1169 | // `old_offset` is still valid because header nodes don't move when the parent resizes. |
| 1140 | parent.last = .wrap(first_floating_ni); | 1170 | |
| 1141 | try old_last.setNext(gpa, .wrap(first_floating_ni), mf); | 1171 | const last_header_ni: Node.Index = last_header: { |
| 1142 | try last_fixed_ni.setNext(gpa, first_floating.next, mf); | 1172 | var header_ni = ni; |
| 1143 | if (first_floating.next.unwrap()) |next_ni| { | 1173 | while (true) { |
| 1144 | next_ni.get(mf).prev = .wrap(last_fixed_ni); | 1174 | const next_ni = header_ni.next(mf).unwrap() orelse break; |
| | 1175 | if (next_ni.position(mf) != .header) break; |
| | 1176 | header_ni = next_ni; |
| 1145 | } | 1177 | } |
| 1146 | try first_floating_ni.setNext(gpa, .none, mf); | 1178 | break :last_header header_ni; |
| 1147 | } | 1179 | }; |
| 1148 | if (first_floating.flags.has_content) { | 1180 | const last_header_offset, const last_header_size = last_header_ni.location(mf).resolve(mf); |
| 1149 | const parent_file_offset = | 1181 | const old_headers_size = last_header_offset + last_header_size; |
| 1150 | parent_ni.fileLocation(mf, false).offset; | 1182 | |
| 1151 | try mf.moveRange( | 1183 | // This is the first footer *inside* of `ni`. |
| 1152 | parent_file_offset + old_first_floating_offset, | 1184 | const first_sub_footer_oni = ni.firstFooter(mf); |
| 1153 | parent_file_offset + new_first_floating_offset, | 1185 | const sub_footers_size = size: { |
| 1154 | first_floating_size, | 1186 | const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; |
| 1155 | ); | 1187 | const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); |
| 1156 | } | 1188 | break :size old_size - first_sub_footer_offset; |
| 1157 | first_floating_ni.setLocationAssumeCapacity( | 1189 | }; |
| 1158 | mf, | 1190 | |
| 1159 | new_first_floating_offset, | 1191 | // We need to shift two things forwards; any header nodes which follow us, and any |
| 1160 | first_floating_size, | 1192 | // footer nodes *within* us (since they need to be at the end of our new size). |
| 1161 | ); | | |
| 1162 | // Continue the search after the just-moved floating node | | |
| 1163 | first_floating_oni = last_fixed.next; | | |
| 1164 | continue; | | |
| 1165 | } else { | | |
| 1166 | assert(direction == .forward); | | |
| 1167 | const new_parent_size = new_last_fixed_offset + last_fixed_size; | | |
| 1168 | if (new_parent_size > old_parent_size) { | | |
| 1169 | try mf.resizeNode( | | |
| 1170 | gpa, | | |
| 1171 | parent_ni, | | |
| 1172 | new_parent_size +| new_parent_size / growth_factor, | | |
| 1173 | ); | | |
| 1174 | _, old_parent_size = parent.location().resolve(mf); | | |
| 1175 | } | | |
| 1176 | } | | |
| 1177 | try mf.ensureCapacityForSetLocation(gpa); | | |
| 1178 | if (last_fixed_ni == ni) { | | |
| 1179 | // The original fixed node now has enough space | | |
| 1180 | last_fixed_ni.setLocationAssumeCapacity( | | |
| 1181 | mf, | | |
| 1182 | old_last_fixed_offset, | | |
| 1183 | new_size, | | |
| 1184 | ); | | |
| 1185 | return; | | |
| 1186 | } | | |
| 1187 | // Move a fixed node into trailing free space | | |
| 1188 | if (last_fixed.flags.has_content) { | | |
| 1189 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; | 1193 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| 1190 | try mf.moveRange( | 1194 | try mf.moveRange( |
| 1191 | parent_file_offset + old_last_fixed_offset, | 1195 | parent_file_offset + old_offset + old_size - sub_footers_size, |
| 1192 | parent_file_offset + new_last_fixed_offset, | 1196 | parent_file_offset + old_offset + new_size - sub_footers_size, |
| 1193 | last_fixed_size, | 1197 | old_headers_size - (old_offset + old_size - sub_footers_size), |
| 1194 | ); | 1198 | ); |
| 1195 | } | | |
| 1196 | last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); | | |
| 1197 | // Retry the previous nodes now that there is enough space | | |
| 1198 | first_floating_oni = .wrap(last_fixed_ni); | | |
| 1199 | last_fixed_ni = last_fixed.prev.unwrap().?; | | |
| 1200 | direction = .reverse; | | |
| 1201 | } | | |
| 1202 | } | | |
| 1203 | | | |
| 1204 | fn realignNode( | | |
| 1205 | mf: *MappedFile, | | |
| 1206 | gpa: Allocator, | | |
| 1207 | ni: Node.Index, | | |
| 1208 | new_alignment: Alignment, | | |
| 1209 | opts: Node.Index.RealignNodeOptions, | | |
| 1210 | ) (Allocator.Error || Io.Cancelable || IoError)!void { | | |
| 1211 | mf.nodes_lock.assertUnlocked(); | | |
| 1212 | | 1199 | |
| 1213 | const node = ni.get(mf); | 1200 | // Any footers inside of us have had their offsets changed due to us growing: |
| 1214 | { | 1201 | if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { |
| 1215 | const prev_alignment = node.flags.alignment; | 1202 | var cur_ni = first_sub_footer_ni; |
| 1216 | node.flags.alignment = new_alignment; | 1203 | while (true) { |
| 1217 | if (new_alignment.compare(.lte, prev_alignment)) return; | 1204 | const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); |
| | 1205 | try cur_ni.setLocation( |
| | 1206 | mf, |
| | 1207 | gpa, |
| | 1208 | old_sub_footer_offset + (new_size - old_size), |
| | 1209 | sub_footer_size, |
| | 1210 | ); |
| | 1211 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| | 1212 | } |
| | 1213 | } |
| | 1214 | |
| | 1215 | // Update the offsets of all header nodes following us: |
| | 1216 | { |
| | 1217 | var moved_header_ni = last_header_ni; |
| | 1218 | while (moved_header_ni != ni) { |
| | 1219 | assert(moved_header_ni.position(mf) == .header); |
| | 1220 | const moved_header_offset, const moved_header_size = moved_header_ni.location(mf).resolve(mf); |
| | 1221 | try moved_header_ni.setLocation( |
| | 1222 | mf, |
| | 1223 | gpa, |
| | 1224 | moved_header_offset - old_size + new_size, |
| | 1225 | moved_header_size, |
| | 1226 | ); |
| | 1227 | moved_header_ni = moved_header_ni.prev(mf).unwrap().?; |
| | 1228 | } |
| | 1229 | } |
| | 1230 | |
| | 1231 | // Finally, update our own size: |
| | 1232 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| | 1233 | return; |
| | 1234 | }, |
| | 1235 | .floating => { |
| | 1236 | try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); |
| | 1237 | }, |
| | 1238 | .footer => { |
| | 1239 | if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { |
| | 1240 | return; |
| | 1241 | } |
| | 1242 | |
| | 1243 | try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); |
| | 1244 | |
| | 1245 | const first_footer_ni: Node.Index = first_footer: { |
| | 1246 | var footer_ni = ni; |
| | 1247 | while (true) { |
| | 1248 | const prev_ni = footer_ni.prev(mf).unwrap() orelse break; |
| | 1249 | if (prev_ni.position(mf) != .footer) break; |
| | 1250 | footer_ni = prev_ni; |
| | 1251 | } |
| | 1252 | break :first_footer footer_ni; |
| | 1253 | }; |
| | 1254 | |
| | 1255 | // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself |
| | 1256 | // a footer within its parent). |
| | 1257 | const first_sub_footer_oni = ni.firstFooter(mf); |
| | 1258 | const sub_footers_size = size: { |
| | 1259 | const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; |
| | 1260 | const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); |
| | 1261 | break :size old_size - first_sub_footer_offset; |
| | 1262 | }; |
| | 1263 | |
| | 1264 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| | 1265 | |
| | 1266 | const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; |
| | 1267 | const new_footers_size = old_footers_size - old_size + new_size; |
| | 1268 | |
| | 1269 | // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node |
| | 1270 | // itself needs to shift its contents, because our offset was shifted backwards by |
| | 1271 | // `new_size - old_size`, and the added bytes should go at the end of this footer node. |
| | 1272 | // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so |
| | 1273 | // we *shouldn't* shift *that* data. |
| | 1274 | const old_footers_start = parent_size - old_footers_size; |
| | 1275 | const new_footers_start = parent_size - new_footers_size; |
| | 1276 | const end_offset = node.location().resolve(mf)[0] + old_size; |
| | 1277 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| | 1278 | try mf.moveRange( |
| | 1279 | parent_file_offset + old_footers_start, |
| | 1280 | parent_file_offset + new_footers_start, |
| | 1281 | end_offset - old_footers_start - sub_footers_size, |
| | 1282 | ); |
| | 1283 | |
| | 1284 | // Update our own offset and size: |
| | 1285 | try ni.setLocation(mf, gpa, end_offset - new_size, new_size); |
| | 1286 | |
| | 1287 | // Any footers inside of us have had their offsets changed due to us growing: |
| | 1288 | if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { |
| | 1289 | var cur_ni = first_sub_footer_ni; |
| | 1290 | while (true) { |
| | 1291 | const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); |
| | 1292 | try cur_ni.setLocation( |
| | 1293 | mf, |
| | 1294 | gpa, |
| | 1295 | old_sub_footer_offset + (new_size - old_size), |
| | 1296 | sub_footer_size, |
| | 1297 | ); |
| | 1298 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| | 1299 | } |
| | 1300 | } |
| | 1301 | |
| | 1302 | // Finally, update the offsets of every footer before us: |
| | 1303 | if (node.prev.unwrap()) |prev_ni| { |
| | 1304 | var maybe_footer_ni = prev_ni; |
| | 1305 | while (true) { |
| | 1306 | switch (maybe_footer_ni.position(mf)) { |
| | 1307 | .header, .floating => break, |
| | 1308 | .footer => {}, |
| | 1309 | } |
| | 1310 | const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); |
| | 1311 | try maybe_footer_ni.setLocation( |
| | 1312 | mf, |
| | 1313 | gpa, |
| | 1314 | moved_footer_offset + old_size - new_size, |
| | 1315 | moved_footer_size, |
| | 1316 | ); |
| | 1317 | maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; |
| | 1318 | } |
| | 1319 | } |
| | 1320 | |
| | 1321 | return; |
| | 1322 | }, |
| 1218 | } | 1323 | } |
| | 1324 | } |
| 1219 | | 1325 | |
| 1220 | const old_offset, const size = node.location().resolve(mf); | 1326 | /// Moves a floating node to an unused region with the given size, which may be greater than the |
| 1221 | const parent_ni = node.parent.unwrap() orelse { | 1327 | /// current size. If `new_alignment` is not `null`, then the offset and size of the new region will |
| 1222 | assert(ni == .root); | 1328 | /// have that alignment instead of `ni.alignment(mf)`. |
| 1223 | return mf.resizeNode(gpa, ni, size); | 1329 | /// |
| 1224 | }; | 1330 | /// Asserts that `ni` is a floating node (and not `.root`). |
| | 1331 | /// |
| | 1332 | /// Asserts that `new_size` is aligned to `new_alignment orelse ni.alignment(mf)`. |
| | 1333 | /// |
| | 1334 | /// Asserts that `new_size` is greater than or equal to the current size of `ni`. |
| | 1335 | fn growFloatingNodeWithAlignment( |
| | 1336 | mf: *MappedFile, |
| | 1337 | gpa: Allocator, |
| | 1338 | ni: Node.Index, |
| | 1339 | new_alignment: ?Alignment, |
| | 1340 | new_size: u64, |
| | 1341 | grow_mode: GrowMode, |
| | 1342 | ) Error!void { |
| | 1343 | mf.nodes_lock.assertUnlocked(); |
| 1225 | | 1344 | |
| 1226 | const new_size = new_alignment.forward(@intCast(size)); | 1345 | const parent_ni = ni.parent(mf).unwrap().?; // `ni` cannot be `.root` |
| 1227 | if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); | 1346 | const old_offset, const old_size = ni.location(mf).resolve(mf); |
| 1228 | | 1347 | |
| 1229 | _, const parent_size = parent_ni.location(mf).resolve(mf); | 1348 | const alignment = new_alignment orelse ni.alignment(mf); |
| 1230 | const trailing_end = trailing_end: { | | |
| 1231 | const next_ni = node.next.unwrap() orelse break :trailing_end parent_size; | | |
| 1232 | const next_offset, _ = next_ni.location(mf).resolve(mf); | | |
| 1233 | break :trailing_end next_offset; | | |
| 1234 | }; | | |
| 1235 | | 1349 | |
| 1236 | if (opts.try_backwards) { | 1350 | assert(new_size >= old_size); |
| 1237 | const backward_offset = new_alignment.backward(@intCast(old_offset)); | 1351 | assert(ni.position(mf) == .floating); |
| 1238 | const prev_end = prev_end: { | 1352 | assert(alignment.check(new_size)); |
| 1239 | const prev_ni = node.prev.unwrap() orelse break :prev_end 0; | 1353 | |
| 1240 | const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); | 1354 | grow_in_place: { |
| 1241 | break :prev_end prev_offset + prev_size; | 1355 | if (!alignment.check(old_offset)) { |
| | 1356 | break :grow_in_place; |
| | 1357 | } |
| | 1358 | const limit: u64 = limit: { |
| | 1359 | const next_ni = ni.next(mf).unwrap() orelse break :limit parent_ni.location(mf).resolve(mf)[1]; |
| | 1360 | const next_offset, _ = next_ni.location(mf).resolve(mf); |
| | 1361 | break :limit next_offset; |
| 1242 | }; | 1362 | }; |
| | 1363 | if (old_offset + new_size > limit) { |
| | 1364 | break :grow_in_place; // the parent is not big enough |
| | 1365 | } |
| | 1366 | // Great, we can grow this node without changing its offset or moving any siblings. |
| | 1367 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| | 1368 | // If we have any footers, we need to move them to the end of our new size, and update their |
| | 1369 | // offsets accordingly. |
| | 1370 | if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { |
| | 1371 | var cur_ni = first_footer_ni; |
| | 1372 | var footers_have_content = false; |
| | 1373 | while (true) { |
| | 1374 | footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; |
| | 1375 | const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); |
| | 1376 | try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); |
| | 1377 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| | 1378 | } |
| | 1379 | if (footers_have_content) { |
| | 1380 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| | 1381 | // This gets the *new* offset because we already updated the offsets above. |
| | 1382 | const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| | 1383 | const footers_size = new_size - new_footers_offset; |
| | 1384 | try mf.moveRange( |
| | 1385 | parent_file_off + old_offset + old_size - footers_size, |
| | 1386 | parent_file_off + old_offset + new_size - footers_size, |
| | 1387 | footers_size, |
| | 1388 | ); |
| | 1389 | } |
| | 1390 | } |
| | 1391 | return; |
| | 1392 | } |
| 1243 | | 1393 | |
| 1244 | if (backward_offset >= prev_end) { | 1394 | const new_loc: struct { |
| 1245 | try mf.ensureCapacityForSetLocation(gpa); | 1395 | offset: u64, |
| | 1396 | prev: Node.Index.Optional, |
| | 1397 | } = new_loc: { |
| | 1398 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| | 1399 | |
| | 1400 | { |
| | 1401 | // See if there's space at the start of the parent. |
| | 1402 | const last_header_oni = parent_ni.lastHeader(mf); |
| | 1403 | const headers_end: u64 = if (last_header_oni.unwrap()) |last_header_ni| headers_end: { |
| | 1404 | const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); |
| | 1405 | break :headers_end last_header_off + last_header_size; |
| | 1406 | } else 0; |
| | 1407 | const limit: u64 = limit: { |
| | 1408 | const after_header_oni: Node.Index.Optional = after_header: { |
| | 1409 | if (last_header_oni.unwrap()) |last_header_ni| { |
| | 1410 | break :after_header last_header_ni.next(mf); |
| | 1411 | } |
| | 1412 | break :after_header parent_ni.first(mf); |
| | 1413 | }; |
| | 1414 | if (after_header_oni.unwrap()) |after_header_ni| { |
| | 1415 | break :limit after_header_ni.location(mf).resolve(mf)[0]; |
| | 1416 | } else { |
| | 1417 | break :limit parent_size; |
| | 1418 | } |
| | 1419 | }; |
| | 1420 | if (alignment.forward(headers_end) + new_size <= limit) { |
| | 1421 | // There's space here! |
| | 1422 | break :new_loc .{ |
| | 1423 | // Put ourselves at the *end* of this range, so that the free space remains at the start of the parent. |
| | 1424 | .offset = alignment.backward(limit - new_size), |
| | 1425 | .prev = last_header_oni, |
| | 1426 | }; |
| | 1427 | } |
| | 1428 | } |
| 1246 | | 1429 | |
| 1247 | if (node.flags.has_content) { | 1430 | // Otherwise, use space at the end of the parent, or make space there if necessary. |
| 1248 | const old_file_offset = ni.fileLocation(mf, false).offset; | 1431 | |
| 1249 | const new_file_offset = (old_file_offset - old_offset) + backward_offset; | 1432 | const first_footer_oni = parent_ni.firstFooter(mf); |
| 1250 | @memmove( | 1433 | |
| 1251 | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], | 1434 | // We know there is a node before the footer[s], because `ni` itself is such a node. |
| 1252 | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], | 1435 | const prev_ni: Node.Index = if (first_footer_oni.unwrap()) |first_footer_ni| prev: { |
| 1253 | ); | 1436 | break :prev first_footer_ni.prev(mf).unwrap().?; |
| 1254 | @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..@intCast(old_file_offset + size)], 0); | 1437 | } else prev: { |
| | 1438 | break :prev parent_ni.last(mf).unwrap().?; |
| | 1439 | }; |
| | 1440 | |
| | 1441 | const result_offset: u64 = result_offset: { |
| | 1442 | if (prev_ni == ni and alignment.check(old_offset)) { |
| | 1443 | // We're already at the end of the parent, and our offset is already well-aligned. |
| | 1444 | // The only reason we didn't simply grow in place earlier is that the parent wasn't |
| | 1445 | // big enough---but now we're resizing the parent anyway, so growing in-place stops |
| | 1446 | // us from unnecessarily moving! |
| | 1447 | break :result_offset old_offset; |
| 1255 | } | 1448 | } |
| | 1449 | // Otherwise, just move after the last node. |
| | 1450 | const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); |
| | 1451 | break :result_offset alignment.forward(prev_offset + prev_size); |
| | 1452 | }; |
| 1256 | | 1453 | |
| 1257 | if (backward_offset + new_size <= trailing_end) { | 1454 | const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| footers_size: { |
| 1258 | ni.setLocationAssumeCapacity(mf, backward_offset, new_size); | 1455 | const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1259 | } else { | 1456 | break :footers_size parent_size - first_footer_offset; |
| 1260 | ni.setLocationAssumeCapacity(mf, backward_offset, size); | 1457 | } else 0; |
| 1261 | try mf.resizeNode(gpa, ni, new_size); | 1458 | |
| | 1459 | const min_parent_size = result_offset + new_size + footers_size; |
| | 1460 | if (parent_size < min_parent_size) { |
| | 1461 | // Okay, at this point we're planning to expand the parent---so before we actually do |
| | 1462 | // that, let's first try the Linux "insert range" fast path. We didn't try it before now |
| | 1463 | // because it would have been more efficient to just move ourselves into existing space. |
| | 1464 | // |
| | 1465 | // If we were given a custom alignment, we cannot pass `grow_mode` directly into the |
| | 1466 | // "insert range" path, because that function is unaware of `new_alignment`. |
| | 1467 | const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; |
| | 1468 | if (alignment.check(old_offset) and |
| | 1469 | try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) |
| | 1470 | { |
| | 1471 | // The Linux fast path did our job for us! |
| | 1472 | return; |
| 1262 | } | 1473 | } |
| 1263 | | 1474 | |
| 1264 | return; | 1475 | // Grow the parent and move to the end of the parent. |
| | 1476 | const new_parent_size = parent_ni.alignment(mf).forward( |
| | 1477 | min_parent_size +| min_parent_size / growth_factor, |
| | 1478 | ); |
| | 1479 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| | 1480 | } |
| | 1481 | |
| | 1482 | break :new_loc .{ |
| | 1483 | .offset = result_offset, |
| | 1484 | .prev = .wrap(prev_ni), |
| | 1485 | }; |
| | 1486 | }; |
| | 1487 | |
| | 1488 | // We've found our new location in `parent_ni`, now to actually move ourselves there. |
| | 1489 | |
| | 1490 | // Footers need to move to a different place than the rest of our content. |
| | 1491 | const footers_size: u64, const footers_have_content: bool = footers: { |
| | 1492 | const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { |
| | 1493 | break :footers .{ 0, false }; |
| | 1494 | }; |
| | 1495 | |
| | 1496 | var cur_ni = first_footer_ni; |
| | 1497 | var footers_have_content = false; |
| | 1498 | while (true) { |
| | 1499 | footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; |
| | 1500 | const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); |
| | 1501 | // Our footers' offsets must change to be at the end of our new size. |
| | 1502 | try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); |
| | 1503 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| 1265 | } | 1504 | } |
| | 1505 | |
| | 1506 | // This is the *new* offset because we already updated the offsets above. |
| | 1507 | const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| | 1508 | const footers_size = new_size - new_footers_offset; |
| | 1509 | |
| | 1510 | break :footers .{ footers_size, footers_have_content }; |
| | 1511 | }; |
| | 1512 | |
| | 1513 | if (ni.get(mf).flags.has_content) { |
| | 1514 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| | 1515 | try mf.moveRange( |
| | 1516 | parent_file_off + old_offset, |
| | 1517 | parent_file_off + new_loc.offset, |
| | 1518 | old_size - footers_size, |
| | 1519 | ); |
| | 1520 | if (footers_have_content) try mf.moveRange( |
| | 1521 | parent_file_off + old_offset + old_size - footers_size, |
| | 1522 | parent_file_off + new_loc.offset + new_size - footers_size, |
| | 1523 | footers_size, |
| | 1524 | ); |
| | 1525 | } else { |
| | 1526 | assert(!footers_have_content); |
| 1266 | } | 1527 | } |
| 1267 | | 1528 | |
| 1268 | const forward_offset = new_alignment.forward(@intCast(old_offset)); | 1529 | try ni.setLocation(mf, gpa, new_loc.offset, new_size); |
| 1269 | if (forward_offset + new_size <= trailing_end) { | 1530 | |
| 1270 | // Shift into the free space if possible | 1531 | if (new_loc.prev != ni.toOptional()) { |
| 1271 | try mf.ensureCapacityForSetLocation(gpa); | 1532 | // We're potentially in a different place in `parent_ni`'s child list, so remove and re-add ourselves. |
| 1272 | if (node.flags.has_content) { | 1533 | try mf.removeNodesFromChildList(gpa, ni, ni); |
| 1273 | const old_file_offset = ni.fileLocation(mf, false).offset; | 1534 | try mf.addNodesToChildListAfter(gpa, new_loc.prev, ni, ni); |
| 1274 | const new_file_offset = (old_file_offset - old_offset) + forward_offset; | 1535 | } |
| 1275 | if (new_file_offset < old_file_offset + size) { | 1536 | } |
| 1276 | @memmove( | 1537 | |
| 1277 | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], | 1538 | /// Attempts to grow `ni` to `new_size` using `FALLOCATE_FL_INSERT_RANGE` on Linux. This strategy |
| 1278 | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], | 1539 | /// has the advantage that it does not require manually moving any bytes in the file, but has the |
| 1279 | ); | 1540 | /// disadvantages that it may increase the file size more than necessary, and that it changes the |
| 1280 | } else try mf.moveRange(old_file_offset, new_file_offset, size); | 1541 | /// offsets of all following nodes, recursively. |
| 1281 | @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..][0..@intCast(new_size - size)], 0); | 1542 | /// |
| | 1543 | /// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` |
| | 1544 | /// without changing any nodes' locations or invalidating any slices. |
| | 1545 | /// |
| | 1546 | /// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node |
| | 1547 | /// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, |
| | 1548 | /// the actual new size of `ni` may be greater than `new_size`. |
| | 1549 | fn growNodeViaInsertRange( |
| | 1550 | mf: *MappedFile, |
| | 1551 | gpa: Allocator, |
| | 1552 | ni: Node.Index, |
| | 1553 | new_size: u64, |
| | 1554 | grow_mode: GrowMode, |
| | 1555 | ) Error!bool { |
| | 1556 | if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { |
| | 1557 | return false; |
| | 1558 | } |
| | 1559 | |
| | 1560 | _, const old_size = ni.location(mf).resolve(mf); |
| | 1561 | |
| | 1562 | // We don't compute the size of the range yet, because depending on `grow_mode` we might want to |
| | 1563 | // bump it based on our sibling and parent nodes' alignments. However, we can do an early check |
| | 1564 | // for cases where we should obviously exit. |
| | 1565 | const requested_range_size = new_size - old_size; |
| | 1566 | if (!mf.flags.block_size.check(requested_range_size)) { |
| | 1567 | // The requested size isn't exactly aligned. |
| | 1568 | switch (grow_mode) { |
| | 1569 | .exact => return false, |
| | 1570 | .minimum => { |
| | 1571 | // We can still choose to allow it by increasing the size a bit, but we shouldn't do |
| | 1572 | // that if it would *significantly* increase the requested size. |
| | 1573 | const block_size = mf.flags.block_size.toByteUnits(); |
| | 1574 | if (requested_range_size < block_size * 2) { |
| | 1575 | // Bumping this size up to the next block boundary would be a quite significant |
| | 1576 | // increase; let's not do it. |
| | 1577 | return false; |
| | 1578 | } |
| | 1579 | }, |
| | 1580 | } |
| | 1581 | } |
| | 1582 | // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump |
| | 1583 | // the size a little more. |
| | 1584 | const min_range_size: u64 = s: { |
| | 1585 | const exact_size = new_size - old_size; |
| | 1586 | if (mf.flags.block_size.check(exact_size)) { |
| | 1587 | break :s exact_size; |
| | 1588 | } |
| | 1589 | switch (grow_mode) { |
| | 1590 | .exact => return false, |
| | 1591 | .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { |
| | 1592 | // We're growing by at least a few blocks, so allow ourselves to bump the size |
| | 1593 | // slightly to give it the needed alignment. |
| | 1594 | break :s mf.flags.block_size.forward(exact_size); |
| | 1595 | } else { |
| | 1596 | return false; |
| | 1597 | }, |
| | 1598 | } |
| | 1599 | }; |
| | 1600 | assert(min_range_size > 0); |
| | 1601 | assert(mf.flags.block_size.check(min_range_size)); |
| | 1602 | |
| | 1603 | const range_file_offset: u64 = range_file_offset: { |
| | 1604 | const node_file_offset = ni.fileLocation(mf, false).offset; |
| | 1605 | const last_ni = ni.last(mf).unwrap() orelse { |
| | 1606 | // If `ni` has no children (i.e. is a leaf node), we need to insert exactly at its end. |
| | 1607 | const range_file_offset = node_file_offset + old_size; |
| | 1608 | if (!mf.flags.block_size.check(range_file_offset)) { |
| | 1609 | return false; |
| | 1610 | } |
| | 1611 | break :range_file_offset range_file_offset; |
| | 1612 | }; |
| | 1613 | const first_footer_oni = ni.firstFooter(mf); |
| | 1614 | const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { |
| | 1615 | const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| | 1616 | break :size old_size - first_footer_offset; |
| | 1617 | } else 0; |
| | 1618 | const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { |
| | 1619 | break :pre_footer first_footer_ni.prev(mf); |
| | 1620 | } else .wrap(last_ni); |
| | 1621 | const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { |
| | 1622 | const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); |
| | 1623 | break :end pre_footer_off + pre_footer_size; |
| | 1624 | } else 0; |
| | 1625 | |
| | 1626 | const min_file_offset = node_file_offset + pre_footer_end; |
| | 1627 | const max_file_offset = node_file_offset + old_size - footers_size; |
| | 1628 | // We can go anywhere between `min_file_offset` and `max_file_offset`. |
| | 1629 | const candidate_file_offset = mf.flags.block_size.forward(min_file_offset); |
| | 1630 | if (candidate_file_offset > max_file_offset) { |
| | 1631 | return false; |
| 1282 | } | 1632 | } |
| | 1633 | break :range_file_offset candidate_file_offset; |
| | 1634 | }; |
| | 1635 | assert(mf.flags.block_size.check(range_file_offset)); |
| | 1636 | |
| | 1637 | const range_size: u64 = range_size: { |
| | 1638 | // For this strategy to be valid, the number of bytes we insert needs to be compatible with |
| | 1639 | // the alignments of all nodes following us (and following our parents, their parents, etc). |
| | 1640 | // We also probably don't want to trigger too many "node moved" events, since doing that |
| | 1641 | // repeatedly could result in a lot of extra work. Therefore, while we traverse parents and |
| | 1642 | // siblings to check their alignment requirements, we will also set an arbitrary limit on |
| | 1643 | // the number of nodes we can move, and give up if we walk more than that. |
| | 1644 | const max_moved_nodes = 32; |
| | 1645 | var num_moved: u32 = 0; |
| | 1646 | var cur_ni = ni; |
| | 1647 | // Alignment required for `range_size`: initially the block size (required for the syscall), |
| | 1648 | // then updated as we traverse based on how the operation would affect surrounding nodes. |
| | 1649 | var need_range_align: Alignment = mf.flags.block_size.max(ni.alignment(mf)); |
| | 1650 | while (true) { |
| | 1651 | // `cur_ni` will grow as a result of the range insertion. Its size must be well-aligned. |
| | 1652 | need_range_align = need_range_align.max(cur_ni.alignment(mf)); |
| | 1653 | |
| | 1654 | // Siblings following `cur_ni` don't get bigger, but their offsets change. |
| | 1655 | while (cur_ni.next(mf).unwrap()) |next_ni| { |
| | 1656 | // Only floating children need well-aligned offsets. |
| | 1657 | if (next_ni.position(mf) == .floating) { |
| | 1658 | need_range_align = need_range_align.max(next_ni.alignment(mf)); |
| | 1659 | } |
| | 1660 | num_moved += 1; |
| | 1661 | if (num_moved > max_moved_nodes) return false; |
| | 1662 | cur_ni = next_ni; |
| | 1663 | } |
| | 1664 | |
| | 1665 | // Move up to the parent. |
| | 1666 | cur_ni = cur_ni.parent(mf).unwrap() orelse break; |
| | 1667 | } |
| | 1668 | // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment |
| | 1669 | // requirement to figure out whether we're actually going to insert a range. |
| | 1670 | if (need_range_align.check(requested_range_size)) { |
| | 1671 | break :range_size requested_range_size; |
| | 1672 | } |
| | 1673 | // Perhaps we're allowed to grow by more than `requested_range_size`? |
| | 1674 | switch (grow_mode) { |
| | 1675 | .exact => return false, |
| | 1676 | .minimum => { |
| | 1677 | const candidate_range_size = need_range_align.forward(min_range_size); |
| | 1678 | // Allow growing by up to 50% more than was requested. |
| | 1679 | if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { |
| | 1680 | break :range_size candidate_range_size; |
| | 1681 | } else { |
| | 1682 | return false; |
| | 1683 | } |
| | 1684 | }, |
| | 1685 | } |
| | 1686 | }; |
| | 1687 | |
| | 1688 | // This `range_size` is compatible with everyone's alignment requirements, and we won't move too |
| | 1689 | // many nodes, so let's do it! |
| | 1690 | |
| | 1691 | mf.memory_map.write(mf.io) catch |err| { |
| | 1692 | mf.io_err = switch (err) { |
| | 1693 | error.Canceled => |e| return e, |
| | 1694 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| | 1695 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| | 1696 | else => |e| e, |
| | 1697 | }; |
| | 1698 | return error.MappedFileIo; |
| | 1699 | }; |
| 1283 | | 1700 | |
| 1284 | ni.setLocationAssumeCapacity(mf, forward_offset, new_size); | 1701 | // If we happen to be inserting at the very end of the file, we need to resize the file instead |
| | 1702 | // of using `FALLOCATE_FL_INSERT_RANGE`. |
| | 1703 | if (range_file_offset == Node.Index.root.location(mf).resolve(mf)[1]) { |
| | 1704 | mf.memory_map.file.setLength(mf.io, range_file_offset + range_size) catch |err| switch (err) { |
| | 1705 | error.Canceled => |e| return e, |
| | 1706 | else => |e| { |
| | 1707 | mf.io_err = e; |
| | 1708 | return error.MappedFileIo; |
| | 1709 | }, |
| | 1710 | }; |
| 1285 | } else { | 1711 | } else { |
| 1286 | const temp_size = new_alignment.forward(@intCast(new_size + 1)); | 1712 | while (true) switch (linux.errno(linux.fallocate( |
| 1287 | try mf.resizeNode(gpa, ni, temp_size); | 1713 | mf.memory_map.file.handle, |
| 1288 | const new_offset, _ = ni.location(mf).resolve(mf); | 1714 | linux.FALLOC.FL_INSERT_RANGE, |
| 1289 | | 1715 | @intCast(range_file_offset), |
| 1290 | try mf.ensureCapacityForSetLocation(gpa); | 1716 | @intCast(range_size), |
| 1291 | | 1717 | ))) { |
| 1292 | // Non-fixed nodes may now be aligned if the resize moved them | 1718 | .SUCCESS => break, |
| 1293 | const new_forward_offset = new_alignment.forward(@intCast(new_offset)); | 1719 | .INTR => continue, |
| 1294 | const final_offset = if (new_forward_offset != new_offset) final_offset: { | 1720 | .NOSYS, .OPNOTSUPP => { |
| 1295 | if (node.flags.has_content) { | 1721 | // After all that setup work, it turns out the operation is actually unsupported! |
| 1296 | const old_file_offset = ni.fileLocation(mf, false).offset; | 1722 | mf.flags.fallocate_insert_range_unsupported = true; |
| 1297 | const new_file_offset = (old_file_offset - new_offset) + new_forward_offset; | 1723 | return false; |
| 1298 | @memmove( | 1724 | }, |
| 1299 | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], | 1725 | else => |e| { |
| 1300 | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], | 1726 | mf.io_err = switch (e) { |
| 1301 | ); | 1727 | .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above |
| 1302 | @memset(mf.memory_map.memory[@intCast(old_file_offset)..@intCast(new_file_offset)], 0); | 1728 | .BADF => unreachable, |
| | 1729 | .FBIG => unreachable, |
| | 1730 | .INVAL => unreachable, |
| | 1731 | .IO => error.InputOutput, |
| | 1732 | .NODEV => error.NotFile, |
| | 1733 | .NOSPC => error.NoSpaceLeft, |
| | 1734 | .PERM => error.PermissionDenied, |
| | 1735 | .SPIPE => error.Unseekable, |
| | 1736 | .TXTBSY => error.FileBusy, |
| | 1737 | else => std.posix.unexpectedErrno(e), |
| | 1738 | }; |
| | 1739 | return error.MappedFileIo; |
| | 1740 | }, |
| | 1741 | }; |
| | 1742 | } |
| | 1743 | |
| | 1744 | // We did it! Now to update all the sizes and offsets. This loop is exactly the same shape as |
| | 1745 | // above, except we're updating locations instead of checking alignments. |
| | 1746 | var cur_ni = ni; |
| | 1747 | while (true) { |
| | 1748 | const this_offset, const this_old_size = cur_ni.location(mf).resolve(mf); |
| | 1749 | if (cur_ni == .root) { |
| | 1750 | try mf.ensureTotalCapacityPrecise(@intCast(this_old_size + range_size)); |
| | 1751 | } |
| | 1752 | try cur_ni.setLocation(mf, gpa, this_offset, this_old_size + range_size); |
| | 1753 | |
| | 1754 | while (cur_ni.next(mf).unwrap()) |next_ni| { |
| | 1755 | const next_old_offset, const next_size = next_ni.location(mf).resolve(mf); |
| | 1756 | try next_ni.setLocation(mf, gpa, next_old_offset + range_size, next_size); |
| | 1757 | cur_ni = next_ni; |
| | 1758 | } |
| | 1759 | |
| | 1760 | cur_ni = cur_ni.parent(mf).unwrap() orelse break; |
| | 1761 | } |
| | 1762 | |
| | 1763 | // The only thing left is to update the offsets of any footers inside of `ni`. |
| | 1764 | if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { |
| | 1765 | var footer_ni = first_footer_ni; |
| | 1766 | while (true) { |
| | 1767 | const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); |
| | 1768 | try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); |
| | 1769 | footer_ni = footer_ni.next(mf).unwrap() orelse break; |
| | 1770 | } |
| | 1771 | } |
| | 1772 | |
| | 1773 | return true; |
| | 1774 | } |
| | 1775 | |
| | 1776 | /// Ensures that `parent_ni` has at least `extra_capacity` padding bytes following its current |
| | 1777 | /// headers, so that the headers can grow into that space. |
| | 1778 | fn ensureAdditionalHeaderCapacity( |
| | 1779 | mf: *MappedFile, |
| | 1780 | gpa: Allocator, |
| | 1781 | parent_ni: Node.Index, |
| | 1782 | extra_capacity: u64, |
| | 1783 | ) Error!void { |
| | 1784 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| | 1785 | |
| | 1786 | const last_header_oni = parent_ni.lastHeader(mf); |
| | 1787 | const first_footer_oni = parent_ni.firstFooter(mf); |
| | 1788 | |
| | 1789 | const headers_size: u64 = headers_size: { |
| | 1790 | const last_header_ni = last_header_oni.unwrap() orelse break :headers_size 0; |
| | 1791 | const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); |
| | 1792 | break :headers_size last_header_off + last_header_size; |
| | 1793 | }; |
| | 1794 | |
| | 1795 | const footers_size: u64 = footers_size: { |
| | 1796 | const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; |
| | 1797 | const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); |
| | 1798 | break :footers_size parent_size - first_footer_off; |
| | 1799 | }; |
| | 1800 | |
| | 1801 | const first_floating_oni: Node.Index.Optional = if (last_header_oni.unwrap()) |last_header_ni| first_floating: { |
| | 1802 | const after_header_ni = last_header_ni.next(mf).unwrap() orelse break :first_floating .none; |
| | 1803 | break :first_floating switch (after_header_ni.position(mf)) { |
| | 1804 | .header => unreachable, |
| | 1805 | .floating => .wrap(after_header_ni), |
| | 1806 | .footer => .none, |
| | 1807 | }; |
| | 1808 | } else first_floating: { |
| | 1809 | const first_ni = parent_ni.first(mf).unwrap() orelse break :first_floating .none; |
| | 1810 | break :first_floating switch (first_ni.position(mf)) { |
| | 1811 | .header => unreachable, |
| | 1812 | .floating => .wrap(first_ni), |
| | 1813 | .footer => .none, |
| | 1814 | }; |
| | 1815 | }; |
| | 1816 | const first_floating_ni = first_floating_oni.unwrap() orelse { |
| | 1817 | // This node has only headers and footers. |
| | 1818 | const min_parent_size = headers_size + extra_capacity + footers_size; |
| | 1819 | if (parent_size < min_parent_size) { |
| | 1820 | const new_parent_size = parent_ni.alignment(mf).forward( |
| | 1821 | min_parent_size +| min_parent_size / growth_factor, |
| | 1822 | ); |
| | 1823 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| | 1824 | } |
| | 1825 | return; |
| | 1826 | }; |
| | 1827 | |
| | 1828 | const last_floating_ni = if (first_footer_oni.unwrap()) |first_footer_ni| last_floating: { |
| | 1829 | break :last_floating first_footer_ni.prev(mf).unwrap().?; |
| | 1830 | } else last_floating: { |
| | 1831 | break :last_floating parent_ni.last(mf).unwrap().?; |
| | 1832 | }; |
| | 1833 | assert(last_floating_ni.position(mf) == .floating); // we know `parent_ni` contains at least `first_floating_ni` |
| | 1834 | |
| | 1835 | // Find the first floating child, if any, which does not overlap the new header space. |
| | 1836 | const first_good_floating_oni: Node.Index.Optional = first_good_floating: { |
| | 1837 | var floating_ni = first_floating_ni; |
| | 1838 | while (true) { |
| | 1839 | const floating_offset, _ = floating_ni.location(mf).resolve(mf); |
| | 1840 | if (floating_offset >= headers_size + extra_capacity) { |
| | 1841 | break :first_good_floating .wrap(floating_ni); |
| | 1842 | } |
| | 1843 | const next_ni = floating_ni.next(mf).unwrap() orelse { |
| | 1844 | break :first_good_floating .none; |
| | 1845 | }; |
| | 1846 | switch (next_ni.position(mf)) { |
| | 1847 | .header => unreachable, // after the last header |
| | 1848 | .floating => floating_ni = next_ni, |
| | 1849 | .footer => break :first_good_floating .none, |
| 1303 | } | 1850 | } |
| | 1851 | } |
| | 1852 | }; |
| | 1853 | |
| | 1854 | if (first_good_floating_oni == first_floating_ni.toOptional()) { |
| | 1855 | // None of the floating children are in our way! That means there's already enough space. |
| | 1856 | return; |
| | 1857 | } |
| | 1858 | |
| | 1859 | const last_moving_ni = if (first_good_floating_oni.unwrap()) |first_good_floating_ni| last_moving: { |
| | 1860 | break :last_moving first_good_floating_ni.prev(mf).unwrap().?; |
| | 1861 | } else last_moving: { |
| | 1862 | break :last_moving last_floating_ni; |
| | 1863 | }; |
| | 1864 | |
| | 1865 | // We are going to move all nodes between `first_floating_ni` and `last_moving_ni` to the end of |
| | 1866 | // the parent. We'll move all the node data in one big block. |
| | 1867 | |
| | 1868 | const moving_offset: u64 = first_floating_ni.location(mf).resolve(mf)[0]; |
| | 1869 | const moving_size: u64 = size: { |
| | 1870 | const last_moving_off, const last_moving_size = last_moving_ni.location(mf).resolve(mf); |
| | 1871 | break :size last_moving_off + last_moving_size - moving_offset; |
| | 1872 | }; |
| | 1873 | |
| | 1874 | var moving_alignment: Alignment = .@"1"; |
| | 1875 | var moving_has_content = false; // optimization: no need to move data if it's all uninitialized |
| | 1876 | { |
| | 1877 | var cur_ni = first_floating_ni; |
| | 1878 | while (true) { |
| | 1879 | moving_alignment = moving_alignment.max(cur_ni.alignment(mf)); |
| | 1880 | moving_has_content = moving_has_content or cur_ni.get(mf).flags.has_content; |
| | 1881 | if (cur_ni == last_moving_ni) break; |
| | 1882 | cur_ni = cur_ni.next(mf).unwrap().?; |
| | 1883 | } |
| | 1884 | } |
| | 1885 | |
| | 1886 | const first_free_offset = free_offset: { |
| | 1887 | const last_floating_off, const last_floating_size = last_floating_ni.location(mf).resolve(mf); |
| | 1888 | break :free_offset @max(last_floating_off + last_floating_size, headers_size + extra_capacity); |
| | 1889 | }; |
| | 1890 | // Alignment is a little tricky here. We don't necessarily want the new offset to be aligned to |
| | 1891 | // `moving_alignment` exactly, because if (e.g.) the first floating node is align(2) and the |
| | 1892 | // second is align(4), then the overall range we're moving may not be 4-byte aligned even though |
| | 1893 | // one of the nodes is. Instead, the old and new offsets must be congruent modulo the alignment. |
| | 1894 | const aligned_dest_offset = moving_alignment.forward(first_free_offset); |
| | 1895 | const dest_offset = aligned_dest_offset + (moving_offset - moving_alignment.backward(moving_offset)); |
| | 1896 | assert(dest_offset % moving_alignment.toByteUnits() == moving_offset % moving_alignment.toByteUnits()); |
| | 1897 | |
| | 1898 | // This expression is correct because `dest_offset` is after all floating nodes (except the ones |
| | 1899 | // we're moving there of course). |
| | 1900 | const min_parent_size = dest_offset + moving_size + footers_size; |
| | 1901 | if (parent_size < min_parent_size) { |
| | 1902 | const new_parent_size = parent_ni.alignment(mf).forward( |
| | 1903 | min_parent_size +| min_parent_size / growth_factor, |
| | 1904 | ); |
| | 1905 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| | 1906 | } |
| | 1907 | |
| | 1908 | if (moving_has_content) { |
| | 1909 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| | 1910 | try mf.moveRange( |
| | 1911 | parent_file_off + moving_offset, |
| | 1912 | parent_file_off + dest_offset, |
| | 1913 | moving_size, |
| | 1914 | ); |
| | 1915 | } |
| | 1916 | |
| | 1917 | // Remove everything between `first_floating_ni` and `last_moving_ni` from the linked list, then |
| | 1918 | // re-insert them in their new position. |
| | 1919 | try mf.removeNodesFromChildList(gpa, first_floating_ni, last_moving_ni); |
| | 1920 | try mf.addNodesToChildListBefore(gpa, first_footer_oni, first_floating_ni, last_moving_ni); |
| | 1921 | |
| | 1922 | // Finally, we need to update the locations of all of those nodes. |
| | 1923 | var cur_ni = first_floating_ni; |
| | 1924 | while (true) { |
| | 1925 | assert(cur_ni.position(mf) == .floating); |
| | 1926 | const old_offset, const old_size = cur_ni.location(mf).resolve(mf); |
| | 1927 | const new_offset = old_offset - moving_offset + dest_offset; |
| | 1928 | assert(cur_ni.alignment(mf).check(new_offset)); |
| | 1929 | try cur_ni.setLocation(mf, gpa, new_offset, old_size); |
| | 1930 | if (cur_ni == last_moving_ni) break; |
| | 1931 | cur_ni = cur_ni.next(mf).unwrap().?; |
| | 1932 | } |
| | 1933 | } |
| | 1934 | |
| | 1935 | /// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current |
| | 1936 | /// footers, so that the footers can grow into that space. |
| | 1937 | fn ensureAdditionalFooterCapacity( |
| | 1938 | mf: *MappedFile, |
| | 1939 | gpa: Allocator, |
| | 1940 | parent_ni: Node.Index, |
| | 1941 | extra_capacity: u64, |
| | 1942 | ) Error!void { |
| | 1943 | // This is way easier than the header case, because we don't need to actually move anything; we |
| | 1944 | // just need to expand the parent if there isn't space, and that will add padding after the |
| | 1945 | // parent's floating children, which is exactly where we want it. |
| | 1946 | |
| | 1947 | const first_footer_oni = parent_ni.firstFooter(mf); |
| | 1948 | |
| | 1949 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| | 1950 | |
| | 1951 | const footers_size: u64 = footers_size: { |
| | 1952 | const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; |
| | 1953 | const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); |
| | 1954 | break :footers_size parent_size - first_footer_off; |
| | 1955 | }; |
| | 1956 | |
| | 1957 | const header_and_floating_end: u64 = end: { |
| | 1958 | const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { |
| | 1959 | break :before_footers first_footer_ni.prev(mf); |
| | 1960 | } else before_footers: { |
| | 1961 | break :before_footers parent_ni.last(mf); |
| | 1962 | }; |
| | 1963 | const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; |
| | 1964 | const offset, const size = before_footers_ni.location(mf).resolve(mf); |
| | 1965 | break :end offset + size; |
| | 1966 | }; |
| 1304 | | 1967 | |
| 1305 | break :final_offset new_forward_offset; | 1968 | assert(header_and_floating_end + footers_size <= parent_size); |
| 1306 | } else new_offset; | | |
| 1307 | | 1969 | |
| 1308 | ni.setLocationAssumeCapacity(mf, final_offset, new_size); | 1970 | const min_parent_size = header_and_floating_end + footers_size + extra_capacity; |
| | 1971 | if (parent_size < min_parent_size) { |
| | 1972 | const new_parent_size = parent_ni.alignment(mf).forward( |
| | 1973 | min_parent_size +| min_parent_size / growth_factor, |
| | 1974 | ); |
| | 1975 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| 1309 | } | 1976 | } |
| 1310 | } | 1977 | } |
| 1311 | | 1978 | |
| | 1979 | fn removeNodesFromChildList( |
| | 1980 | mf: *MappedFile, |
| | 1981 | gpa: Allocator, |
| | 1982 | first_remove_ni: Node.Index, |
| | 1983 | last_remove_ni: Node.Index, |
| | 1984 | ) Allocator.Error!void { |
| | 1985 | const parent_ni = first_remove_ni.parent(mf).unwrap().?; |
| | 1986 | assert(last_remove_ni.parent(mf).unwrap().? == parent_ni); |
| | 1987 | |
| | 1988 | const prev_oni = first_remove_ni.prev(mf); |
| | 1989 | const next_oni = last_remove_ni.next(mf); |
| | 1990 | |
| | 1991 | if (prev_oni.unwrap()) |prev_ni| { |
| | 1992 | assert(prev_ni.next(mf).unwrap().? == first_remove_ni); |
| | 1993 | try prev_ni.setNext(gpa, next_oni, mf); |
| | 1994 | } else { |
| | 1995 | assert(parent_ni.first(mf).unwrap().? == first_remove_ni); |
| | 1996 | parent_ni.get(mf).first = next_oni; |
| | 1997 | } |
| | 1998 | |
| | 1999 | if (next_oni.unwrap()) |next_ni| { |
| | 2000 | assert(next_ni.prev(mf).unwrap().? == last_remove_ni); |
| | 2001 | next_ni.get(mf).prev = prev_oni; |
| | 2002 | } else { |
| | 2003 | assert(parent_ni.last(mf).unwrap().? == last_remove_ni); |
| | 2004 | parent_ni.get(mf).last = prev_oni; |
| | 2005 | } |
| | 2006 | } |
| | 2007 | /// Assumes `first_add_ni` and `last_add_ni` are connected, and that all nodes in between them |
| | 2008 | /// already have their `parent` field correctly populated. |
| | 2009 | /// |
| | 2010 | /// To add a single node, set `first_add_ni` equal to `last_add_ni`. |
| | 2011 | fn addNodesToChildListBefore( |
| | 2012 | mf: *MappedFile, |
| | 2013 | gpa: Allocator, |
| | 2014 | /// `null` means to add at the end of the parent. |
| | 2015 | next_oni: Node.Index.Optional, |
| | 2016 | first_add_ni: Node.Index, |
| | 2017 | last_add_ni: Node.Index, |
| | 2018 | ) Allocator.Error!void { |
| | 2019 | const parent_ni = first_add_ni.parent(mf).unwrap().?; |
| | 2020 | assert(last_add_ni.parent(mf).unwrap().? == parent_ni); |
| | 2021 | if (next_oni.unwrap()) |next_ni| { |
| | 2022 | assert(next_ni.parent(mf).unwrap().? == parent_ni); |
| | 2023 | } |
| | 2024 | |
| | 2025 | const prev_oni: Node.Index.Optional = if (next_oni.unwrap()) |next_ni| prev: { |
| | 2026 | break :prev next_ni.prev(mf); |
| | 2027 | } else prev: { |
| | 2028 | break :prev parent_ni.last(mf); |
| | 2029 | }; |
| | 2030 | |
| | 2031 | first_add_ni.get(mf).prev = prev_oni; |
| | 2032 | try last_add_ni.setNext(gpa, next_oni, mf); |
| | 2033 | |
| | 2034 | if (prev_oni.unwrap()) |prev_ni| { |
| | 2035 | assert(prev_ni.next(mf) == next_oni); |
| | 2036 | try prev_ni.setNext(gpa, .wrap(first_add_ni), mf); |
| | 2037 | } else { |
| | 2038 | assert(parent_ni.first(mf) == next_oni); |
| | 2039 | parent_ni.get(mf).first = .wrap(first_add_ni); |
| | 2040 | } |
| | 2041 | |
| | 2042 | if (next_oni.unwrap()) |next_ni| { |
| | 2043 | assert(next_ni.prev(mf) == prev_oni); |
| | 2044 | next_ni.get(mf).prev = .wrap(last_add_ni); |
| | 2045 | } else { |
| | 2046 | assert(parent_ni.last(mf) == prev_oni); |
| | 2047 | parent_ni.get(mf).last = .wrap(last_add_ni); |
| | 2048 | } |
| | 2049 | } |
| | 2050 | fn addNodesToChildListAfter( |
| | 2051 | mf: *MappedFile, |
| | 2052 | gpa: Allocator, |
| | 2053 | /// `null` means to add at the start of the parent. |
| | 2054 | prev_oni: Node.Index.Optional, |
| | 2055 | first_add_ni: Node.Index, |
| | 2056 | last_add_ni: Node.Index, |
| | 2057 | ) Allocator.Error!void { |
| | 2058 | const next_oni: Node.Index.Optional = next: { |
| | 2059 | if (prev_oni.unwrap()) |prev_ni| break :next prev_ni.next(mf); |
| | 2060 | const parent_ni = first_add_ni.parent(mf).unwrap().?; |
| | 2061 | break :next parent_ni.first(mf); |
| | 2062 | }; |
| | 2063 | return mf.addNodesToChildListBefore(gpa, next_oni, first_add_ni, last_add_ni); |
| | 2064 | } |
| | 2065 | |
| | 2066 | fn realignNode( |
| | 2067 | mf: *MappedFile, |
| | 2068 | gpa: Allocator, |
| | 2069 | ni: Node.Index, |
| | 2070 | new_alignment: Alignment, |
| | 2071 | ) Error!void { |
| | 2072 | mf.nodes_lock.assertUnlocked(); |
| | 2073 | |
| | 2074 | const old_offset, const old_size = ni.location(mf).resolve(mf); |
| | 2075 | |
| | 2076 | if (ni == .root or ni.position(mf) != .floating) { |
| | 2077 | // Only this node's size is aligned, not its offset. |
| | 2078 | if (!new_alignment.check(old_size)) { |
| | 2079 | assert(new_alignment.compare(.gt, ni.alignment(mf))); |
| | 2080 | try mf.growNode( |
| | 2081 | gpa, |
| | 2082 | ni, |
| | 2083 | new_alignment.forward(old_size), |
| | 2084 | .exact, // because `growNode` is not aware that the size needs to match `new_alignment` |
| | 2085 | ); |
| | 2086 | } |
| | 2087 | } else { |
| | 2088 | // This is a floating node, so its size and offset are both aligned. |
| | 2089 | if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { |
| | 2090 | assert(new_alignment.compare(.gt, ni.alignment(mf))); |
| | 2091 | try mf.growFloatingNodeWithAlignment( |
| | 2092 | gpa, |
| | 2093 | ni, |
| | 2094 | new_alignment, |
| | 2095 | new_alignment.forward(old_size), |
| | 2096 | .minimum, |
| | 2097 | ); |
| | 2098 | } |
| | 2099 | } |
| | 2100 | |
| | 2101 | ni.get(mf).flags.alignment = new_alignment; |
| | 2102 | } |
| | 2103 | |
| 1312 | fn updateWriters(mf: *MappedFile) void { | 2104 | fn updateWriters(mf: *MappedFile) void { |
| 1313 | var writers_it = mf.writers.first; | 2105 | var writers_it = mf.writers.first; |
| 1314 | while (writers_it) |writer_node| : (writers_it = writer_node.next) { | 2106 | while (writers_it) |writer_node| : (writers_it = writer_node.next) { |
| ... | @@ -1317,10 +2109,47 @@ fn updateWriters(mf: *MappedFile) void { | ... | @@ -1317,10 +2109,47 @@ fn updateWriters(mf: *MappedFile) void { |
| 1317 | } | 2109 | } |
| 1318 | } | 2110 | } |
| 1319 | | 2111 | |
| 1320 | fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { | 2112 | fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) Error!void { |
| 1321 | // make a copy of this node at the new location | 2113 | if (old_file_offset == new_file_offset) return; |
| 1322 | try mf.copyRange(old_file_offset, new_file_offset, size); | 2114 | |
| 1323 | // delete the copy of this node at the old location | 2115 | if (old_file_offset >= new_file_offset + size or |
| | 2116 | new_file_offset >= old_file_offset + size) |
| | 2117 | { |
| | 2118 | const n = try mf.copyFileRange( |
| | 2119 | mf.memory_map.file, |
| | 2120 | old_file_offset, |
| | 2121 | new_file_offset, |
| | 2122 | size, |
| | 2123 | ); |
| | 2124 | @memcpy( |
| | 2125 | mf.memory_map.memory[@intCast(new_file_offset + n)..][0..@intCast(size - n)], |
| | 2126 | mf.memory_map.memory[@intCast(old_file_offset + n)..][0..@intCast(size - n)], |
| | 2127 | ); |
| | 2128 | |
| | 2129 | try mf.zeroRange(old_file_offset, size); |
| | 2130 | |
| | 2131 | return; |
| | 2132 | } |
| | 2133 | |
| | 2134 | // TODO: if the non-overlapping region is greater than or equal to a filesystem block, is it |
| | 2135 | // ever worth doing multiple `copyFileRange` calls instead of a big `@memmove`? |
| | 2136 | |
| | 2137 | @memmove( |
| | 2138 | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], |
| | 2139 | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], |
| | 2140 | ); |
| | 2141 | |
| | 2142 | if (new_file_offset > old_file_offset) { |
| | 2143 | const clear_size = new_file_offset - old_file_offset; |
| | 2144 | assert(clear_size < size); |
| | 2145 | try mf.zeroRange(old_file_offset, clear_size); |
| | 2146 | } else { |
| | 2147 | const clear_size = old_file_offset - new_file_offset; |
| | 2148 | assert(clear_size < size); |
| | 2149 | try mf.zeroRange(new_file_offset + size, clear_size); |
| | 2150 | } |
| | 2151 | } |
| | 2152 | fn zeroRange(mf: *MappedFile, file_offset: u64, size: u64) Error!void { |
| 1324 | if (is_linux and | 2153 | if (is_linux and |
| 1325 | !mf.flags.fallocate_punch_hole_unsupported and | 2154 | !mf.flags.fallocate_punch_hole_unsupported and |
| 1326 | size >= mf.flags.block_size.toByteUnits() * 2 - 1) | 2155 | size >= mf.flags.block_size.toByteUnits() * 2 - 1) |
| ... | @@ -1328,147 +2157,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: | ... | @@ -1328,147 +2157,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: |
| 1328 | while (true) switch (linux.errno(linux.fallocate( | 2157 | while (true) switch (linux.errno(linux.fallocate( |
| 1329 | mf.memory_map.file.handle, | 2158 | mf.memory_map.file.handle, |
| 1330 | linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, | 2159 | linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, |
| 1331 | @intCast(old_file_offset), | 2160 | @intCast(file_offset), |
| 1332 | @intCast(size), | 2161 | @intCast(size), |
| 1333 | ))) { | 2162 | ))) { |
| 1334 | .SUCCESS => return, | 2163 | .SUCCESS => return, |
| 1335 | .INTR => continue, | 2164 | .INTR => continue, |
| 1336 | .BADF, .FBIG, .INVAL => unreachable, | | |
| 1337 | .IO => return error.InputOutput, | | |
| 1338 | .NODEV => return error.NotFile, | | |
| 1339 | .NOSPC => return error.NoSpaceLeft, | | |
| 1340 | .NOSYS, .OPNOTSUPP => { | 2165 | .NOSYS, .OPNOTSUPP => { |
| 1341 | mf.flags.fallocate_punch_hole_unsupported = true; | 2166 | mf.flags.fallocate_punch_hole_unsupported = true; |
| 1342 | break; // fall back to slow path | 2167 | break; // fall back to slow path |
| 1343 | }, | 2168 | }, |
| 1344 | .PERM => return error.PermissionDenied, | 2169 | else => |e| { |
| 1345 | .SPIPE => return error.Unseekable, | 2170 | mf.io_err = switch (e) { |
| 1346 | .TXTBSY => return error.FileBusy, | 2171 | .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above |
| 1347 | else => |e| return std.posix.unexpectedErrno(e), | 2172 | .BADF => unreachable, |
| | 2173 | .FBIG => unreachable, |
| | 2174 | .INVAL => unreachable, |
| | 2175 | .IO => error.InputOutput, |
| | 2176 | .NODEV => error.NotFile, |
| | 2177 | .NOSPC => error.NoSpaceLeft, |
| | 2178 | .PERM => error.PermissionDenied, |
| | 2179 | .SPIPE => error.Unseekable, |
| | 2180 | .TXTBSY => error.FileBusy, |
| | 2181 | else => std.posix.unexpectedErrno(e), |
| | 2182 | }; |
| | 2183 | return error.MappedFileIo; |
| | 2184 | }, |
| 1348 | }; | 2185 | }; |
| 1349 | } | 2186 | } |
| 1350 | @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0); | 2187 | @memset(mf.memory_map.memory[@intCast(file_offset)..][0..@intCast(size)], 0); |
| 1351 | } | | |
| 1352 | | | |
| 1353 | fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { | | |
| 1354 | const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size); | | |
| 1355 | if (copy_size < size) @memcpy( | | |
| 1356 | mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], | | |
| 1357 | mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], | | |
| 1358 | ); | | |
| 1359 | } | 2188 | } |
| 1360 | | | |
| 1361 | fn copyFileRange( | 2189 | fn copyFileRange( |
| 1362 | mf: *MappedFile, | 2190 | mf: *MappedFile, |
| 1363 | old_file: Io.File, | 2191 | old_file: Io.File, |
| 1364 | old_file_offset: u64, | 2192 | old_file_offset: u64, |
| 1365 | new_file_offset: u64, | 2193 | new_file_offset: u64, |
| 1366 | size: u64, | 2194 | size: u64, |
| 1367 | ) (Io.Cancelable || IoError)!u64 { | 2195 | ) Error!u64 { |
| | 2196 | if (!is_linux or mf.flags.copy_file_range_unsupported) { |
| | 2197 | return 0; |
| | 2198 | } |
| | 2199 | |
| | 2200 | const min_size = mf.flags.block_size.toByteUnits() * 2 - 1; |
| | 2201 | if (size < min_size) return 0; |
| | 2202 | |
| 1368 | const io = mf.io; | 2203 | const io = mf.io; |
| 1369 | mf.memory_map.write(io) catch |err| switch (err) { | 2204 | mf.memory_map.write(io) catch |err| { |
| 1370 | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking | 2205 | mf.io_err = switch (err) { |
| 1371 | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing | 2206 | error.Canceled => |e| return e, |
| 1372 | else => |e| return e, | 2207 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| | 2208 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| | 2209 | else => |e| e, |
| | 2210 | }; |
| | 2211 | return error.MappedFileIo; |
| 1373 | }; | 2212 | }; |
| 1374 | var remaining_size = size; | 2213 | var remaining_size = size; |
| 1375 | if (is_linux and !mf.flags.copy_file_range_unsupported) { | 2214 | var old_file_offset_mut: i64 = @intCast(old_file_offset); |
| 1376 | var old_file_offset_mut: i64 = @intCast(old_file_offset); | 2215 | var new_file_offset_mut: i64 = @intCast(new_file_offset); |
| 1377 | var new_file_offset_mut: i64 = @intCast(new_file_offset); | 2216 | while (remaining_size >= min_size) { |
| 1378 | while (remaining_size >= mf.flags.block_size.toByteUnits() * 2 - 1) { | 2217 | const copy_len = linux.copy_file_range( |
| 1379 | const copy_len = linux.copy_file_range( | 2218 | old_file.handle, |
| 1380 | old_file.handle, | 2219 | &old_file_offset_mut, |
| 1381 | &old_file_offset_mut, | 2220 | mf.memory_map.file.handle, |
| 1382 | mf.memory_map.file.handle, | 2221 | &new_file_offset_mut, |
| 1383 | &new_file_offset_mut, | 2222 | @intCast(remaining_size), |
| 1384 | @intCast(remaining_size), | 2223 | 0, |
| 1385 | 0, | 2224 | ); |
| 1386 | ); | 2225 | switch (linux.errno(copy_len)) { |
| 1387 | switch (linux.errno(copy_len)) { | 2226 | .SUCCESS => { |
| 1388 | .SUCCESS => { | 2227 | if (copy_len == 0) break; |
| 1389 | if (copy_len == 0) break; | 2228 | remaining_size -= copy_len; |
| 1390 | remaining_size -= copy_len; | 2229 | if (remaining_size == 0) break; |
| 1391 | if (remaining_size == 0) break; | 2230 | }, |
| 1392 | }, | 2231 | .INTR => continue, |
| 1393 | .INTR => continue, | 2232 | .NOSYS, .OPNOTSUPP, .XDEV => { |
| 1394 | .BADF, .FBIG, .INVAL, .OVERFLOW => unreachable, | 2233 | mf.flags.copy_file_range_unsupported = true; |
| 1395 | .IO => return error.InputOutput, | 2234 | break; |
| 1396 | .ISDIR => return error.IsDir, | 2235 | }, |
| 1397 | .NOMEM => return error.SystemResources, | 2236 | else => |e| { |
| 1398 | .NOSPC => return error.NoSpaceLeft, | 2237 | mf.io_err = switch (e) { |
| 1399 | .NOSYS, .OPNOTSUPP, .XDEV => { | 2238 | .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP, .XDEV => unreachable, // handled above |
| 1400 | mf.flags.copy_file_range_unsupported = true; | 2239 | .BADF => unreachable, |
| 1401 | break; | 2240 | .FBIG => unreachable, |
| 1402 | }, | 2241 | .INVAL => unreachable, |
| 1403 | .PERM => return error.PermissionDenied, | 2242 | .OVERFLOW => unreachable, |
| 1404 | .TXTBSY => return error.FileBusy, | 2243 | .IO => error.InputOutput, |
| 1405 | else => |e| return std.posix.unexpectedErrno(e), | 2244 | .ISDIR => error.IsDir, |
| 1406 | } | 2245 | .NOMEM => error.SystemResources, |
| | 2246 | .NOSPC => error.NoSpaceLeft, |
| | 2247 | .PERM => error.PermissionDenied, |
| | 2248 | .TXTBSY => error.FileBusy, |
| | 2249 | else => std.posix.unexpectedErrno(e), |
| | 2250 | }; |
| | 2251 | return error.MappedFileIo; |
| | 2252 | }, |
| 1407 | } | 2253 | } |
| 1408 | } | 2254 | } |
| 1409 | return size - remaining_size; | 2255 | return size - remaining_size; |
| 1410 | } | 2256 | } |
| 1411 | | 2257 | |
| 1412 | fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: Allocator) Allocator.Error!void { | | |
| 1413 | try mf.large.ensureUnusedCapacity(gpa, 2); | | |
| 1414 | try mf.updates.ensureUnusedCapacity(gpa, 2); | | |
| 1415 | } | | |
| 1416 | | | |
| 1417 | pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void { | 2258 | pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void { |
| 1418 | mf.ensureTotalCapacityInner(new_capacity) catch |err| switch (err) { | | |
| 1419 | error.OutOfMemory, | | |
| 1420 | error.Canceled, | | |
| 1421 | => |e| return e, | | |
| 1422 | | | |
| 1423 | else => |e| { | | |
| 1424 | mf.io_err = e; | | |
| 1425 | return error.MappedFileIo; | | |
| 1426 | }, | | |
| 1427 | }; | | |
| 1428 | } | | |
| 1429 | fn ensureTotalCapacityInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { | | |
| 1430 | if (mf.memory_map.memory.len >= new_capacity) return; | 2259 | if (mf.memory_map.memory.len >= new_capacity) return; |
| 1431 | try mf.ensureTotalCapacityPreciseInner(new_capacity +| new_capacity / growth_factor); | 2260 | try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor); |
| 1432 | } | 2261 | } |
| 1433 | | 2262 | |
| 1434 | pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void { | 2263 | pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void { |
| 1435 | mf.ensureTotalCapacityPreciseInner(new_capacity) catch |err| switch (err) { | | |
| 1436 | error.OutOfMemory, | | |
| 1437 | error.Canceled, | | |
| 1438 | => |e| return e, | | |
| 1439 | | | |
| 1440 | else => |e| { | | |
| 1441 | mf.io_err = e; | | |
| 1442 | return error.MappedFileIo; | | |
| 1443 | }, | | |
| 1444 | }; | | |
| 1445 | } | | |
| 1446 | fn ensureTotalCapacityPreciseInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { | | |
| 1447 | if (mf.memory_map.memory.len >= new_capacity) return; | 2264 | if (mf.memory_map.memory.len >= new_capacity) return; |
| 1448 | const io = mf.io; | 2265 | const io = mf.io; |
| 1449 | const aligned_capacity = mf.flags.block_size.forward(new_capacity); | 2266 | const aligned_capacity: usize = @intCast( |
| | 2267 | mf.flags.block_size.forward(new_capacity), |
| | 2268 | ); |
| 1450 | | 2269 | |
| 1451 | if (mf.memory_map.memory.len > 0) { | 2270 | if (mf.memory_map.memory.len > 0) { |
| 1452 | if (mf.memory_map.setLength(io, aligned_capacity)) |_| { | 2271 | if (mf.memory_map.setLength(io, aligned_capacity)) |_| { |
| 1453 | return; | 2272 | return; |
| 1454 | } else |err| switch (err) { | 2273 | } else |err| switch (err) { |
| 1455 | error.OperationUnsupported => {}, | 2274 | error.OperationUnsupported => {}, |
| 1456 | else => |e| return e, | 2275 | error.OutOfMemory, error.Canceled => |e| return e, |
| | 2276 | else => |e| { |
| | 2277 | mf.io_err = e; |
| | 2278 | return error.MappedFileIo; |
| | 2279 | }, |
| 1457 | } | 2280 | } |
| 1458 | | 2281 | |
| 1459 | mf.memory_map.write(io) catch |err| switch (err) { | 2282 | mf.memory_map.write(io) catch |err| { |
| 1460 | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking | 2283 | mf.io_err = switch (err) { |
| 1461 | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing | 2284 | error.Canceled => |e| return e, |
| 1462 | else => |e| return e, | 2285 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| | 2286 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| | 2287 | else => |e| e, |
| | 2288 | }; |
| | 2289 | return error.MappedFileIo; |
| 1463 | }; | 2290 | }; |
| 1464 | unmap(mf); | 2291 | unmap(mf); |
| 1465 | } | 2292 | } |
| 1466 | | 2293 | |
| 1467 | const file = mf.memory_map.file; | 2294 | const file = mf.memory_map.file; |
| 1468 | mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| switch (err) { | 2295 | mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| { |
| 1469 | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking | 2296 | mf.io_err = switch (err) { |
| 1470 | error.NotOpenForReading => return error.Unexpected, // we definitely opened the file for writing | 2297 | error.OutOfMemory, error.Canceled => |e| return e, |
| 1471 | else => |e| return e, | 2298 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| | 2299 | error.NotOpenForReading => error.Unexpected, // we definitely opened the file for writing |
| | 2300 | else => |e| e, |
| | 2301 | }; |
| | 2302 | return error.MappedFileIo; |
| 1472 | }; | 2303 | }; |
| 1473 | } | 2304 | } |
| 1474 | | 2305 | |
| ... | @@ -1487,7 +2318,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable || error{MappedFileIo})!void { | ... | @@ -1487,7 +2318,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable || error{MappedFileIo})!void { |
| 1487 | | 2318 | |
| 1488 | error.WouldBlock, // file was not opened as non-blocking | 2319 | error.WouldBlock, // file was not opened as non-blocking |
| 1489 | error.NotOpenForWriting, // we definitely opened the file for writing | 2320 | error.NotOpenForWriting, // we definitely opened the file for writing |
| 1490 | error.ReadOnlyFileSystem, | 2321 | error.ReadOnlyFileSystem, // again, we opened the file for writing |
| 1491 | => { | 2322 | => { |
| 1492 | mf.io_err = error.Unexpected; | 2323 | mf.io_err = error.Unexpected; |
| 1493 | return error.MappedFileIo; | 2324 | return error.MappedFileIo; |
| ... | @@ -1512,211 +2343,276 @@ fn verify(mf: *MappedFile) void { | ... | @@ -1512,211 +2343,276 @@ fn verify(mf: *MappedFile) void { |
| 1512 | assert(root.next == .none); | 2343 | assert(root.next == .none); |
| 1513 | mf.verifyNode(.root); | 2344 | mf.verifyNode(.root); |
| 1514 | } | 2345 | } |
| 1515 | | | |
| 1516 | fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { | 2346 | fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { |
| 1517 | const parent = parent_ni.get(mf); | 2347 | const parent = parent_ni.get(mf); |
| 1518 | const parent_offset, const parent_size = parent.location().resolve(mf); | 2348 | _, const parent_size = parent.location().resolve(mf); |
| 1519 | var prev_ni: Node.Index = .none; | 2349 | |
| | 2350 | var prev_oni: Node.Index.Optional = .none; |
| 1520 | var prev_end: u64 = 0; | 2351 | var prev_end: u64 = 0; |
| 1521 | var ni = parent.first; | 2352 | var prev_pos: Node.Position = .header; |
| 1522 | while (true) { | 2353 | var oni = parent.first; |
| 1523 | if (ni == .none) { | 2354 | while (oni.unwrap()) |ni| { |
| 1524 | assert(parent.last == prev_ni); | | |
| 1525 | return; | | |
| 1526 | } | | |
| 1527 | const node = ni.get(mf); | 2355 | const node = ni.get(mf); |
| 1528 | assert(node.parent == parent_ni); | 2356 | assert(node.parent == parent_ni.toOptional()); |
| | 2357 | assert(node.prev == prev_oni); |
| | 2358 | |
| 1529 | const offset, const size = node.location().resolve(mf); | 2359 | const offset, const size = node.location().resolve(mf); |
| 1530 | assert(node.flags.alignment.check(@intCast(offset))); | | |
| 1531 | assert(node.flags.alignment.check(@intCast(size))); | | |
| 1532 | const end = offset + size; | 2360 | const end = offset + size; |
| 1533 | assert(end <= parent_offset + parent_size); | 2361 | |
| | 2362 | assert(node.flags.alignment.check(size)); |
| 1534 | assert(offset >= prev_end); | 2363 | assert(offset >= prev_end); |
| 1535 | assert(node.prev == prev_ni); | 2364 | assert(end <= parent_size); |
| | 2365 | |
| | 2366 | switch (node.flags.position) { |
| | 2367 | .header => { |
| | 2368 | assert(prev_pos == .header); |
| | 2369 | assert(offset == prev_end); |
| | 2370 | }, |
| | 2371 | .floating => { |
| | 2372 | assert(prev_pos != .footer); |
| | 2373 | assert(node.flags.alignment.check(offset)); |
| | 2374 | }, |
| | 2375 | .footer => { |
| | 2376 | if (prev_pos == .footer) assert(offset == prev_end); |
| | 2377 | }, |
| | 2378 | } |
| | 2379 | |
| 1536 | mf.verifyNode(ni); | 2380 | mf.verifyNode(ni); |
| 1537 | prev_ni = ni; | 2381 | |
| | 2382 | prev_oni = .wrap(ni); |
| 1538 | prev_end = end; | 2383 | prev_end = end; |
| 1539 | ni = node.next; | 2384 | prev_pos = ni.position(mf); |
| | 2385 | |
| | 2386 | oni = node.next; |
| | 2387 | } |
| | 2388 | assert(parent.last == prev_oni); |
| | 2389 | if (prev_pos == .footer) { |
| | 2390 | assert(prev_end == parent_size); |
| 1540 | } | 2391 | } |
| 1541 | } | 2392 | } |
| 1542 | | 2393 | |
| 1543 | const testing = std.testing; | 2394 | test "fuzz node operations" { |
| 1544 | fn testVerifyContent(mf: *@This(), ni: Node.Index, value: u8, init_len: usize) !void { | 2395 | try std.testing.fuzz({}, fuzzOneNodeOperations, .{}); |
| 1545 | // Not using std.mem.allEqual, so we can get useful output | | |
| 1546 | const slice = ni.slice(mf); | | |
| 1547 | var buf: [256]u8 = undefined; | | |
| 1548 | @memset(buf[0..init_len], value); | | |
| 1549 | @memset(buf[init_len..], 0); | | |
| 1550 | try testing.expectEqualSlices(u8, buf[0..slice.len], slice); | | |
| 1551 | } | 2396 | } |
| | 2397 | fn fuzzOneNodeOperations(_: void, smith: *std.testing.Smith) anyerror!void { |
| | 2398 | const gpa = std.testing.allocator; |
| | 2399 | const io = std.testing.io; |
| 1552 | | 2400 | |
| 1553 | test { | 2401 | var tmp_dir = std.testing.tmpDir(.{}); |
| 1554 | const gpa = testing.allocator; | | |
| 1555 | | | |
| 1556 | var tmp_dir = testing.tmpDir(.{}); | | |
| 1557 | defer tmp_dir.cleanup(); | 2402 | defer tmp_dir.cleanup(); |
| 1558 | | 2403 | |
| 1559 | var file = try tmp_dir.dir.createFile(testing.io, "test.mf", .{ .read = true }); | 2404 | var tmp_file = try tmp_dir.dir.createFile(io, "test.mf", .{ .read = true }); |
| 1560 | defer file.close(testing.io); | 2405 | defer tmp_file.close(io); |
| 1561 | | 2406 | |
| 1562 | var mf = try init(file, gpa, testing.io); | 2407 | var mf: MappedFile = try .init(tmp_file, gpa, io); |
| 1563 | defer mf.deinit(gpa); | 2408 | defer mf.deinit(gpa); |
| 1564 | | 2409 | |
| 1565 | const a = try mf.addFirstChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); | 2410 | var nodes: std.array_hash_map.Auto(MappedFile.Node.Index, struct { |
| 1566 | const c = try mf.addLastChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); | 2411 | parent: MappedFile.Node.Index.Optional, |
| 1567 | const b = try mf.addNodeAfter(gpa, a, .{ .fixed = true, .alignment = .@"16" }); | 2412 | position: MappedFile.Node.Position, |
| 1568 | const d = try mf.addNodeAfter(gpa, b, .{ .alignment = .@"4" }); | 2413 | num_headers: u32, |
| | 2414 | num_footers: u32, |
| | 2415 | /// For leaf nodes, this value is whether we have initialized the contents of the node or |
| | 2416 | /// not. For non-leaf nodes, this value is unspecified and should be ignored. |
| | 2417 | initialized: bool, |
| | 2418 | }) = .empty; |
| | 2419 | defer nodes.deinit(gpa); |
| | 2420 | |
| | 2421 | // When initializing a leaf node, we will place its 4-byte node index at the start of its range, |
| | 2422 | // and the bitwise NOT of its node index at the end of its range (both little-endian). This is |
| | 2423 | // just a simple way to put distinct values we can validate at all node boundaries. |
| | 2424 | |
| | 2425 | try nodes.putNoClobber(gpa, .root, .{ |
| | 2426 | .parent = .none, |
| | 2427 | .position = .floating, |
| | 2428 | .num_headers = 0, |
| | 2429 | .num_footers = 0, |
| | 2430 | .initialized = false, |
| | 2431 | }); |
| | 2432 | |
| | 2433 | // Allow a range of alignments, with most nodes having a small alignment of 1--32 bytes (most |
| | 2434 | // commonly 1 byte), but with a small chance for some large alignments too. |
| | 2435 | const alignment_weights: []const std.testing.Smith.Weight = comptime &.{ |
| | 2436 | .value(Alignment, .@"1", 20), |
| | 2437 | .value(Alignment, .@"2", 5), |
| | 2438 | .value(Alignment, .@"4", 5), |
| | 2439 | .value(Alignment, .@"8", 5), |
| | 2440 | .value(Alignment, .@"16", 5), |
| | 2441 | .value(Alignment, .@"32", 5), |
| | 2442 | .value(Alignment, .fromByteUnits(0x200), 1), |
| | 2443 | .value(Alignment, .fromByteUnits(0x400), 1), |
| | 2444 | .value(Alignment, .fromByteUnits(0x800), 1), |
| | 2445 | .value(Alignment, .fromByteUnits(0x1000), 1), |
| | 2446 | .value(Alignment, .fromByteUnits(0x2000), 1), |
| | 2447 | .value(Alignment, .fromByteUnits(0x4000), 1), |
| | 2448 | .value(Alignment, .fromByteUnits(0x8000), 1), |
| | 2449 | }; |
| 1569 | | 2450 | |
| 1570 | const a_init_size = 8; | 2451 | const min_nonzero_size = 2 * @sizeOf(MappedFile.Node.Index); |
| 1571 | const b_init_size = 16; | 2452 | const max_size = 0x10_000; |
| 1572 | const c_init_size = 24; | 2453 | const initial_size_weights: []const std.testing.Smith.Weight = comptime &.{ |
| 1573 | const d_init_size = 28; | 2454 | // initially, make nodes just as likely to be empty as non-empty |
| | 2455 | .value(u64, 0, max_size - min_nonzero_size + 1), |
| | 2456 | .rangeAtMost(u64, min_nonzero_size, max_size, 1), |
| | 2457 | }; |
| 1574 | | 2458 | |
| 1575 | // Resize without content | 2459 | while (!smith.eos()) switch (smith.value(enum { add, resize, realign })) { |
| 1576 | { | 2460 | .add => { |
| 1577 | // Verify size is aligned forward | 2461 | const parent_ni = nodes.keys()[smith.index(nodes.count())]; |
| 1578 | try d.resize(&mf, gpa, d_init_size - 1); | | |
| 1579 | try a.resize(&mf, gpa, a_init_size - 2); | | |
| 1580 | try c.resize(&mf, gpa, c_init_size); | | |
| 1581 | try b.resize(&mf, gpa, b_init_size); | | |
| 1582 | mf.verify(); | | |
| 1583 | | | |
| 1584 | const a_loc, const a_size = a.location(&mf).resolve(&mf); | | |
| 1585 | const b_loc, const b_size = b.location(&mf).resolve(&mf); | | |
| 1586 | const c_loc, const c_size = c.location(&mf).resolve(&mf); | | |
| 1587 | _, const d_size = d.location(&mf).resolve(&mf); | | |
| 1588 | try testing.expect(a_size >= a_init_size); | | |
| 1589 | try testing.expect(b_size >= b_init_size); | | |
| 1590 | try testing.expect(c_size >= c_init_size); | | |
| 1591 | try testing.expect(d_size >= d_init_size); | | |
| 1592 | try testing.expect(b_loc >= a_loc + a_size); | | |
| 1593 | try testing.expect(c_loc >= b_loc + b_size); | | |
| 1594 | } | | |
| 1595 | | 2462 | |
| 1596 | const a_exp_size = 24; | 2463 | const alignment = smith.valueWeighted(Alignment, alignment_weights); |
| 1597 | const b_exp_size = 28; | 2464 | const size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); |
| 1598 | const c_exp_size = 48; | | |
| 1599 | const d_exp_size = 32; | | |
| 1600 | | 2465 | |
| 1601 | // Resize with content | 2466 | const position = smith.valueWeighted(Node.Position, comptime &.{ |
| 1602 | { | 2467 | // make floating nodes more common than header and footer nodes |
| 1603 | @memset(a.slice(&mf)[0..a_init_size], 0xaa); | 2468 | .value(Node.Position, .header, 1), |
| 1604 | @memset(b.slice(&mf)[0..b_init_size], 0xbb); | 2469 | .value(Node.Position, .footer, 1), |
| 1605 | @memset(c.slice(&mf)[0..c_init_size], 0xcc); | 2470 | .value(Node.Position, .floating, 4), |
| 1606 | @memset(d.slice(&mf)[0..d_init_size], 0xdd); | 2471 | }); |
| 1607 | | 2472 | const new_ni: Node.Index = switch (position) { |
| 1608 | try a.resize(&mf, gpa, a_exp_size); | 2473 | .header => new_ni: { |
| 1609 | try b.resize(&mf, gpa, b_exp_size); | 2474 | const parent_info = nodes.getPtr(parent_ni).?; |
| 1610 | try c.resize(&mf, gpa, c_exp_size); | 2475 | const prev_oni: Node.Index.Optional = prev_oni: { |
| 1611 | try d.resize(&mf, gpa, d_exp_size); | 2476 | const n = smith.valueRangeAtMost(u32, 0, parent_info.num_headers); |
| 1612 | mf.verify(); | 2477 | if (n == 0) break :prev_oni .none; |
| 1613 | | 2478 | var cur_ni = parent_ni.first(&mf).unwrap().?; |
| 1614 | const a_loc, const a_size = a.location(&mf).resolve(&mf); | 2479 | for (1..n) |_| cur_ni = cur_ni.next(&mf).unwrap().?; |
| 1615 | const b_loc, const b_size = b.location(&mf).resolve(&mf); | 2480 | break :prev_oni .wrap(cur_ni); |
| 1616 | const c_loc, const c_size = c.location(&mf).resolve(&mf); | 2481 | }; |
| 1617 | _, const d_size = d.location(&mf).resolve(&mf); | 2482 | const new_ni = try parent_ni.addHeaderChildAfter(&mf, gpa, prev_oni, .{ |
| 1618 | try testing.expect(a_size >= a_exp_size); | 2483 | .size = size, |
| 1619 | try testing.expect(b_size >= b_exp_size); | 2484 | .alignment = alignment, |
| 1620 | try testing.expect(c_size >= c_exp_size); | 2485 | }); |
| 1621 | try testing.expect(d_size >= d_exp_size); | 2486 | parent_info.num_headers += 1; |
| 1622 | try testing.expect(b_loc >= a_loc + a_size); | 2487 | break :new_ni new_ni; |
| 1623 | try testing.expect(c_loc >= b_loc + b_size); | 2488 | }, |
| 1624 | | | |
| 1625 | try testVerifyContent(&mf, a, 0xaa, a_init_size); | | |
| 1626 | try testVerifyContent(&mf, b, 0xbb, b_init_size); | | |
| 1627 | try testVerifyContent(&mf, c, 0xcc, c_init_size); | | |
| 1628 | try testVerifyContent(&mf, d, 0xdd, d_init_size); | | |
| 1629 | } | | |
| 1630 | | 2489 | |
| 1631 | const child_init: []const struct { Alignment, usize } = &.{ | 2490 | .floating => try parent_ni.addFloatingChild(&mf, gpa, .{ |
| 1632 | .{ .@"16", 16 }, | 2491 | .size = size, |
| 1633 | .{ .@"1", 1 }, | 2492 | .alignment = alignment, |
| 1634 | .{ .@"1", 19 }, | 2493 | }), |
| 1635 | .{ .@"1", 3 }, | 2494 | |
| 1636 | .{ .@"8", 30 }, | 2495 | .footer => new_ni: { |
| 1637 | .{ .@"2", 5 }, | 2496 | const parent_info = nodes.getPtr(parent_ni).?; |
| 1638 | .{ .@"1", 60 }, | 2497 | const next_oni: Node.Index.Optional = next_oni: { |
| 1639 | .{ .@"2", 2 }, | 2498 | const n = smith.valueRangeAtMost(u32, 0, parent_info.num_footers); |
| 1640 | .{ .@"16", 32 }, | 2499 | if (n == 0) break :next_oni .none; |
| 1641 | }; | 2500 | var cur_ni = parent_ni.last(&mf).unwrap().?; |
| | 2501 | for (1..n) |_| cur_ni = cur_ni.prev(&mf).unwrap().?; |
| | 2502 | break :next_oni .wrap(cur_ni); |
| | 2503 | }; |
| | 2504 | const new_ni = try parent_ni.addFooterChildBefore(&mf, gpa, next_oni, .{ |
| | 2505 | .size = size, |
| | 2506 | .alignment = alignment, |
| | 2507 | }); |
| | 2508 | parent_info.num_footers += 1; |
| | 2509 | break :new_ni new_ni; |
| | 2510 | }, |
| | 2511 | }; |
| 1642 | | 2512 | |
| 1643 | var children: [child_init.len]Node.Index = undefined; | 2513 | const initialize = size > 0 and smith.value(bool); |
| | 2514 | if (initialize) { |
| | 2515 | const slice = new_ni.slice(&mf); |
| | 2516 | std.mem.writeInt(u32, slice[0..4], @backingInt(new_ni), .little); |
| | 2517 | std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(new_ni), .little); |
| | 2518 | } |
| 1644 | | 2519 | |
| 1645 | // Differently-aligned fixed sibling nodes | 2520 | try nodes.putNoClobber(gpa, new_ni, .{ |
| 1646 | { | 2521 | .parent = .wrap(parent_ni), |
| 1647 | for (children[0 .. children.len - 1], child_init[0 .. children.len - 1], 0..) |*ni, opts, i| { | 2522 | .position = position, |
| 1648 | ni.* = try mf.addLastChildNode(gpa, b, .{ | 2523 | .num_headers = 0, |
| 1649 | .alignment = opts.@"0", | 2524 | .num_footers = 0, |
| 1650 | .size = opts.@"1", | 2525 | .initialized = initialize, |
| 1651 | .fixed = true, | | |
| 1652 | }); | 2526 | }); |
| | 2527 | }, |
| 1653 | | 2528 | |
| 1654 | @memset(ni.slice(&mf)[0..opts.@"1"], @intCast(i + 1)); | 2529 | .resize => { |
| 1655 | } | 2530 | const ni = nodes.keys()[smith.index(nodes.count())]; |
| 1656 | // Shift differently-aligned nodes by inserting a node | 2531 | const node_info = nodes.getPtr(ni).?; |
| 1657 | children[children.len - 1] = try mf.addNodeAfter(gpa, children[3], .{ | | |
| 1658 | .alignment = child_init[children.len - 1].@"0", | | |
| 1659 | .size = child_init[children.len - 1].@"1", | | |
| 1660 | .fixed = true, | | |
| 1661 | }); | | |
| 1662 | @memset(children[children.len - 1].slice(&mf), @intCast(children.len)); | | |
| 1663 | | | |
| 1664 | mf.verify(); | | |
| 1665 | for (children, child_init, 0..) |ni, opts, i| { | | |
| 1666 | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); | | |
| 1667 | } | | |
| 1668 | } | | |
| 1669 | | 2532 | |
| 1670 | // Shifting child nodes forward due via resize of parent.prev | 2533 | const alignment = ni.alignment(&mf); |
| 1671 | { | | |
| 1672 | try testing.expect(a.location(&mf).resolve(&mf)[1] < 64); | | |
| 1673 | try a.resize(&mf, gpa, 64); | | |
| 1674 | | 2534 | |
| 1675 | try testVerifyContent(&mf, a, 0xaa, a_init_size); | 2535 | if (ni.first(&mf) == .none and smith.value(bool)) { |
| 1676 | try testVerifyContent(&mf, c, 0xcc, c_init_size); | 2536 | // Since this is a leaf node, we can use `resizeLeaf`. |
| 1677 | try testVerifyContent(&mf, d, 0xdd, d_init_size); | 2537 | const new_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); |
| 1678 | for (children, child_init, 0..) |ni, opts, i| { | 2538 | try ni.resizeLeaf(&mf, gpa, new_size); |
| 1679 | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); | 2539 | if (new_size == 0) { |
| 1680 | } | 2540 | node_info.initialized = false; |
| 1681 | } | 2541 | } |
| | 2542 | } else { |
| | 2543 | const min_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); |
| | 2544 | try ni.ensureMinimumSize(&mf, gpa, min_size); |
| | 2545 | } |
| 1682 | | 2546 | |
| 1683 | // Re-align last node into trailing free space within parent | 2547 | if (ni.first(&mf) == .none) { |
| 1684 | { | 2548 | // This is a leaf node, so it can contain data. |
| 1685 | try b.resize(&mf, gpa, b.location(&mf).resolve(&mf)[1] + 64); | 2549 | if (node_info.initialized) { |
| | 2550 | // It's already initialized, so we'll write the expected footer at the new end. |
| | 2551 | const slice = ni.slice(&mf); |
| | 2552 | std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); |
| | 2553 | } else if (ni.location(&mf).resolve(&mf)[1] > 0) { |
| | 2554 | // It was uninitialized, but it has a non-zero size, so maybe we'd like to |
| | 2555 | // initialize it now? |
| | 2556 | if (smith.value(bool)) { |
| | 2557 | node_info.initialized = true; |
| | 2558 | const slice = ni.slice(&mf); |
| | 2559 | std.mem.writeInt(u32, slice[0..4], @backingInt(ni), .little); |
| | 2560 | std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); |
| | 2561 | } |
| | 2562 | } |
| | 2563 | } |
| | 2564 | }, |
| | 2565 | .realign => { |
| | 2566 | const ni = nodes.keys()[smith.index(nodes.count())]; |
| | 2567 | const new_alignment = smith.valueWeighted(Alignment, alignment_weights); |
| | 2568 | if (new_alignment.compare(.gt, ni.alignment(&mf))) { |
| | 2569 | _, const old_size = ni.location(&mf).resolve(&mf); |
| | 2570 | try ni.realign(&mf, gpa, new_alignment); |
| | 2571 | if (ni.first(&mf) == .none and nodes.get(ni).?.initialized) { |
| | 2572 | const slice = ni.slice(&mf); |
| | 2573 | @memmove(slice[slice.len - 4 ..][0..4], slice[old_size - 4 ..][0..4]); |
| | 2574 | } |
| | 2575 | } |
| | 2576 | }, |
| | 2577 | }; |
| 1686 | | 2578 | |
| 1687 | const last = children[children.len - 2]; | 2579 | mf.verify(); |
| 1688 | try last.realign(&mf, gpa, .@"4", true); | | |
| 1689 | mf.verify(); | | |
| 1690 | | 2580 | |
| 1691 | for (children, child_init, 0..) |ni, opts, i| | 2581 | for (nodes.keys(), nodes.values()) |ni, expected| { |
| 1692 | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); | 2582 | try std.testing.expectEqual(expected.parent, ni.parent(&mf)); |
| 1693 | try testVerifyContent(&mf, c, 0xcc, c_init_size); | 2583 | if (ni != .root) { |
| 1694 | } | 2584 | try std.testing.expectEqual(expected.position, ni.position(&mf)); |
| | 2585 | } |
| 1695 | | 2586 | |
| 1696 | // Re-align, shifting sibling nodes | 2587 | { |
| 1697 | { | 2588 | var num_headers: u32 = 0; |
| 1698 | try children[1].realign(&mf, gpa, .@"8", true); | 2589 | var header_oni = ni.lastHeader(&mf); |
| 1699 | mf.verify(); | 2590 | while (header_oni.unwrap()) |header_ni| { |
| | 2591 | num_headers += 1; |
| | 2592 | header_oni = header_ni.prev(&mf); |
| | 2593 | } |
| | 2594 | try std.testing.expectEqual(expected.num_headers, num_headers); |
| | 2595 | } |
| 1700 | | 2596 | |
| 1701 | for (children, child_init, 0..) |ni, opts, i| | 2597 | { |
| 1702 | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); | 2598 | var num_footers: u32 = 0; |
| 1703 | try testVerifyContent(&mf, c, 0xcc, c_init_size); | 2599 | var footer_oni = ni.firstFooter(&mf); |
| 1704 | } | 2600 | while (footer_oni.unwrap()) |footer_ni| { |
| | 2601 | num_footers += 1; |
| | 2602 | footer_oni = footer_ni.next(&mf); |
| | 2603 | } |
| | 2604 | try std.testing.expectEqual(expected.num_footers, num_footers); |
| | 2605 | } |
| 1705 | | 2606 | |
| 1706 | // Shrink and shift start of trailing node into free space | 2607 | if (ni.first(&mf) == .none and expected.initialized) { |
| 1707 | { | 2608 | const slice = ni.sliceConst(&mf); |
| 1708 | try mf.shrinkNode(gpa, a, 16, true); | 2609 | if (slice.len > 0) { |
| 1709 | mf.verify(); | 2610 | try std.testing.expect(slice.len >= min_nonzero_size); |
| 1710 | | 2611 | const header = std.mem.readInt(u32, slice[0..4], .little); |
| 1711 | const a_loc, const a_size = a.location(&mf).resolve(&mf); | 2612 | const footer = std.mem.readInt(u32, slice[slice.len - 4 ..][0..4], .little); |
| 1712 | const b_loc, _ = b.location(&mf).resolve(&mf); | 2613 | try std.testing.expectEqual(@backingInt(ni), header); |
| 1713 | try testing.expectEqual(b_loc, a_loc + a_size); | 2614 | try std.testing.expectEqual(~@backingInt(ni), footer); |
| 1714 | | 2615 | } |
| 1715 | try testVerifyContent(&mf, a, 0xaa, a_init_size); | | |
| 1716 | try testVerifyContent(&mf, c, 0xcc, c_init_size); | | |
| 1717 | try testVerifyContent(&mf, d, 0xdd, d_init_size); | | |
| 1718 | for (children, child_init, 0..) |ni, opts, i| { | | |
| 1719 | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); | | |
| 1720 | } | 2616 | } |
| 1721 | } | 2617 | } |
| 1722 | } | 2618 | } |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-08-23 11:21:31+01:00
Commit is signed but in an unrecognized format.