| ... | ... | @@ -183,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel |
| 183 | 183 | .fallocate_insert_range_unsupported = false, |
| 184 | 184 | .fallocate_punch_hole_unsupported = false, |
| 185 | 185 | }; |
| 186 | | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 187 | | const root_ni = try mf.addNode(gpa, .{ .add_node = .{ |
| 188 | | .size = size, |
| 189 | | .alignment = mf.flags.block_size, |
| 190 | | .fixed = true, |
| 191 | | } }); |
| 192 | | assert(root_ni == .root); |
| 193 | | try mf.ensureTotalCapacityInner(@intCast(size)); |
| 186 | |
| 187 | const root_location: Node.Location = l: { |
| 188 | if (std.math.cast(u32, size)) |small_size| { |
| 189 | break :l .{ .small = .{ .offset = 0, .size = small_size } }; |
| 190 | } |
| 191 | try mf.large.appendSlice(gpa, &.{ 0, size }); |
| 192 | break :l .{ .large = .{ .index = 0 } }; |
| 193 | }; |
| 194 | try mf.nodes.append(gpa, .{ |
| 195 | .parent = .none, |
| 196 | .prev = .none, |
| 197 | .next = .none, |
| 198 | .first = .none, |
| 199 | .last = .none, |
| 200 | .flags = .{ |
| 201 | .alignment = mf.flags.block_size, |
| 202 | .position = .floating, |
| 203 | .bubbles_moved = true, |
| 204 | .enable_next_moved = false, |
| 205 | .location_tag = root_location, |
| 206 | .moved = false, |
| 207 | .resized = false, |
| 208 | .next_moved = false, |
| 209 | .has_content = false, |
| 210 | }, |
| 211 | .location_payload = switch (root_location) { |
| 212 | .small => |small| .{ .small = small }, |
| 213 | .large => |large| .{ .large = large }, |
| 214 | }, |
| 215 | }); |
| 216 | |
| 217 | mf.ensureTotalCapacity(@intCast(size)) catch |err| switch (err) { |
| 218 | error.MappedFileIo => return mf.io_err.?, |
| 219 | else => |e| return e, |
| 220 | }; |
| 221 | |
| 194 | 222 | return mf; |
| 195 | 223 | } |
| 196 | 224 | |
| ... | ... | @@ -213,24 +241,53 @@ pub const Node = extern struct { |
| 213 | 241 | flags: Flags, |
| 214 | 242 | location_payload: Location.Payload, |
| 215 | 243 | |
| 244 | /// Any non-leaf node may designate its first N children as "header" nodes. This means that its |
| 245 | /// first N children must be densely packed together and positioned at the start of the parent. |
| 246 | /// The implementation guarantees that it will never re-order these nodes, nor will it introduce |
| 247 | /// padding between them. |
| 248 | /// |
| 249 | /// Likewise, any non-leaf node may designate its *last* M children as "footer" nodes, which are |
| 250 | /// like header nodes except they are positioned at the *end* of the parent rather than the |
| 251 | /// start. |
| 252 | /// |
| 253 | /// Nodes which are neither headers nor footers are called "floating". The implementation is |
| 254 | /// always free to re-order floating nodes relative to one another, and to add or remove padding |
| 255 | /// between them. |
| 256 | pub const Position = enum(u2) { |
| 257 | header, |
| 258 | footer, |
| 259 | floating, |
| 260 | }; |
| 261 | |
| 216 | 262 | pub const Flags = packed struct(u32) { |
| 217 | | location_tag: Location.Tag, |
| 263 | /// While the number of header and footer nodes within a parent node is logically a part of |
| 264 | /// that parent, we actually store this information on the child nodes for efficiency: this |
| 265 | /// field indicates whether each child is a header node, a footer node, or a floating node. |
| 266 | /// |
| 267 | /// This value is meaningless for the root node, so is arbitrarily set to `.floating`. |
| 268 | position: Position, |
| 269 | /// For floating nodes, this node's offset into its parent will always be aligned to this |
| 270 | /// boundary. (This is not the case for header and footer nodes due to the requirement that |
| 271 | /// they be densely packed against the start/end of the parent node.) |
| 272 | /// |
| 273 | /// This node's size will also always be aligned to this boundary. (This applies regardless |
| 274 | /// of whether this is a floating node, a header node, or a footer node.) |
| 218 | 275 | alignment: Alignment, |
| 219 | | /// Whether this node can be moved. |
| 220 | | fixed: bool, |
| 276 | /// Whether `moved` events on this node bubble down to children. |
| 277 | bubbles_moved: bool, |
| 278 | /// Whether `next_moved` events are reported in `updates`. |
| 279 | enable_next_moved: bool, |
| 280 | |
| 281 | location_tag: Location.Tag, |
| 221 | 282 | /// Whether this node has been moved. |
| 222 | 283 | moved: bool, |
| 223 | 284 | /// Whether this node has been resized. |
| 224 | 285 | resized: bool, |
| 225 | 286 | /// Whether the next sibling has moved or is a different node. |
| 226 | 287 | next_moved: bool, |
| 227 | | /// Whether this node might contain non-zero bytes. |
| 288 | /// Whether this node might contain initialized bytes. |
| 228 | 289 | has_content: bool, |
| 229 | | /// Whether `moved` events on this node bubble down to children. |
| 230 | | bubbles_moved: bool, |
| 231 | | /// Whether `next_moved` events are reported in `updates`. |
| 232 | | enable_next_moved: bool, |
| 233 | | unused: u18 = 0, |
| 290 | unused: u17 = 0, |
| 234 | 291 | }; |
| 235 | 292 | |
| 236 | 293 | pub const Location = union(enum(u1)) { |
| ... | ... | @@ -267,6 +324,18 @@ pub const Node = extern struct { |
| 267 | 324 | } |
| 268 | 325 | }; |
| 269 | 326 | |
| 327 | pub const AddOptions = struct { |
| 328 | /// Must be aligned to the given `alignment`. |
| 329 | size: u64 = 0, |
| 330 | alignment: Alignment = .@"1", |
| 331 | bubbles_moved: bool = true, |
| 332 | enable_next_moved: bool = false, |
| 333 | |
| 334 | moved: bool = false, |
| 335 | resized: bool = false, |
| 336 | next_moved: bool = false, |
| 337 | }; |
| 338 | |
| 270 | 339 | pub const Index = enum(u32) { |
| 271 | 340 | root, |
| 272 | 341 | _, |
| ... | ... | @@ -292,6 +361,70 @@ pub const Node = extern struct { |
| 292 | 361 | return &mf.nodes.items[@backingInt(ni)]; |
| 293 | 362 | } |
| 294 | 363 | |
| 364 | /// Adds a floating child node to `parent_ni`. Returns the index of the new child. |
| 365 | pub fn addFloatingChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { |
| 366 | return mf.addNode(gpa, .{ |
| 367 | .add_options = opts, |
| 368 | .position = .floating, |
| 369 | .parent = parent_ni, |
| 370 | .prev = parent_ni.lastHeader(mf), |
| 371 | }); |
| 372 | } |
| 373 | /// Adds a header child node to `parent_ni`. Returns the index of the new child. |
| 374 | /// |
| 375 | /// Asserts that `parent_ni` has no existing header children. |
| 376 | pub fn addOnlyHeaderChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { |
| 377 | if (parent_ni.first(mf).unwrap()) |first_ni| { |
| 378 | assert(first_ni.position(mf) != .header); // `parent_ni` already has a header child |
| 379 | } |
| 380 | return parent_ni.addHeaderChildAfter(mf, gpa, .none, opts); |
| 381 | } |
| 382 | /// Adds a header child node to `parent_ni`. Returns the index of the new child. |
| 383 | /// |
| 384 | /// If `prev_oni` is `.none`, the new child is placed at the very start of the parent, |
| 385 | /// before any existing header nodes. |
| 386 | /// |
| 387 | /// Otherwise, asserts that `prev_oni` is a header node and a child of `parent_ni`, and |
| 388 | /// places the new child node immediately after `prev_oni`. |
| 389 | pub fn addHeaderChildAfter(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, prev_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { |
| 390 | return mf.addNode(gpa, .{ |
| 391 | .add_options = opts, |
| 392 | .position = .header, |
| 393 | .parent = parent_ni, |
| 394 | .prev = prev_oni, |
| 395 | }); |
| 396 | } |
| 397 | /// Adds a footer child node to `parent_ni`. Returns the index of the new child. |
| 398 | /// |
| 399 | /// Asserts that `parent_ni` has no existing footer children. |
| 400 | pub fn addOnlyFooterChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { |
| 401 | if (parent_ni.last(mf).unwrap()) |last_ni| { |
| 402 | assert(last_ni.position(mf) != .footer); // `parent_ni` already has a footer child |
| 403 | } |
| 404 | return parent_ni.addFooterChildBefore(mf, gpa, .none, opts); |
| 405 | } |
| 406 | /// Adds a footer child node to `parent_ni`. Returns the index of the new child. |
| 407 | /// |
| 408 | /// If `next_oni` is `.none`, the new child is placed at the very end of the parent, after |
| 409 | /// any existing footer nodes. |
| 410 | /// |
| 411 | /// Otherwise, asserts that `next_oni` is a footer node and a child of `parent_ni`, and |
| 412 | /// places the new child node immediately before `next_oni`. |
| 413 | pub fn addFooterChildBefore(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, next_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { |
| 414 | const prev_oni: Node.Index.Optional = prev: { |
| 415 | const next_ni = next_oni.unwrap() orelse { |
| 416 | break :prev parent_ni.last(mf); |
| 417 | }; |
| 418 | break :prev next_ni.prev(mf); |
| 419 | }; |
| 420 | return mf.addNode(gpa, .{ |
| 421 | .add_options = opts, |
| 422 | .position = .footer, |
| 423 | .parent = parent_ni, |
| 424 | .prev = prev_oni, |
| 425 | }); |
| 426 | } |
| 427 | |
| 295 | 428 | /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. |
| 296 | 429 | pub const toOptional = Optional.wrap; |
| 297 | 430 | |
| ... | ... | @@ -299,19 +432,54 @@ pub const Node = extern struct { |
| 299 | 432 | return ni.get(mf).parent; |
| 300 | 433 | } |
| 301 | 434 | |
| 435 | pub fn first(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| 436 | return ni.get(mf).first; |
| 437 | } |
| 438 | |
| 439 | pub fn last(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| 440 | return ni.get(mf).last; |
| 441 | } |
| 442 | |
| 443 | fn lastHeader(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| 444 | var header_ni = ni.first(mf).unwrap() orelse return .none; |
| 445 | if (header_ni.position(mf) != .header) return .none; |
| 446 | while (true) { |
| 447 | const next_ni = header_ni.next(mf).unwrap() orelse break; |
| 448 | if (next_ni.position(mf) != .header) break; |
| 449 | header_ni = next_ni; |
| 450 | } |
| 451 | return .wrap(header_ni); |
| 452 | } |
| 453 | fn firstFooter(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| 454 | var footer_ni = ni.last(mf).unwrap() orelse return .none; |
| 455 | if (footer_ni.position(mf) != .footer) return .none; |
| 456 | while (true) { |
| 457 | const prev_ni = footer_ni.prev(mf).unwrap() orelse break; |
| 458 | if (prev_ni.position(mf) != .footer) break; |
| 459 | footer_ni = prev_ni; |
| 460 | } |
| 461 | return .wrap(footer_ni); |
| 462 | } |
| 463 | |
| 464 | /// Asserts that `ni` is not `.root`, because `Position` is meaningless for the root node. |
| 465 | pub fn position(ni: Node.Index, mf: *const MappedFile) Node.Position { |
| 466 | assert(ni != .root); |
| 467 | return ni.get(mf).flags.position; |
| 468 | } |
| 469 | |
| 302 | 470 | pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| 303 | 471 | return ni.get(mf).next; |
| 304 | 472 | } |
| 305 | 473 | fn setNext( |
| 306 | | prev_ni: Node.Index, |
| 474 | ni: Node.Index, |
| 307 | 475 | gpa: Allocator, |
| 308 | 476 | next_ni: Node.Index.Optional, |
| 309 | 477 | mf: *MappedFile, |
| 310 | 478 | ) Allocator.Error!void { |
| 311 | | const prev_next = &prev_ni.get(mf).next; |
| 312 | | if (prev_next.* == next_ni) return; |
| 313 | | prev_next.* = next_ni; |
| 314 | | try prev_ni.nextMoved(gpa, mf); |
| 479 | const next_ptr = &ni.get(mf).next; |
| 480 | if (next_ptr.* == next_ni) return; |
| 481 | next_ptr.* = next_ni; |
| 482 | try ni.nextMoved(gpa, mf); |
| 315 | 483 | } |
| 316 | 484 | |
| 317 | 485 | pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { |
| ... | ... | @@ -421,8 +589,14 @@ pub const Node = extern struct { |
| 421 | 589 | return ni.get(mf).flags.alignment; |
| 422 | 590 | } |
| 423 | 591 | |
| 424 | | fn setLocationAssumeCapacity(ni: Node.Index, mf: *MappedFile, offset: u64, size: u64) void { |
| 592 | fn setLocation(ni: Node.Index, mf: *MappedFile, gpa: Allocator, offset: u64, size: u64) Allocator.Error!void { |
| 593 | try mf.large.ensureUnusedCapacity(gpa, 2); |
| 594 | try mf.updates.ensureUnusedCapacity(gpa, 2); |
| 425 | 595 | const node = ni.get(mf); |
| 596 | if (node.flags.position == .floating) { |
| 597 | assert(node.flags.alignment.check(offset)); |
| 598 | } |
| 599 | assert(node.flags.alignment.check(size)); |
| 426 | 600 | if (size == 0) node.flags.has_content = false; |
| 427 | 601 | switch (node.location()) { |
| 428 | 602 | .small => |small| { |
| ... | ... | @@ -485,62 +659,46 @@ pub const Node = extern struct { |
| 485 | 659 | return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; |
| 486 | 660 | } |
| 487 | 661 | |
| 488 | | pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { |
| 489 | | mf.resizeNode(gpa, ni, size) catch |err| switch (err) { |
| 490 | | error.OutOfMemory, |
| 491 | | error.Canceled, |
| 492 | | => |e| return e, |
| 493 | | else => |e| { |
| 494 | | mf.io_err = e; |
| 495 | | return error.MappedFileIo; |
| 496 | | }, |
| 497 | | }; |
| 498 | | var writers_it = mf.writers.first; |
| 499 | | while (writers_it) |writer_node| : (writers_it = writer_node.next) { |
| 500 | | const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); |
| 501 | | w.interface.buffer = w.ni.slice(mf); |
| 502 | | } |
| 662 | /// Ensures that the size of `ni` is at least `min_size`. Valid for any node. |
| 663 | /// |
| 664 | /// Applies `growth_factor` if necessary (so the caller should *not* apply `growth_factor`). |
| 665 | pub fn ensureMinimumSize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, min_size: u64) Error!void { |
| 666 | _, const current_size = ni.location(mf).resolve(mf); |
| 667 | if (current_size >= min_size) return; |
| 668 | const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); |
| 669 | try mf.growNode(gpa, ni, new_size, .minimum); |
| 670 | mf.updateWriters(); |
| 503 | 671 | } |
| 504 | 672 | |
| 505 | | pub const RealignNodeOptions = struct { |
| 506 | | /// Shift the node backwards if possible |
| 507 | | try_backwards: bool = false, |
| 508 | | }; |
| 509 | | |
| 510 | | /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. |
| 511 | | /// Asserts that `ni` is not `.root`. |
| 512 | | pub fn realign( |
| 513 | | ni: Node.Index, |
| 514 | | mf: *MappedFile, |
| 515 | | gpa: Allocator, |
| 516 | | new_alignment: Alignment, |
| 517 | | opts: RealignNodeOptions, |
| 518 | | ) Error!void { |
| 519 | | mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { |
| 520 | | error.OutOfMemory, |
| 521 | | error.Canceled, |
| 522 | | => |e| return e, |
| 523 | | else => |e| { |
| 524 | | mf.io_err = e; |
| 525 | | return error.MappedFileIo; |
| 526 | | }, |
| 527 | | }; |
| 673 | /// Sets the size of `ni` to exactly `size`. |
| 674 | /// |
| 675 | /// Asserts that `ni` is a leaf node, i.e. has no children. |
| 676 | /// |
| 677 | /// Asserts that `size` is aligned to `ni.alignment(mf)`. |
| 678 | pub fn resizeLeaf(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { |
| 679 | assert(ni.first(mf) == .none); |
| 680 | // The alignment of `size` is asserted by `shrinkLeafNode` and `growNode`. |
| 681 | _, const old_size = ni.location(mf).resolve(mf); |
| 682 | switch (std.math.order(size, old_size)) { |
| 683 | .lt => try mf.shrinkLeafNode(gpa, ni, size), |
| 684 | .eq => {}, // `old_size` must be well-aligned, so `size` is too |
| 685 | .gt => try mf.growNode(gpa, ni, size, .exact), |
| 686 | } |
| 528 | 687 | mf.updateWriters(); |
| 529 | 688 | } |
| 530 | 689 | |
| 531 | | /// Shrink a node to `size`, exactly. |
| 532 | | /// Asserts that the new size can contain all the children. |
| 533 | | /// If `shift_next` is set, then the following node is shifted backwards into |
| 534 | | /// the free space as much as alignment allows. |
| 535 | | /// Asserts that `size` is >= the end of the last child node. |
| 536 | | pub fn shrink( |
| 690 | /// Updates a node's alignment to exactly `new_alignment`. Valid for any node. |
| 691 | /// |
| 692 | /// If the node's current offset or size is not sufficiently aligned, it will be moved |
| 693 | /// and/or resized to match the new alignment. The node's size may be increased by any |
| 694 | /// amount, as if `ensureMinimumSize` were used. |
| 695 | pub fn realign( |
| 537 | 696 | ni: Node.Index, |
| 538 | 697 | mf: *MappedFile, |
| 539 | 698 | gpa: Allocator, |
| 540 | | size: u64, |
| 541 | | shift_next: bool, |
| 699 | new_alignment: Alignment, |
| 542 | 700 | ) Error!void { |
| 543 | | try mf.shrinkNode(gpa, ni, size, shift_next); |
| 701 | try mf.realignNode(gpa, ni, new_alignment); |
| 544 | 702 | mf.updateWriters(); |
| 545 | 703 | } |
| 546 | 704 | |
| ... | ... | @@ -644,16 +802,9 @@ pub const Node = extern struct { |
| 644 | 802 | file_reader.pos, |
| 645 | 803 | w.ni.fileLocation(w.mf, true).offset + interface.end, |
| 646 | 804 | limit.minInt(interface.unusedCapacityLen()), |
| 647 | | ) catch |err| switch (err) { |
| 648 | | error.Canceled => |e| { |
| 649 | | w.err = e; |
| 650 | | return error.WriteFailed; |
| 651 | | }, |
| 652 | | else => |e| { |
| 653 | | w.mf.io_err = e; |
| 654 | | w.err = error.MappedFileIo; |
| 655 | | return error.WriteFailed; |
| 656 | | }, |
| 805 | ) catch |err| { |
| 806 | w.err = err; |
| 807 | return error.WriteFailed; |
| 657 | 808 | }); |
| 658 | 809 | if (n == 0) return error.Unimplemented; |
| 659 | 810 | file_reader.pos += n; |
| ... | ... | @@ -680,10 +831,8 @@ pub const Node = extern struct { |
| 680 | 831 | unused_capacity: usize, |
| 681 | 832 | ) Io.Writer.Error!void { |
| 682 | 833 | _ = preserve; |
| 683 | | const total_capacity = interface.end + unused_capacity; |
| 684 | | if (interface.buffer.len >= total_capacity) return; |
| 685 | 834 | const w: *Writer = @fieldParentPtr("interface", interface); |
| 686 | | w.ni.resize(w.mf, w.gpa, total_capacity +| total_capacity / growth_factor) catch |err| { |
| 835 | w.ni.ensureMinimumSize(w.mf, w.gpa, interface.end + unused_capacity) catch |err| { |
| 687 | 836 | w.err = err; |
| 688 | 837 | return error.WriteFailed; |
| 689 | 838 | }; |
| ... | ... | @@ -691,624 +840,1267 @@ pub const Node = extern struct { |
| 691 | 840 | }; |
| 692 | 841 | |
| 693 | 842 | comptime { |
| 694 | | if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 32); |
| 843 | if (!std.debug.runtime_safety) assert(@sizeOf(Node) == 32); |
| 695 | 844 | } |
| 696 | 845 | }; |
| 697 | 846 | |
| 847 | /// Asserts that `opts.position` is compatible with `opts.prev` (i.e. that this addition will not |
| 848 | /// violate the requirement that header nodes come before floating nodes come before footer nodes). |
| 698 | 849 | fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { |
| 699 | | parent: Node.Index.Optional = .none, |
| 700 | | prev: Node.Index.Optional = .none, |
| 701 | | next: Node.Index.Optional = .none, |
| 702 | | offset: u64 = 0, |
| 703 | | add_node: AddNodeOptions, |
| 704 | | }) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { |
| 850 | add_options: Node.AddOptions, |
| 851 | position: Node.Position, |
| 852 | parent: Node.Index, |
| 853 | /// If `position == .floating`, this is just used as an initial value, and may be immediately |
| 854 | /// replaced when finding a location for this node. In this case, it is still necessary that |
| 855 | /// `prev` be compatible with `position` (so `prev` must be either a floating node or the last |
| 856 | /// header node in `parent`). |
| 857 | prev: Node.Index.Optional, |
| 858 | }) Error!Node.Index { |
| 705 | 859 | mf.nodes_lock.assertUnlocked(); |
| 706 | | const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: { |
| 707 | | if (std.math.cast(u32, opts.offset)) |small_offset| break :location .{ .small, .{ |
| 708 | | .small = .{ .offset = small_offset, .size = 0 }, |
| 709 | | } }; |
| 710 | | try mf.large.ensureUnusedCapacity(gpa, 2); |
| 711 | | defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); |
| 712 | | break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; |
| 713 | | }; |
| 714 | 860 | |
| 715 | | const free_ni: Node.Index, const free_node: *Node = if (mf.free_ni.unwrap()) |free_ni| free: { |
| 716 | | const free_node = free_ni.get(mf); |
| 717 | | mf.free_ni = free_node.next; |
| 718 | | break :free .{ free_ni, free_node }; |
| 719 | | } else .{ |
| 720 | | @fromBackingInt(@intCast(mf.nodes.items.len)), |
| 721 | | mf.nodes.addOneAssumeCapacity(), |
| 861 | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 862 | try mf.large.ensureUnusedCapacity(gpa, 2); |
| 863 | |
| 864 | const new_ni: Node.Index = new: { |
| 865 | if (mf.free_ni.unwrap()) |free_ni| { |
| 866 | mf.free_ni = free_ni.get(mf).next; |
| 867 | break :new free_ni; |
| 868 | } |
| 869 | const new_ni: Node.Index = @fromBackingInt(@intCast(mf.nodes.items.len)); |
| 870 | _ = mf.nodes.addOneAssumeCapacity(); |
| 871 | break :new new_ni; |
| 722 | 872 | }; |
| 723 | 873 | |
| 724 | | if (opts.prev.unwrap()) |prev_ni| { |
| 725 | | try prev_ni.setNext(gpa, .wrap(free_ni), mf); |
| 726 | | } else if (opts.parent.unwrap()) |parent_ni| { |
| 727 | | parent_ni.get(mf).first = .wrap(free_ni); |
| 728 | | } else { |
| 729 | | assert(free_ni == .root); |
| 730 | | } |
| 874 | const next_oni: Node.Index.Optional = if (opts.prev.unwrap()) |prev_ni| next: { |
| 875 | assert(prev_ni.parent(mf) == opts.parent.toOptional()); // `prev` is not a child of `parent` |
| 876 | break :next prev_ni.get(mf).next; |
| 877 | } else opts.parent.first(mf); |
| 731 | 878 | |
| 732 | | if (opts.next.unwrap()) |next_ni| { |
| 733 | | next_ni.get(mf).prev = .wrap(free_ni); |
| 734 | | } else if (opts.parent.unwrap()) |parent_ni| { |
| 735 | | parent_ni.get(mf).last = .wrap(free_ni); |
| 736 | | } else { |
| 737 | | assert(free_ni == .root); |
| 879 | // Validate node ordering |
| 880 | switch (opts.position) { |
| 881 | .floating => { |
| 882 | if (opts.prev.unwrap()) |prev_ni| { |
| 883 | assert(prev_ni.position(mf) != .footer); // tried to add floating node after footer node |
| 884 | } |
| 885 | if (next_oni.unwrap()) |next_ni| { |
| 886 | assert(next_ni.position(mf) != .header); // tried to add floating node before header node |
| 887 | } |
| 888 | }, |
| 889 | .header => if (opts.prev.unwrap()) |prev_ni| { |
| 890 | switch (prev_ni.position(mf)) { |
| 891 | .header => {}, |
| 892 | .floating => unreachable, // tried to add header node after floating node |
| 893 | .footer => unreachable, // tried to add header node after footer node |
| 894 | } |
| 895 | }, |
| 896 | .footer => if (next_oni.unwrap()) |next_ni| { |
| 897 | switch (next_ni.position(mf)) { |
| 898 | .header => unreachable, // tried to add footer node before header node |
| 899 | .floating => unreachable, // tried to add footer node before floating node |
| 900 | .footer => {}, |
| 901 | } |
| 902 | }, |
| 738 | 903 | } |
| 739 | 904 | |
| 740 | | free_node.* = .{ |
| 741 | | .parent = opts.parent, |
| 742 | | .prev = opts.prev, |
| 743 | | .next = opts.next, |
| 905 | // Initialize the node as empty with alignment 1 |
| 906 | const location: Node.Location = loc: { |
| 907 | const offset: u64 = switch (opts.position) { |
| 908 | .header, .floating => offset: { |
| 909 | const prev_ni = opts.prev.unwrap() orelse break :offset 0; |
| 910 | const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); |
| 911 | break :offset prev_offset + prev_size; |
| 912 | }, |
| 913 | .footer => offset: { |
| 914 | const next_ni = next_oni.unwrap() orelse { |
| 915 | _, const parent_size = opts.parent.location(mf).resolve(mf); |
| 916 | break :offset parent_size; |
| 917 | }; |
| 918 | const next_offset, _ = next_ni.location(mf).resolve(mf); |
| 919 | break :offset next_offset; |
| 920 | }, |
| 921 | }; |
| 922 | if (std.math.cast(u32, offset)) |small_offset| { |
| 923 | break :loc .{ .small = .{ .offset = small_offset, .size = 0 } }; |
| 924 | } |
| 925 | const large_index = mf.large.items.len; |
| 926 | mf.large.appendSliceAssumeCapacity(&.{ offset, 0 }); |
| 927 | break :loc .{ .large = .{ .index = large_index } }; |
| 928 | }; |
| 929 | new_ni.get(mf).* = .{ |
| 930 | .parent = .wrap(opts.parent), |
| 931 | .prev = .none, |
| 932 | .next = .none, |
| 744 | 933 | .first = .none, |
| 745 | 934 | .last = .none, |
| 746 | 935 | .flags = .{ |
| 747 | | .location_tag = location_tag, |
| 936 | .position = opts.position, |
| 748 | 937 | .alignment = .@"1", |
| 749 | | .fixed = opts.add_node.fixed, |
| 750 | | .moved = true, |
| 751 | | .resized = true, |
| 752 | | .next_moved = true, |
| 938 | .bubbles_moved = opts.add_options.bubbles_moved, |
| 939 | .enable_next_moved = opts.add_options.enable_next_moved, |
| 940 | .location_tag = location, |
| 941 | .moved = false, |
| 942 | .resized = false, |
| 943 | .next_moved = false, |
| 753 | 944 | .has_content = false, |
| 754 | | .bubbles_moved = opts.add_node.bubbles_moved, |
| 755 | | .enable_next_moved = opts.add_node.enable_next_moved, |
| 756 | 945 | }, |
| 757 | | .location_payload = location_payload, |
| 946 | .location_payload = switch (location) { |
| 947 | .small => |small| .{ .small = small }, |
| 948 | .large => |large| .{ .large = large }, |
| 949 | }, |
| 758 | 950 | }; |
| 759 | 951 | |
| 760 | | { |
| 761 | | defer { |
| 762 | | free_node.flags.moved = false; |
| 763 | | free_node.flags.resized = false; |
| 764 | | free_node.flags.next_moved = false; |
| 765 | | } |
| 766 | | try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{}); |
| 767 | | try mf.resizeNode(gpa, free_ni, opts.add_node.size); |
| 952 | try mf.addNodesToChildListBefore(gpa, next_oni, new_ni, new_ni); |
| 953 | |
| 954 | try mf.realignNode(gpa, new_ni, opts.add_options.alignment); |
| 955 | if (opts.add_options.size > 0) { |
| 956 | try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); |
| 768 | 957 | } |
| 769 | 958 | mf.updateWriters(); |
| 770 | | if (opts.add_node.moved) try free_ni.moved(gpa, mf); |
| 771 | | if (opts.add_node.resized) try free_ni.resized(gpa, mf); |
| 772 | | if (opts.add_node.next_moved) try free_ni.nextMoved(gpa, mf); |
| 773 | | return free_ni; |
| 774 | | } |
| 775 | 959 | |
| 776 | | pub const AddNodeOptions = struct { |
| 777 | | size: u64 = 0, |
| 778 | | alignment: Alignment = .@"1", |
| 779 | | fixed: bool = false, |
| 780 | | moved: bool = false, |
| 781 | | resized: bool = false, |
| 782 | | next_moved: bool = false, |
| 783 | | bubbles_moved: bool = true, |
| 784 | | enable_next_moved: bool = false, |
| 785 | | }; |
| 960 | new_ni.get(mf).flags.moved = false; |
| 961 | new_ni.get(mf).flags.resized = false; |
| 962 | new_ni.get(mf).flags.next_moved = false; |
| 786 | 963 | |
| 787 | | pub fn addOnlyChildNode( |
| 788 | | mf: *MappedFile, |
| 789 | | gpa: Allocator, |
| 790 | | parent_ni: Node.Index, |
| 791 | | opts: AddNodeOptions, |
| 792 | | ) Error!Node.Index { |
| 793 | | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 794 | | const parent = parent_ni.get(mf); |
| 795 | | assert(parent.first == .none and parent.last == .none); |
| 796 | | return mf.addNode(gpa, .{ |
| 797 | | .parent = .wrap(parent_ni), |
| 798 | | .add_node = opts, |
| 799 | | }) catch |err| switch (err) { |
| 800 | | error.OutOfMemory, |
| 801 | | error.Canceled, |
| 802 | | => |e| return e, |
| 803 | | else => |e| { |
| 804 | | mf.io_err = e; |
| 805 | | return error.MappedFileIo; |
| 806 | | }, |
| 807 | | }; |
| 808 | | } |
| 964 | if (opts.add_options.moved) try new_ni.moved(gpa, mf); |
| 965 | if (opts.add_options.resized) try new_ni.resized(gpa, mf); |
| 966 | if (opts.add_options.next_moved) try new_ni.nextMoved(gpa, mf); |
| 809 | 967 | |
| 810 | | pub fn addFirstChildNode( |
| 811 | | mf: *MappedFile, |
| 812 | | gpa: Allocator, |
| 813 | | parent_ni: Node.Index, |
| 814 | | opts: AddNodeOptions, |
| 815 | | ) Error!Node.Index { |
| 816 | | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 817 | | const parent = parent_ni.get(mf); |
| 818 | | return mf.addNode(gpa, .{ |
| 819 | | .parent = .wrap(parent_ni), |
| 820 | | .next = parent.first, |
| 821 | | .add_node = opts, |
| 822 | | }) catch |err| switch (err) { |
| 823 | | error.OutOfMemory, |
| 824 | | error.Canceled, |
| 825 | | => |e| return e, |
| 826 | | else => |e| { |
| 827 | | mf.io_err = e; |
| 828 | | return error.MappedFileIo; |
| 829 | | }, |
| 830 | | }; |
| 968 | return new_ni; |
| 831 | 969 | } |
| 832 | 970 | |
| 833 | | pub fn addLastChildNode( |
| 971 | fn shrinkLeafNode( |
| 834 | 972 | mf: *MappedFile, |
| 835 | 973 | gpa: Allocator, |
| 836 | | parent_ni: Node.Index, |
| 837 | | opts: AddNodeOptions, |
| 838 | | ) Error!Node.Index { |
| 839 | | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 840 | | const parent = parent_ni.get(mf); |
| 841 | | return mf.addNode(gpa, .{ |
| 842 | | .parent = .wrap(parent_ni), |
| 843 | | .prev = parent.last, |
| 844 | | .offset = offset: { |
| 845 | | const last_ni = parent.last.unwrap() orelse break :offset 0; |
| 846 | | const last_offset, const last_size = last_ni.location(mf).resolve(mf); |
| 847 | | break :offset last_offset + last_size; |
| 848 | | }, |
| 849 | | .add_node = opts, |
| 850 | | }) catch |err| switch (err) { |
| 851 | | error.OutOfMemory, |
| 852 | | error.Canceled, |
| 853 | | => |e| return e, |
| 854 | | else => |e| { |
| 855 | | mf.io_err = e; |
| 974 | ni: Node.Index, |
| 975 | new_size: u64, |
| 976 | ) Error!void { |
| 977 | mf.nodes_lock.assertUnlocked(); |
| 978 | |
| 979 | const old_offset, const old_size = ni.location(mf).resolve(mf); |
| 980 | |
| 981 | assert(new_size < old_size); |
| 982 | assert(ni.alignment(mf).check(new_size)); |
| 983 | assert(ni.first(mf) == .none); // `ni` must be a leaf node |
| 984 | |
| 985 | const parent_ni = ni.parent(mf).unwrap() orelse { |
| 986 | assert(ni == .root); |
| 987 | mf.memory_map.write(mf.io) catch |err| { |
| 988 | mf.io_err = switch (err) { |
| 989 | error.Canceled => |e| return e, |
| 990 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 991 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 992 | else => |e| e, |
| 993 | }; |
| 856 | 994 | return error.MappedFileIo; |
| 857 | | }, |
| 995 | }; |
| 996 | mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { |
| 997 | error.Canceled => |e| return e, |
| 998 | else => |e| { |
| 999 | mf.io_err = e; |
| 1000 | return error.MappedFileIo; |
| 1001 | }, |
| 1002 | }; |
| 1003 | try mf.ensureTotalCapacityPrecise(@intCast(new_size)); |
| 1004 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1005 | return; |
| 858 | 1006 | }; |
| 859 | | } |
| 860 | 1007 | |
| 861 | | pub fn addNodeAfter( |
| 862 | | mf: *MappedFile, |
| 863 | | gpa: Allocator, |
| 864 | | prev_ni: Node.Index, |
| 865 | | opts: AddNodeOptions, |
| 866 | | ) Error!Node.Index { |
| 867 | | try mf.nodes.ensureUnusedCapacity(gpa, 1); |
| 868 | | const prev = prev_ni.get(mf); |
| 869 | | const prev_offset, const prev_size = prev.location().resolve(mf); |
| 870 | | return mf.addNode(gpa, .{ |
| 871 | | .parent = prev.parent, |
| 872 | | .prev = .wrap(prev_ni), |
| 873 | | .next = prev.next, |
| 874 | | .offset = prev_offset + prev_size, |
| 875 | | .add_node = opts, |
| 876 | | }) catch |err| switch (err) { |
| 877 | | error.OutOfMemory, |
| 878 | | error.Canceled, |
| 879 | | => |e| return e, |
| 880 | | else => |e| { |
| 881 | | mf.io_err = e; |
| 882 | | return error.MappedFileIo; |
| 1008 | switch (ni.position(mf)) { |
| 1009 | .header => { |
| 1010 | const shift = old_size - new_size; |
| 1011 | |
| 1012 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1013 | |
| 1014 | // We need to shift backwards all header nodes following us. |
| 1015 | const next_header_ni = ni.next(mf).unwrap() orelse return; |
| 1016 | if (next_header_ni.position(mf) != .header) return; |
| 1017 | |
| 1018 | var header_ni = next_header_ni; |
| 1019 | while (true) { |
| 1020 | const old_header_off, const old_header_size = header_ni.location(mf).resolve(mf); |
| 1021 | try header_ni.setLocation(mf, gpa, old_header_off - shift, old_header_size); |
| 1022 | |
| 1023 | const next_ni = header_ni.next(mf).unwrap() orelse break; |
| 1024 | if (next_ni.position(mf) != .header) break; |
| 1025 | header_ni = next_ni; |
| 1026 | } |
| 1027 | |
| 1028 | // Now we must shift the actual header bytes of those nodes backwards. |
| 1029 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| 1030 | const move_src_off = old_offset + old_size; |
| 1031 | const move_dest_off = old_offset + new_size; |
| 1032 | assert(next_header_ni.location(mf).resolve(mf)[0] == move_dest_off); // `move_dest_off` because we already updated the location |
| 1033 | const move_size = size: { |
| 1034 | // `header_ni` is the last header in the parent. |
| 1035 | const last_off, const last_size = header_ni.location(mf).resolve(mf); |
| 1036 | const move_end = last_off + last_size; |
| 1037 | break :size move_end - move_dest_off; // `move_dest_off` because we already updated the location |
| 1038 | }; |
| 1039 | try mf.moveRange( |
| 1040 | parent_file_off + move_src_off, |
| 1041 | parent_file_off + move_dest_off, |
| 1042 | move_size, |
| 1043 | ); |
| 883 | 1044 | }, |
| 884 | | }; |
| 885 | | } |
| 1045 | .floating => { |
| 1046 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1047 | }, |
| 1048 | .footer => { |
| 1049 | const shift = old_size - new_size; |
| 886 | 1050 | |
| 887 | | fn shrinkNode( |
| 888 | | mf: *MappedFile, |
| 889 | | gpa: Allocator, |
| 890 | | ni: Node.Index, |
| 891 | | size: u64, |
| 892 | | shift_next: bool, |
| 893 | | ) !void { |
| 894 | | mf.nodes_lock.assertUnlocked(); |
| 895 | | const node = ni.get(mf); |
| 896 | | const old_offset, _ = node.location().resolve(mf); |
| 1051 | const new_offset = old_offset + shift; |
| 1052 | try ni.setLocation(mf, gpa, new_offset, new_size); |
| 897 | 1053 | |
| 898 | | // This would require unmapping first |
| 899 | | assert(ni != .root); |
| 1054 | const prev_footers_size = prev_footers_size: { |
| 1055 | // We need to shift forwards all footer nodes preceding us. |
| 1056 | const prev_footer_ni = ni.prev(mf).unwrap() orelse { |
| 1057 | break :prev_footers_size 0; |
| 1058 | }; |
| 1059 | if (prev_footer_ni.position(mf) != .footer) { |
| 1060 | break :prev_footers_size 0; |
| 1061 | } |
| 900 | 1062 | |
| 901 | | if (node.last.unwrap()) |last_ni| { |
| 902 | | const last = last_ni.get(mf); |
| 903 | | const last_offset, const last_size = last.location().resolve(mf); |
| 904 | | assert(last_offset + last_size > size); |
| 905 | | } |
| 1063 | var footer_ni = prev_footer_ni; |
| 1064 | while (true) { |
| 1065 | const old_footer_off, const old_footer_size = footer_ni.location(mf).resolve(mf); |
| 1066 | try footer_ni.setLocation(mf, gpa, old_footer_off + shift, old_footer_size); |
| 906 | 1067 | |
| 907 | | try mf.large.ensureUnusedCapacity(gpa, 4); |
| 908 | | try mf.updates.ensureUnusedCapacity(gpa, 4); |
| 1068 | const prev_ni = footer_ni.prev(mf).unwrap() orelse break; |
| 1069 | if (prev_ni.position(mf) != .footer) break; |
| 1070 | footer_ni = prev_ni; |
| 1071 | } |
| 909 | 1072 | |
| 910 | | ni.setLocationAssumeCapacity(mf, old_offset, size); |
| 911 | | if (!shift_next) return; |
| 912 | | const next_ni = node.next.unwrap() orelse return; |
| 1073 | // `footer_ni` is the first footer in the parent. This expression gets its *new* |
| 1074 | // offset because we already did the `setLocation` calls. |
| 1075 | const first_footer_new_offset = footer_ni.location(mf).resolve(mf)[0]; |
| 913 | 1076 | |
| 914 | | const next = next_ni.get(mf); |
| 915 | | const old_next_offset, const next_size = next.location().resolve(mf); |
| 916 | | const padding = old_next_offset - (old_offset + size); |
| 917 | | const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); |
| 1077 | break :prev_footers_size new_offset - first_footer_new_offset; |
| 1078 | }; |
| 918 | 1079 | |
| 919 | | if (next.flags.has_content and new_next_offset < old_next_offset) { |
| 920 | | const old_file_offset = next_ni.fileLocation(mf, false).offset; |
| 921 | | const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; |
| 922 | | @memmove( |
| 923 | | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], |
| 924 | | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(next_size)], |
| 925 | | ); |
| 926 | | @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); |
| 1080 | // Now we must shift the actual footer bytes forwards, including our own. |
| 1081 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| 1082 | try mf.moveRange( |
| 1083 | parent_file_offset + old_offset - prev_footers_size, |
| 1084 | parent_file_offset + new_offset - prev_footers_size, |
| 1085 | prev_footers_size + new_size, |
| 1086 | ); |
| 1087 | }, |
| 927 | 1088 | } |
| 928 | | |
| 929 | | next_ni.setLocationAssumeCapacity(mf, new_next_offset, next_size); |
| 930 | 1089 | } |
| 931 | 1090 | |
| 932 | | fn resizeNode( |
| 1091 | const GrowMode = enum { exact, minimum }; |
| 1092 | |
| 1093 | /// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. |
| 1094 | /// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. |
| 1095 | /// |
| 1096 | /// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). |
| 1097 | /// |
| 1098 | /// Asserts that `new_size` is greater than the current size of `ni`. |
| 1099 | fn growNode( |
| 933 | 1100 | mf: *MappedFile, |
| 934 | 1101 | gpa: Allocator, |
| 935 | 1102 | ni: Node.Index, |
| 936 | | requested_size: u64, |
| 937 | | ) (Allocator.Error || Io.Cancelable || IoError)!void { |
| 1103 | new_size: u64, |
| 1104 | grow_mode: GrowMode, |
| 1105 | ) Error!void { |
| 938 | 1106 | mf.nodes_lock.assertUnlocked(); |
| 939 | | const io = mf.io; |
| 1107 | |
| 940 | 1108 | const node = ni.get(mf); |
| 1109 | |
| 941 | 1110 | const old_offset, const old_size = node.location().resolve(mf); |
| 942 | | const new_size = node.flags.alignment.forward(@intCast(requested_size)); |
| 943 | 1111 | |
| 944 | | // Resize the entire file |
| 1112 | assert(node.flags.alignment.check(old_size)); |
| 1113 | assert(node.flags.alignment.check(new_size)); |
| 1114 | assert(new_size > old_size); |
| 1115 | |
| 945 | 1116 | const parent_ni = node.parent.unwrap() orelse { |
| 946 | 1117 | assert(ni == .root); |
| 947 | | try mf.ensureCapacityForSetLocation(gpa); |
| 948 | | mf.memory_map.write(io) catch |err| switch (err) { |
| 949 | | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking |
| 950 | | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing |
| 951 | | else => |e| return e, |
| 952 | | }; |
| 953 | | try mf.memory_map.file.setLength(io, new_size); |
| 954 | | try mf.ensureTotalCapacityInner(@intCast(new_size)); |
| 955 | | ni.setLocationAssumeCapacity(mf, old_offset, new_size); |
| 956 | | return; |
| 957 | | }; |
| 958 | | const parent = parent_ni.get(mf); |
| 959 | | _, var old_parent_size = parent.location().resolve(mf); |
| 960 | | const trailing_end = trailing_end: { |
| 961 | | const next_ni = node.next.unwrap() orelse break :trailing_end old_parent_size; |
| 962 | | const next_offset, _ = next_ni.location(mf).resolve(mf); |
| 963 | | break :trailing_end next_offset; |
| 964 | | }; |
| 965 | | assert(old_offset + old_size <= trailing_end); |
| 966 | | if (old_offset + new_size <= trailing_end) { |
| 967 | | // Expand the node into trailing free space |
| 968 | | try mf.ensureCapacityForSetLocation(gpa); |
| 969 | | ni.setLocationAssumeCapacity(mf, old_offset, new_size); |
| 970 | | return; |
| 971 | | } |
| 972 | | insert_range: { |
| 973 | | if (!is_linux) break :insert_range; |
| 974 | | if (mf.flags.fallocate_insert_range_unsupported) break :insert_range; |
| 975 | | |
| 976 | | // We need the node to be aligned to `mf.flags.block_size` in the file in order to use this |
| 977 | | // fast path. It is not sufficient to check `node.flags.alignment`, because that doesn't |
| 978 | | // necessarily mean that all *parent* nodes are equally aligned; instead we must compute the |
| 979 | | // actual file offset. |
| 980 | | const range_file_offset = ni.fileLocation(mf, false).offset + old_size; |
| 981 | | const range_size = node.flags.alignment.forward( |
| 982 | | @intCast(requested_size +| requested_size / growth_factor), |
| 983 | | ) - old_size; |
| 984 | | if (!mf.flags.block_size.check(@intCast(range_file_offset))) break :insert_range; |
| 985 | | if (!mf.flags.block_size.check(@intCast(range_size))) break :insert_range; |
| 986 | | |
| 987 | | mf.memory_map.write(io) catch |err| switch (err) { |
| 988 | | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking |
| 989 | | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing |
| 990 | | else => |e| return e, |
| 1118 | |
| 1119 | if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { |
| 1120 | return; |
| 1121 | } |
| 1122 | |
| 1123 | mf.memory_map.write(mf.io) catch |err| { |
| 1124 | mf.io_err = switch (err) { |
| 1125 | error.Canceled => |e| return e, |
| 1126 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 1127 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 1128 | else => |e| e, |
| 1129 | }; |
| 1130 | return error.MappedFileIo; |
| 991 | 1131 | }; |
| 992 | | // Ask the filesystem driver to insert extents into the file without copying any data |
| 993 | | const last_offset, const last_size = parent.last.unwrap().?.location(mf).resolve(mf); |
| 994 | | const last_end = last_offset + last_size; |
| 995 | | assert(last_end <= old_parent_size); |
| 996 | | _, const file_size = Node.Index.root.location(mf).resolve(mf); |
| 997 | | while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) { |
| 998 | | .lt => linux.fallocate( |
| 999 | | mf.memory_map.file.handle, |
| 1000 | | linux.FALLOC.FL_INSERT_RANGE, |
| 1001 | | @intCast(range_file_offset), |
| 1002 | | @intCast(range_size), |
| 1003 | | ), |
| 1004 | | .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)), |
| 1005 | | .gt => unreachable, |
| 1006 | | })) { |
| 1007 | | .SUCCESS => { |
| 1008 | | var enclosing_ni = ni; |
| 1009 | | while (true) { |
| 1010 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1011 | | const enclosing = enclosing_ni.get(mf); |
| 1012 | | const enclosing_offset, const old_enclosing_size = |
| 1013 | | enclosing.location().resolve(mf); |
| 1014 | | const new_enclosing_size = old_enclosing_size + range_size; |
| 1015 | | enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); |
| 1016 | | if (enclosing_ni == .root) { |
| 1017 | | assert(enclosing_offset == 0); |
| 1018 | | try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); |
| 1019 | | break; |
| 1020 | | } |
| 1021 | | var after_oni = enclosing.next; |
| 1022 | | while (after_oni.unwrap()) |after_ni| { |
| 1023 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1024 | | const after = after_ni.get(mf); |
| 1025 | | const after_offset, const after_size = after.location().resolve(mf); |
| 1026 | | after_ni.setLocationAssumeCapacity( |
| 1027 | | mf, |
| 1028 | | range_size + after_offset, |
| 1029 | | after_size, |
| 1030 | | ); |
| 1031 | | after_oni = after.next; |
| 1032 | | } |
| 1033 | | enclosing_ni = enclosing.parent.unwrap().?; |
| 1034 | | } |
| 1035 | | return; |
| 1036 | | }, |
| 1037 | | .INTR => continue, |
| 1038 | | .BADF, .FBIG, .INVAL => unreachable, |
| 1039 | | .IO => return error.InputOutput, |
| 1040 | | .NODEV => return error.NotFile, |
| 1041 | | .NOSPC => return error.NoSpaceLeft, |
| 1042 | | .NOSYS, .OPNOTSUPP => { |
| 1043 | | mf.flags.fallocate_insert_range_unsupported = true; |
| 1044 | | break :insert_range; |
| 1132 | mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { |
| 1133 | error.Canceled => |e| return e, |
| 1134 | else => |e| { |
| 1135 | mf.io_err = e; |
| 1136 | return error.MappedFileIo; |
| 1045 | 1137 | }, |
| 1046 | | .PERM => return error.PermissionDenied, |
| 1047 | | .SPIPE => return error.Unseekable, |
| 1048 | | .TXTBSY => return error.FileBusy, |
| 1049 | | else => |e| return std.posix.unexpectedErrno(e), |
| 1050 | 1138 | }; |
| 1051 | | } |
| 1052 | | if (node.next == .none) { |
| 1053 | | // As this is the last node, we simply need more space in the parent |
| 1054 | | const new_parent_size = old_offset + new_size; |
| 1055 | | try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); |
| 1056 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1057 | | ni.setLocationAssumeCapacity(mf, old_offset, new_size); |
| 1058 | | return; |
| 1059 | | } |
| 1060 | | if (!node.flags.fixed) { |
| 1061 | | // Make space at the end of the parent for this floating node |
| 1062 | | const last = parent.last.unwrap().?.get(mf); |
| 1063 | | const last_offset, const last_size = last.location().resolve(mf); |
| 1064 | | const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); |
| 1065 | | const new_parent_size = new_offset + new_size; |
| 1066 | | if (new_parent_size > old_parent_size) |
| 1067 | | try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); |
| 1068 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1069 | | const next_ni = node.next.unwrap().?; |
| 1070 | | next_ni.get(mf).prev = node.prev; |
| 1071 | | if (node.prev.unwrap()) |prev_ni| { |
| 1072 | | try prev_ni.setNext(gpa, .wrap(next_ni), mf); |
| 1073 | | } else { |
| 1074 | | parent.first = .wrap(next_ni); |
| 1075 | | } |
| 1076 | | try parent.last.unwrap().?.setNext(gpa, .wrap(ni), mf); |
| 1077 | | node.prev = parent.last; |
| 1078 | | try ni.setNext(gpa, .none, mf); |
| 1079 | | parent.last = .wrap(ni); |
| 1080 | | if (node.flags.has_content) { |
| 1081 | | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| 1139 | try mf.ensureTotalCapacityPrecise(@intCast(new_size)); |
| 1140 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1141 | // We need to move any footers to be at the *new* end of the file. |
| 1142 | if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { |
| 1143 | const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1144 | const footers_size = old_size - old_footers_offset; |
| 1082 | 1145 | try mf.moveRange( |
| 1083 | | parent_file_offset + old_offset, |
| 1084 | | parent_file_offset + new_offset, |
| 1085 | | old_size, |
| 1146 | old_footers_offset, |
| 1147 | old_footers_offset + (new_size - old_size), |
| 1148 | footers_size, |
| 1086 | 1149 | ); |
| 1150 | // Also update the footers' locations. |
| 1151 | var cur_ni = first_footer_ni; |
| 1152 | while (true) { |
| 1153 | const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); |
| 1154 | try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); |
| 1155 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| 1156 | } |
| 1087 | 1157 | } |
| 1088 | | ni.setLocationAssumeCapacity(mf, new_offset, new_size); |
| 1089 | 1158 | return; |
| 1090 | | } |
| 1091 | | // Search for the first floating node following this fixed node |
| 1092 | | var last_fixed_ni = ni; |
| 1093 | | var first_floating_oni = node.next; |
| 1094 | | var shift = new_size - old_size; |
| 1095 | | var max_shift_align: Alignment = .@"1"; |
| 1096 | | var direction: enum { forward, reverse } = .forward; |
| 1097 | | while (true) { |
| 1098 | | const last_fixed = last_fixed_ni.get(mf); |
| 1099 | | assert(last_fixed.flags.fixed); |
| 1100 | | const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); |
| 1101 | | const new_last_fixed_offset = old_last_fixed_offset + shift; |
| 1102 | | if (first_floating_oni.unwrap()) |first_floating_ni| make_space: { |
| 1103 | | const first_floating = first_floating_ni.get(mf); |
| 1104 | | const old_first_floating_offset, const first_floating_size = |
| 1105 | | first_floating.location().resolve(mf); |
| 1106 | | assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); |
| 1107 | | if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) |
| 1108 | | break :make_space; |
| 1109 | | assert(direction == .forward); |
| 1110 | | max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); |
| 1111 | | if (first_floating.flags.fixed) { |
| 1112 | | shift = max_shift_align.forward(@intCast( |
| 1113 | | @max(shift, first_floating_size), |
| 1114 | | )); |
| 1115 | | |
| 1116 | | // Not enough space, try the next node |
| 1117 | | last_fixed_ni = first_floating_ni; |
| 1118 | | first_floating_oni = first_floating.next; |
| 1119 | | continue; |
| 1120 | | } |
| 1121 | | // Move the found floating node to make space for preceding fixed nodes |
| 1122 | | const last = parent.last.unwrap().?.get(mf); |
| 1123 | | const last_offset, const last_size = last.location().resolve(mf); |
| 1124 | | const new_first_floating_offset = max_shift_align.forward( |
| 1125 | | @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), |
| 1126 | | ); |
| 1127 | | const new_parent_size = new_first_floating_offset + first_floating_size; |
| 1128 | | if (new_parent_size > old_parent_size) { |
| 1129 | | try mf.resizeNode( |
| 1130 | | gpa, |
| 1131 | | parent_ni, |
| 1132 | | new_parent_size +| new_parent_size / growth_factor, |
| 1133 | | ); |
| 1134 | | _, old_parent_size = parent.location().resolve(mf); |
| 1159 | }; |
| 1160 | |
| 1161 | switch (node.flags.position) { |
| 1162 | .header => { |
| 1163 | if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { |
| 1164 | return; |
| 1135 | 1165 | } |
| 1136 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1137 | | if (parent.last.unwrap().? != first_floating_ni) { |
| 1138 | | const old_last = parent.last.unwrap().?; |
| 1139 | | first_floating.prev = .wrap(old_last); |
| 1140 | | parent.last = .wrap(first_floating_ni); |
| 1141 | | try old_last.setNext(gpa, .wrap(first_floating_ni), mf); |
| 1142 | | try last_fixed_ni.setNext(gpa, first_floating.next, mf); |
| 1143 | | if (first_floating.next.unwrap()) |next_ni| { |
| 1144 | | next_ni.get(mf).prev = .wrap(last_fixed_ni); |
| 1166 | |
| 1167 | try mf.ensureAdditionalHeaderCapacity(gpa, parent_ni, new_size - old_size); |
| 1168 | |
| 1169 | // `old_offset` is still valid because header nodes don't move when the parent resizes. |
| 1170 | |
| 1171 | const last_header_ni: Node.Index = last_header: { |
| 1172 | var header_ni = ni; |
| 1173 | while (true) { |
| 1174 | const next_ni = header_ni.next(mf).unwrap() orelse break; |
| 1175 | if (next_ni.position(mf) != .header) break; |
| 1176 | header_ni = next_ni; |
| 1145 | 1177 | } |
| 1146 | | try first_floating_ni.setNext(gpa, .none, mf); |
| 1147 | | } |
| 1148 | | if (first_floating.flags.has_content) { |
| 1149 | | const parent_file_offset = |
| 1150 | | parent_ni.fileLocation(mf, false).offset; |
| 1151 | | try mf.moveRange( |
| 1152 | | parent_file_offset + old_first_floating_offset, |
| 1153 | | parent_file_offset + new_first_floating_offset, |
| 1154 | | first_floating_size, |
| 1155 | | ); |
| 1156 | | } |
| 1157 | | first_floating_ni.setLocationAssumeCapacity( |
| 1158 | | mf, |
| 1159 | | new_first_floating_offset, |
| 1160 | | first_floating_size, |
| 1161 | | ); |
| 1162 | | // Continue the search after the just-moved floating node |
| 1163 | | first_floating_oni = last_fixed.next; |
| 1164 | | continue; |
| 1165 | | } else { |
| 1166 | | assert(direction == .forward); |
| 1167 | | const new_parent_size = new_last_fixed_offset + last_fixed_size; |
| 1168 | | if (new_parent_size > old_parent_size) { |
| 1169 | | try mf.resizeNode( |
| 1170 | | gpa, |
| 1171 | | parent_ni, |
| 1172 | | new_parent_size +| new_parent_size / growth_factor, |
| 1173 | | ); |
| 1174 | | _, old_parent_size = parent.location().resolve(mf); |
| 1175 | | } |
| 1176 | | } |
| 1177 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1178 | | if (last_fixed_ni == ni) { |
| 1179 | | // The original fixed node now has enough space |
| 1180 | | last_fixed_ni.setLocationAssumeCapacity( |
| 1181 | | mf, |
| 1182 | | old_last_fixed_offset, |
| 1183 | | new_size, |
| 1184 | | ); |
| 1185 | | return; |
| 1186 | | } |
| 1187 | | // Move a fixed node into trailing free space |
| 1188 | | if (last_fixed.flags.has_content) { |
| 1178 | break :last_header header_ni; |
| 1179 | }; |
| 1180 | const last_header_offset, const last_header_size = last_header_ni.location(mf).resolve(mf); |
| 1181 | const old_headers_size = last_header_offset + last_header_size; |
| 1182 | |
| 1183 | // This is the first footer *inside* of `ni`. |
| 1184 | const first_sub_footer_oni = ni.firstFooter(mf); |
| 1185 | const sub_footers_size = size: { |
| 1186 | const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; |
| 1187 | const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); |
| 1188 | break :size old_size - first_sub_footer_offset; |
| 1189 | }; |
| 1190 | |
| 1191 | // We need to shift two things forwards; any header nodes which follow us, and any |
| 1192 | // footer nodes *within* us (since they need to be at the end of our new size). |
| 1189 | 1193 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| 1190 | 1194 | try mf.moveRange( |
| 1191 | | parent_file_offset + old_last_fixed_offset, |
| 1192 | | parent_file_offset + new_last_fixed_offset, |
| 1193 | | last_fixed_size, |
| 1195 | parent_file_offset + old_offset + old_size - sub_footers_size, |
| 1196 | parent_file_offset + old_offset + new_size - sub_footers_size, |
| 1197 | old_headers_size - (old_offset + old_size - sub_footers_size), |
| 1194 | 1198 | ); |
| 1195 | | } |
| 1196 | | last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); |
| 1197 | | // Retry the previous nodes now that there is enough space |
| 1198 | | first_floating_oni = .wrap(last_fixed_ni); |
| 1199 | | last_fixed_ni = last_fixed.prev.unwrap().?; |
| 1200 | | direction = .reverse; |
| 1201 | | } |
| 1202 | | } |
| 1203 | | |
| 1204 | | fn realignNode( |
| 1205 | | mf: *MappedFile, |
| 1206 | | gpa: Allocator, |
| 1207 | | ni: Node.Index, |
| 1208 | | new_alignment: Alignment, |
| 1209 | | opts: Node.Index.RealignNodeOptions, |
| 1210 | | ) (Allocator.Error || Io.Cancelable || IoError)!void { |
| 1211 | | mf.nodes_lock.assertUnlocked(); |
| 1212 | 1199 | |
| 1213 | | const node = ni.get(mf); |
| 1214 | | { |
| 1215 | | const prev_alignment = node.flags.alignment; |
| 1216 | | node.flags.alignment = new_alignment; |
| 1217 | | if (new_alignment.compare(.lte, prev_alignment)) return; |
| 1200 | // Any footers inside of us have had their offsets changed due to us growing: |
| 1201 | if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { |
| 1202 | var cur_ni = first_sub_footer_ni; |
| 1203 | while (true) { |
| 1204 | const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); |
| 1205 | try cur_ni.setLocation( |
| 1206 | mf, |
| 1207 | gpa, |
| 1208 | old_sub_footer_offset + (new_size - old_size), |
| 1209 | sub_footer_size, |
| 1210 | ); |
| 1211 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| 1212 | } |
| 1213 | } |
| 1214 | |
| 1215 | // Update the offsets of all header nodes following us: |
| 1216 | { |
| 1217 | var moved_header_ni = last_header_ni; |
| 1218 | while (moved_header_ni != ni) { |
| 1219 | assert(moved_header_ni.position(mf) == .header); |
| 1220 | const moved_header_offset, const moved_header_size = moved_header_ni.location(mf).resolve(mf); |
| 1221 | try moved_header_ni.setLocation( |
| 1222 | mf, |
| 1223 | gpa, |
| 1224 | moved_header_offset - old_size + new_size, |
| 1225 | moved_header_size, |
| 1226 | ); |
| 1227 | moved_header_ni = moved_header_ni.prev(mf).unwrap().?; |
| 1228 | } |
| 1229 | } |
| 1230 | |
| 1231 | // Finally, update our own size: |
| 1232 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1233 | return; |
| 1234 | }, |
| 1235 | .floating => { |
| 1236 | try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); |
| 1237 | }, |
| 1238 | .footer => { |
| 1239 | if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { |
| 1240 | return; |
| 1241 | } |
| 1242 | |
| 1243 | try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); |
| 1244 | |
| 1245 | const first_footer_ni: Node.Index = first_footer: { |
| 1246 | var footer_ni = ni; |
| 1247 | while (true) { |
| 1248 | const prev_ni = footer_ni.prev(mf).unwrap() orelse break; |
| 1249 | if (prev_ni.position(mf) != .footer) break; |
| 1250 | footer_ni = prev_ni; |
| 1251 | } |
| 1252 | break :first_footer footer_ni; |
| 1253 | }; |
| 1254 | |
| 1255 | // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself |
| 1256 | // a footer within its parent). |
| 1257 | const first_sub_footer_oni = ni.firstFooter(mf); |
| 1258 | const sub_footers_size = size: { |
| 1259 | const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; |
| 1260 | const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); |
| 1261 | break :size old_size - first_sub_footer_offset; |
| 1262 | }; |
| 1263 | |
| 1264 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| 1265 | |
| 1266 | const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; |
| 1267 | const new_footers_size = old_footers_size - old_size + new_size; |
| 1268 | |
| 1269 | // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node |
| 1270 | // itself needs to shift its contents, because our offset was shifted backwards by |
| 1271 | // `new_size - old_size`, and the added bytes should go at the end of this footer node. |
| 1272 | // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so |
| 1273 | // we *shouldn't* shift *that* data. |
| 1274 | const old_footers_start = parent_size - old_footers_size; |
| 1275 | const new_footers_start = parent_size - new_footers_size; |
| 1276 | const end_offset = node.location().resolve(mf)[0] + old_size; |
| 1277 | const parent_file_offset = parent_ni.fileLocation(mf, false).offset; |
| 1278 | try mf.moveRange( |
| 1279 | parent_file_offset + old_footers_start, |
| 1280 | parent_file_offset + new_footers_start, |
| 1281 | end_offset - old_footers_start - sub_footers_size, |
| 1282 | ); |
| 1283 | |
| 1284 | // Update our own offset and size: |
| 1285 | try ni.setLocation(mf, gpa, end_offset - new_size, new_size); |
| 1286 | |
| 1287 | // Any footers inside of us have had their offsets changed due to us growing: |
| 1288 | if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { |
| 1289 | var cur_ni = first_sub_footer_ni; |
| 1290 | while (true) { |
| 1291 | const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); |
| 1292 | try cur_ni.setLocation( |
| 1293 | mf, |
| 1294 | gpa, |
| 1295 | old_sub_footer_offset + (new_size - old_size), |
| 1296 | sub_footer_size, |
| 1297 | ); |
| 1298 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| 1299 | } |
| 1300 | } |
| 1301 | |
| 1302 | // Finally, update the offsets of every footer before us: |
| 1303 | if (node.prev.unwrap()) |prev_ni| { |
| 1304 | var maybe_footer_ni = prev_ni; |
| 1305 | while (true) { |
| 1306 | switch (maybe_footer_ni.position(mf)) { |
| 1307 | .header, .floating => break, |
| 1308 | .footer => {}, |
| 1309 | } |
| 1310 | const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); |
| 1311 | try maybe_footer_ni.setLocation( |
| 1312 | mf, |
| 1313 | gpa, |
| 1314 | moved_footer_offset + old_size - new_size, |
| 1315 | moved_footer_size, |
| 1316 | ); |
| 1317 | maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; |
| 1318 | } |
| 1319 | } |
| 1320 | |
| 1321 | return; |
| 1322 | }, |
| 1218 | 1323 | } |
| 1324 | } |
| 1219 | 1325 | |
| 1220 | | const old_offset, const size = node.location().resolve(mf); |
| 1221 | | const parent_ni = node.parent.unwrap() orelse { |
| 1222 | | assert(ni == .root); |
| 1223 | | return mf.resizeNode(gpa, ni, size); |
| 1224 | | }; |
| 1326 | /// Moves a floating node to an unused region with the given size, which may be greater than the |
| 1327 | /// current size. If `new_alignment` is not `null`, then the offset and size of the new region will |
| 1328 | /// have that alignment instead of `ni.alignment(mf)`. |
| 1329 | /// |
| 1330 | /// Asserts that `ni` is a floating node (and not `.root`). |
| 1331 | /// |
| 1332 | /// Asserts that `new_size` is aligned to `new_alignment orelse ni.alignment(mf)`. |
| 1333 | /// |
| 1334 | /// Asserts that `new_size` is greater than or equal to the current size of `ni`. |
| 1335 | fn growFloatingNodeWithAlignment( |
| 1336 | mf: *MappedFile, |
| 1337 | gpa: Allocator, |
| 1338 | ni: Node.Index, |
| 1339 | new_alignment: ?Alignment, |
| 1340 | new_size: u64, |
| 1341 | grow_mode: GrowMode, |
| 1342 | ) Error!void { |
| 1343 | mf.nodes_lock.assertUnlocked(); |
| 1225 | 1344 | |
| 1226 | | const new_size = new_alignment.forward(@intCast(size)); |
| 1227 | | if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); |
| 1345 | const parent_ni = ni.parent(mf).unwrap().?; // `ni` cannot be `.root` |
| 1346 | const old_offset, const old_size = ni.location(mf).resolve(mf); |
| 1228 | 1347 | |
| 1229 | | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| 1230 | | const trailing_end = trailing_end: { |
| 1231 | | const next_ni = node.next.unwrap() orelse break :trailing_end parent_size; |
| 1232 | | const next_offset, _ = next_ni.location(mf).resolve(mf); |
| 1233 | | break :trailing_end next_offset; |
| 1234 | | }; |
| 1348 | const alignment = new_alignment orelse ni.alignment(mf); |
| 1235 | 1349 | |
| 1236 | | if (opts.try_backwards) { |
| 1237 | | const backward_offset = new_alignment.backward(@intCast(old_offset)); |
| 1238 | | const prev_end = prev_end: { |
| 1239 | | const prev_ni = node.prev.unwrap() orelse break :prev_end 0; |
| 1240 | | const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); |
| 1241 | | break :prev_end prev_offset + prev_size; |
| 1350 | assert(new_size >= old_size); |
| 1351 | assert(ni.position(mf) == .floating); |
| 1352 | assert(alignment.check(new_size)); |
| 1353 | |
| 1354 | grow_in_place: { |
| 1355 | if (!alignment.check(old_offset)) { |
| 1356 | break :grow_in_place; |
| 1357 | } |
| 1358 | const limit: u64 = limit: { |
| 1359 | const next_ni = ni.next(mf).unwrap() orelse break :limit parent_ni.location(mf).resolve(mf)[1]; |
| 1360 | const next_offset, _ = next_ni.location(mf).resolve(mf); |
| 1361 | break :limit next_offset; |
| 1242 | 1362 | }; |
| 1363 | if (old_offset + new_size > limit) { |
| 1364 | break :grow_in_place; // the parent is not big enough |
| 1365 | } |
| 1366 | // Great, we can grow this node without changing its offset or moving any siblings. |
| 1367 | try ni.setLocation(mf, gpa, old_offset, new_size); |
| 1368 | // If we have any footers, we need to move them to the end of our new size, and update their |
| 1369 | // offsets accordingly. |
| 1370 | if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { |
| 1371 | var cur_ni = first_footer_ni; |
| 1372 | var footers_have_content = false; |
| 1373 | while (true) { |
| 1374 | footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; |
| 1375 | const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); |
| 1376 | try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); |
| 1377 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| 1378 | } |
| 1379 | if (footers_have_content) { |
| 1380 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| 1381 | // This gets the *new* offset because we already updated the offsets above. |
| 1382 | const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1383 | const footers_size = new_size - new_footers_offset; |
| 1384 | try mf.moveRange( |
| 1385 | parent_file_off + old_offset + old_size - footers_size, |
| 1386 | parent_file_off + old_offset + new_size - footers_size, |
| 1387 | footers_size, |
| 1388 | ); |
| 1389 | } |
| 1390 | } |
| 1391 | return; |
| 1392 | } |
| 1243 | 1393 | |
| 1244 | | if (backward_offset >= prev_end) { |
| 1245 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1394 | const new_loc: struct { |
| 1395 | offset: u64, |
| 1396 | prev: Node.Index.Optional, |
| 1397 | } = new_loc: { |
| 1398 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| 1399 | |
| 1400 | { |
| 1401 | // See if there's space at the start of the parent. |
| 1402 | const last_header_oni = parent_ni.lastHeader(mf); |
| 1403 | const headers_end: u64 = if (last_header_oni.unwrap()) |last_header_ni| headers_end: { |
| 1404 | const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); |
| 1405 | break :headers_end last_header_off + last_header_size; |
| 1406 | } else 0; |
| 1407 | const limit: u64 = limit: { |
| 1408 | const after_header_oni: Node.Index.Optional = after_header: { |
| 1409 | if (last_header_oni.unwrap()) |last_header_ni| { |
| 1410 | break :after_header last_header_ni.next(mf); |
| 1411 | } |
| 1412 | break :after_header parent_ni.first(mf); |
| 1413 | }; |
| 1414 | if (after_header_oni.unwrap()) |after_header_ni| { |
| 1415 | break :limit after_header_ni.location(mf).resolve(mf)[0]; |
| 1416 | } else { |
| 1417 | break :limit parent_size; |
| 1418 | } |
| 1419 | }; |
| 1420 | if (alignment.forward(headers_end) + new_size <= limit) { |
| 1421 | // There's space here! |
| 1422 | break :new_loc .{ |
| 1423 | // Put ourselves at the *end* of this range, so that the free space remains at the start of the parent. |
| 1424 | .offset = alignment.backward(limit - new_size), |
| 1425 | .prev = last_header_oni, |
| 1426 | }; |
| 1427 | } |
| 1428 | } |
| 1246 | 1429 | |
| 1247 | | if (node.flags.has_content) { |
| 1248 | | const old_file_offset = ni.fileLocation(mf, false).offset; |
| 1249 | | const new_file_offset = (old_file_offset - old_offset) + backward_offset; |
| 1250 | | @memmove( |
| 1251 | | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], |
| 1252 | | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], |
| 1253 | | ); |
| 1254 | | @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..@intCast(old_file_offset + size)], 0); |
| 1430 | // Otherwise, use space at the end of the parent, or make space there if necessary. |
| 1431 | |
| 1432 | const first_footer_oni = parent_ni.firstFooter(mf); |
| 1433 | |
| 1434 | // We know there is a node before the footer[s], because `ni` itself is such a node. |
| 1435 | const prev_ni: Node.Index = if (first_footer_oni.unwrap()) |first_footer_ni| prev: { |
| 1436 | break :prev first_footer_ni.prev(mf).unwrap().?; |
| 1437 | } else prev: { |
| 1438 | break :prev parent_ni.last(mf).unwrap().?; |
| 1439 | }; |
| 1440 | |
| 1441 | const result_offset: u64 = result_offset: { |
| 1442 | if (prev_ni == ni and alignment.check(old_offset)) { |
| 1443 | // We're already at the end of the parent, and our offset is already well-aligned. |
| 1444 | // The only reason we didn't simply grow in place earlier is that the parent wasn't |
| 1445 | // big enough---but now we're resizing the parent anyway, so growing in-place stops |
| 1446 | // us from unnecessarily moving! |
| 1447 | break :result_offset old_offset; |
| 1255 | 1448 | } |
| 1449 | // Otherwise, just move after the last node. |
| 1450 | const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); |
| 1451 | break :result_offset alignment.forward(prev_offset + prev_size); |
| 1452 | }; |
| 1256 | 1453 | |
| 1257 | | if (backward_offset + new_size <= trailing_end) { |
| 1258 | | ni.setLocationAssumeCapacity(mf, backward_offset, new_size); |
| 1259 | | } else { |
| 1260 | | ni.setLocationAssumeCapacity(mf, backward_offset, size); |
| 1261 | | try mf.resizeNode(gpa, ni, new_size); |
| 1454 | const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| footers_size: { |
| 1455 | const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1456 | break :footers_size parent_size - first_footer_offset; |
| 1457 | } else 0; |
| 1458 | |
| 1459 | const min_parent_size = result_offset + new_size + footers_size; |
| 1460 | if (parent_size < min_parent_size) { |
| 1461 | // Okay, at this point we're planning to expand the parent---so before we actually do |
| 1462 | // that, let's first try the Linux "insert range" fast path. We didn't try it before now |
| 1463 | // because it would have been more efficient to just move ourselves into existing space. |
| 1464 | // |
| 1465 | // If we were given a custom alignment, we cannot pass `grow_mode` directly into the |
| 1466 | // "insert range" path, because that function is unaware of `new_alignment`. |
| 1467 | const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; |
| 1468 | if (alignment.check(old_offset) and |
| 1469 | try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) |
| 1470 | { |
| 1471 | // The Linux fast path did our job for us! |
| 1472 | return; |
| 1262 | 1473 | } |
| 1263 | 1474 | |
| 1264 | | return; |
| 1475 | // Grow the parent and move to the end of the parent. |
| 1476 | const new_parent_size = parent_ni.alignment(mf).forward( |
| 1477 | min_parent_size +| min_parent_size / growth_factor, |
| 1478 | ); |
| 1479 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| 1480 | } |
| 1481 | |
| 1482 | break :new_loc .{ |
| 1483 | .offset = result_offset, |
| 1484 | .prev = .wrap(prev_ni), |
| 1485 | }; |
| 1486 | }; |
| 1487 | |
| 1488 | // We've found our new location in `parent_ni`, now to actually move ourselves there. |
| 1489 | |
| 1490 | // Footers need to move to a different place than the rest of our content. |
| 1491 | const footers_size: u64, const footers_have_content: bool = footers: { |
| 1492 | const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { |
| 1493 | break :footers .{ 0, false }; |
| 1494 | }; |
| 1495 | |
| 1496 | var cur_ni = first_footer_ni; |
| 1497 | var footers_have_content = false; |
| 1498 | while (true) { |
| 1499 | footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; |
| 1500 | const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); |
| 1501 | // Our footers' offsets must change to be at the end of our new size. |
| 1502 | try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); |
| 1503 | cur_ni = cur_ni.next(mf).unwrap() orelse break; |
| 1265 | 1504 | } |
| 1505 | |
| 1506 | // This is the *new* offset because we already updated the offsets above. |
| 1507 | const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1508 | const footers_size = new_size - new_footers_offset; |
| 1509 | |
| 1510 | break :footers .{ footers_size, footers_have_content }; |
| 1511 | }; |
| 1512 | |
| 1513 | if (ni.get(mf).flags.has_content) { |
| 1514 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| 1515 | try mf.moveRange( |
| 1516 | parent_file_off + old_offset, |
| 1517 | parent_file_off + new_loc.offset, |
| 1518 | old_size - footers_size, |
| 1519 | ); |
| 1520 | if (footers_have_content) try mf.moveRange( |
| 1521 | parent_file_off + old_offset + old_size - footers_size, |
| 1522 | parent_file_off + new_loc.offset + new_size - footers_size, |
| 1523 | footers_size, |
| 1524 | ); |
| 1525 | } else { |
| 1526 | assert(!footers_have_content); |
| 1266 | 1527 | } |
| 1267 | 1528 | |
| 1268 | | const forward_offset = new_alignment.forward(@intCast(old_offset)); |
| 1269 | | if (forward_offset + new_size <= trailing_end) { |
| 1270 | | // Shift into the free space if possible |
| 1271 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1272 | | if (node.flags.has_content) { |
| 1273 | | const old_file_offset = ni.fileLocation(mf, false).offset; |
| 1274 | | const new_file_offset = (old_file_offset - old_offset) + forward_offset; |
| 1275 | | if (new_file_offset < old_file_offset + size) { |
| 1276 | | @memmove( |
| 1277 | | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], |
| 1278 | | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], |
| 1279 | | ); |
| 1280 | | } else try mf.moveRange(old_file_offset, new_file_offset, size); |
| 1281 | | @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..][0..@intCast(new_size - size)], 0); |
| 1529 | try ni.setLocation(mf, gpa, new_loc.offset, new_size); |
| 1530 | |
| 1531 | if (new_loc.prev != ni.toOptional()) { |
| 1532 | // We're potentially in a different place in `parent_ni`'s child list, so remove and re-add ourselves. |
| 1533 | try mf.removeNodesFromChildList(gpa, ni, ni); |
| 1534 | try mf.addNodesToChildListAfter(gpa, new_loc.prev, ni, ni); |
| 1535 | } |
| 1536 | } |
| 1537 | |
| 1538 | /// Attempts to grow `ni` to `new_size` using `FALLOCATE_FL_INSERT_RANGE` on Linux. This strategy |
| 1539 | /// has the advantage that it does not require manually moving any bytes in the file, but has the |
| 1540 | /// disadvantages that it may increase the file size more than necessary, and that it changes the |
| 1541 | /// offsets of all following nodes, recursively. |
| 1542 | /// |
| 1543 | /// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` |
| 1544 | /// without changing any nodes' locations or invalidating any slices. |
| 1545 | /// |
| 1546 | /// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node |
| 1547 | /// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, |
| 1548 | /// the actual new size of `ni` may be greater than `new_size`. |
| 1549 | fn growNodeViaInsertRange( |
| 1550 | mf: *MappedFile, |
| 1551 | gpa: Allocator, |
| 1552 | ni: Node.Index, |
| 1553 | new_size: u64, |
| 1554 | grow_mode: GrowMode, |
| 1555 | ) Error!bool { |
| 1556 | if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { |
| 1557 | return false; |
| 1558 | } |
| 1559 | |
| 1560 | _, const old_size = ni.location(mf).resolve(mf); |
| 1561 | |
| 1562 | // We don't compute the size of the range yet, because depending on `grow_mode` we might want to |
| 1563 | // bump it based on our sibling and parent nodes' alignments. However, we can do an early check |
| 1564 | // for cases where we should obviously exit. |
| 1565 | const requested_range_size = new_size - old_size; |
| 1566 | if (!mf.flags.block_size.check(requested_range_size)) { |
| 1567 | // The requested size isn't exactly aligned. |
| 1568 | switch (grow_mode) { |
| 1569 | .exact => return false, |
| 1570 | .minimum => { |
| 1571 | // We can still choose to allow it by increasing the size a bit, but we shouldn't do |
| 1572 | // that if it would *significantly* increase the requested size. |
| 1573 | const block_size = mf.flags.block_size.toByteUnits(); |
| 1574 | if (requested_range_size < block_size * 2) { |
| 1575 | // Bumping this size up to the next block boundary would be a quite significant |
| 1576 | // increase; let's not do it. |
| 1577 | return false; |
| 1578 | } |
| 1579 | }, |
| 1580 | } |
| 1581 | } |
| 1582 | // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump |
| 1583 | // the size a little more. |
| 1584 | const min_range_size: u64 = s: { |
| 1585 | const exact_size = new_size - old_size; |
| 1586 | if (mf.flags.block_size.check(exact_size)) { |
| 1587 | break :s exact_size; |
| 1588 | } |
| 1589 | switch (grow_mode) { |
| 1590 | .exact => return false, |
| 1591 | .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { |
| 1592 | // We're growing by at least a few blocks, so allow ourselves to bump the size |
| 1593 | // slightly to give it the needed alignment. |
| 1594 | break :s mf.flags.block_size.forward(exact_size); |
| 1595 | } else { |
| 1596 | return false; |
| 1597 | }, |
| 1598 | } |
| 1599 | }; |
| 1600 | assert(min_range_size > 0); |
| 1601 | assert(mf.flags.block_size.check(min_range_size)); |
| 1602 | |
| 1603 | const range_file_offset: u64 = range_file_offset: { |
| 1604 | const node_file_offset = ni.fileLocation(mf, false).offset; |
| 1605 | const last_ni = ni.last(mf).unwrap() orelse { |
| 1606 | // If `ni` has no children (i.e. is a leaf node), we need to insert exactly at its end. |
| 1607 | const range_file_offset = node_file_offset + old_size; |
| 1608 | if (!mf.flags.block_size.check(range_file_offset)) { |
| 1609 | return false; |
| 1610 | } |
| 1611 | break :range_file_offset range_file_offset; |
| 1612 | }; |
| 1613 | const first_footer_oni = ni.firstFooter(mf); |
| 1614 | const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { |
| 1615 | const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); |
| 1616 | break :size old_size - first_footer_offset; |
| 1617 | } else 0; |
| 1618 | const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { |
| 1619 | break :pre_footer first_footer_ni.prev(mf); |
| 1620 | } else .wrap(last_ni); |
| 1621 | const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { |
| 1622 | const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); |
| 1623 | break :end pre_footer_off + pre_footer_size; |
| 1624 | } else 0; |
| 1625 | |
| 1626 | const min_file_offset = node_file_offset + pre_footer_end; |
| 1627 | const max_file_offset = node_file_offset + old_size - footers_size; |
| 1628 | // We can go anywhere between `min_file_offset` and `max_file_offset`. |
| 1629 | const candidate_file_offset = mf.flags.block_size.forward(min_file_offset); |
| 1630 | if (candidate_file_offset > max_file_offset) { |
| 1631 | return false; |
| 1282 | 1632 | } |
| 1633 | break :range_file_offset candidate_file_offset; |
| 1634 | }; |
| 1635 | assert(mf.flags.block_size.check(range_file_offset)); |
| 1636 | |
| 1637 | const range_size: u64 = range_size: { |
| 1638 | // For this strategy to be valid, the number of bytes we insert needs to be compatible with |
| 1639 | // the alignments of all nodes following us (and following our parents, their parents, etc). |
| 1640 | // We also probably don't want to trigger too many "node moved" events, since doing that |
| 1641 | // repeatedly could result in a lot of extra work. Therefore, while we traverse parents and |
| 1642 | // siblings to check their alignment requirements, we will also set an arbitrary limit on |
| 1643 | // the number of nodes we can move, and give up if we walk more than that. |
| 1644 | const max_moved_nodes = 32; |
| 1645 | var num_moved: u32 = 0; |
| 1646 | var cur_ni = ni; |
| 1647 | // Alignment required for `range_size`: initially the block size (required for the syscall), |
| 1648 | // then updated as we traverse based on how the operation would affect surrounding nodes. |
| 1649 | var need_range_align: Alignment = mf.flags.block_size.max(ni.alignment(mf)); |
| 1650 | while (true) { |
| 1651 | // `cur_ni` will grow as a result of the range insertion. Its size must be well-aligned. |
| 1652 | need_range_align = need_range_align.max(cur_ni.alignment(mf)); |
| 1653 | |
| 1654 | // Siblings following `cur_ni` don't get bigger, but their offsets change. |
| 1655 | while (cur_ni.next(mf).unwrap()) |next_ni| { |
| 1656 | // Only floating children need well-aligned offsets. |
| 1657 | if (next_ni.position(mf) == .floating) { |
| 1658 | need_range_align = need_range_align.max(next_ni.alignment(mf)); |
| 1659 | } |
| 1660 | num_moved += 1; |
| 1661 | if (num_moved > max_moved_nodes) return false; |
| 1662 | cur_ni = next_ni; |
| 1663 | } |
| 1664 | |
| 1665 | // Move up to the parent. |
| 1666 | cur_ni = cur_ni.parent(mf).unwrap() orelse break; |
| 1667 | } |
| 1668 | // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment |
| 1669 | // requirement to figure out whether we're actually going to insert a range. |
| 1670 | if (need_range_align.check(requested_range_size)) { |
| 1671 | break :range_size requested_range_size; |
| 1672 | } |
| 1673 | // Perhaps we're allowed to grow by more than `requested_range_size`? |
| 1674 | switch (grow_mode) { |
| 1675 | .exact => return false, |
| 1676 | .minimum => { |
| 1677 | const candidate_range_size = need_range_align.forward(min_range_size); |
| 1678 | // Allow growing by up to 50% more than was requested. |
| 1679 | if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { |
| 1680 | break :range_size candidate_range_size; |
| 1681 | } else { |
| 1682 | return false; |
| 1683 | } |
| 1684 | }, |
| 1685 | } |
| 1686 | }; |
| 1687 | |
| 1688 | // This `range_size` is compatible with everyone's alignment requirements, and we won't move too |
| 1689 | // many nodes, so let's do it! |
| 1690 | |
| 1691 | mf.memory_map.write(mf.io) catch |err| { |
| 1692 | mf.io_err = switch (err) { |
| 1693 | error.Canceled => |e| return e, |
| 1694 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 1695 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 1696 | else => |e| e, |
| 1697 | }; |
| 1698 | return error.MappedFileIo; |
| 1699 | }; |
| 1283 | 1700 | |
| 1284 | | ni.setLocationAssumeCapacity(mf, forward_offset, new_size); |
| 1701 | // If we happen to be inserting at the very end of the file, we need to resize the file instead |
| 1702 | // of using `FALLOCATE_FL_INSERT_RANGE`. |
| 1703 | if (range_file_offset == Node.Index.root.location(mf).resolve(mf)[1]) { |
| 1704 | mf.memory_map.file.setLength(mf.io, range_file_offset + range_size) catch |err| switch (err) { |
| 1705 | error.Canceled => |e| return e, |
| 1706 | else => |e| { |
| 1707 | mf.io_err = e; |
| 1708 | return error.MappedFileIo; |
| 1709 | }, |
| 1710 | }; |
| 1285 | 1711 | } else { |
| 1286 | | const temp_size = new_alignment.forward(@intCast(new_size + 1)); |
| 1287 | | try mf.resizeNode(gpa, ni, temp_size); |
| 1288 | | const new_offset, _ = ni.location(mf).resolve(mf); |
| 1289 | | |
| 1290 | | try mf.ensureCapacityForSetLocation(gpa); |
| 1291 | | |
| 1292 | | // Non-fixed nodes may now be aligned if the resize moved them |
| 1293 | | const new_forward_offset = new_alignment.forward(@intCast(new_offset)); |
| 1294 | | const final_offset = if (new_forward_offset != new_offset) final_offset: { |
| 1295 | | if (node.flags.has_content) { |
| 1296 | | const old_file_offset = ni.fileLocation(mf, false).offset; |
| 1297 | | const new_file_offset = (old_file_offset - new_offset) + new_forward_offset; |
| 1298 | | @memmove( |
| 1299 | | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], |
| 1300 | | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], |
| 1301 | | ); |
| 1302 | | @memset(mf.memory_map.memory[@intCast(old_file_offset)..@intCast(new_file_offset)], 0); |
| 1712 | while (true) switch (linux.errno(linux.fallocate( |
| 1713 | mf.memory_map.file.handle, |
| 1714 | linux.FALLOC.FL_INSERT_RANGE, |
| 1715 | @intCast(range_file_offset), |
| 1716 | @intCast(range_size), |
| 1717 | ))) { |
| 1718 | .SUCCESS => break, |
| 1719 | .INTR => continue, |
| 1720 | .NOSYS, .OPNOTSUPP => { |
| 1721 | // After all that setup work, it turns out the operation is actually unsupported! |
| 1722 | mf.flags.fallocate_insert_range_unsupported = true; |
| 1723 | return false; |
| 1724 | }, |
| 1725 | else => |e| { |
| 1726 | mf.io_err = switch (e) { |
| 1727 | .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above |
| 1728 | .BADF => unreachable, |
| 1729 | .FBIG => unreachable, |
| 1730 | .INVAL => unreachable, |
| 1731 | .IO => error.InputOutput, |
| 1732 | .NODEV => error.NotFile, |
| 1733 | .NOSPC => error.NoSpaceLeft, |
| 1734 | .PERM => error.PermissionDenied, |
| 1735 | .SPIPE => error.Unseekable, |
| 1736 | .TXTBSY => error.FileBusy, |
| 1737 | else => std.posix.unexpectedErrno(e), |
| 1738 | }; |
| 1739 | return error.MappedFileIo; |
| 1740 | }, |
| 1741 | }; |
| 1742 | } |
| 1743 | |
| 1744 | // We did it! Now to update all the sizes and offsets. This loop is exactly the same shape as |
| 1745 | // above, except we're updating locations instead of checking alignments. |
| 1746 | var cur_ni = ni; |
| 1747 | while (true) { |
| 1748 | const this_offset, const this_old_size = cur_ni.location(mf).resolve(mf); |
| 1749 | if (cur_ni == .root) { |
| 1750 | try mf.ensureTotalCapacityPrecise(@intCast(this_old_size + range_size)); |
| 1751 | } |
| 1752 | try cur_ni.setLocation(mf, gpa, this_offset, this_old_size + range_size); |
| 1753 | |
| 1754 | while (cur_ni.next(mf).unwrap()) |next_ni| { |
| 1755 | const next_old_offset, const next_size = next_ni.location(mf).resolve(mf); |
| 1756 | try next_ni.setLocation(mf, gpa, next_old_offset + range_size, next_size); |
| 1757 | cur_ni = next_ni; |
| 1758 | } |
| 1759 | |
| 1760 | cur_ni = cur_ni.parent(mf).unwrap() orelse break; |
| 1761 | } |
| 1762 | |
| 1763 | // The only thing left is to update the offsets of any footers inside of `ni`. |
| 1764 | if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { |
| 1765 | var footer_ni = first_footer_ni; |
| 1766 | while (true) { |
| 1767 | const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); |
| 1768 | try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); |
| 1769 | footer_ni = footer_ni.next(mf).unwrap() orelse break; |
| 1770 | } |
| 1771 | } |
| 1772 | |
| 1773 | return true; |
| 1774 | } |
| 1775 | |
| 1776 | /// Ensures that `parent_ni` has at least `extra_capacity` padding bytes following its current |
| 1777 | /// headers, so that the headers can grow into that space. |
| 1778 | fn ensureAdditionalHeaderCapacity( |
| 1779 | mf: *MappedFile, |
| 1780 | gpa: Allocator, |
| 1781 | parent_ni: Node.Index, |
| 1782 | extra_capacity: u64, |
| 1783 | ) Error!void { |
| 1784 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| 1785 | |
| 1786 | const last_header_oni = parent_ni.lastHeader(mf); |
| 1787 | const first_footer_oni = parent_ni.firstFooter(mf); |
| 1788 | |
| 1789 | const headers_size: u64 = headers_size: { |
| 1790 | const last_header_ni = last_header_oni.unwrap() orelse break :headers_size 0; |
| 1791 | const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); |
| 1792 | break :headers_size last_header_off + last_header_size; |
| 1793 | }; |
| 1794 | |
| 1795 | const footers_size: u64 = footers_size: { |
| 1796 | const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; |
| 1797 | const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); |
| 1798 | break :footers_size parent_size - first_footer_off; |
| 1799 | }; |
| 1800 | |
| 1801 | const first_floating_oni: Node.Index.Optional = if (last_header_oni.unwrap()) |last_header_ni| first_floating: { |
| 1802 | const after_header_ni = last_header_ni.next(mf).unwrap() orelse break :first_floating .none; |
| 1803 | break :first_floating switch (after_header_ni.position(mf)) { |
| 1804 | .header => unreachable, |
| 1805 | .floating => .wrap(after_header_ni), |
| 1806 | .footer => .none, |
| 1807 | }; |
| 1808 | } else first_floating: { |
| 1809 | const first_ni = parent_ni.first(mf).unwrap() orelse break :first_floating .none; |
| 1810 | break :first_floating switch (first_ni.position(mf)) { |
| 1811 | .header => unreachable, |
| 1812 | .floating => .wrap(first_ni), |
| 1813 | .footer => .none, |
| 1814 | }; |
| 1815 | }; |
| 1816 | const first_floating_ni = first_floating_oni.unwrap() orelse { |
| 1817 | // This node has only headers and footers. |
| 1818 | const min_parent_size = headers_size + extra_capacity + footers_size; |
| 1819 | if (parent_size < min_parent_size) { |
| 1820 | const new_parent_size = parent_ni.alignment(mf).forward( |
| 1821 | min_parent_size +| min_parent_size / growth_factor, |
| 1822 | ); |
| 1823 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| 1824 | } |
| 1825 | return; |
| 1826 | }; |
| 1827 | |
| 1828 | const last_floating_ni = if (first_footer_oni.unwrap()) |first_footer_ni| last_floating: { |
| 1829 | break :last_floating first_footer_ni.prev(mf).unwrap().?; |
| 1830 | } else last_floating: { |
| 1831 | break :last_floating parent_ni.last(mf).unwrap().?; |
| 1832 | }; |
| 1833 | assert(last_floating_ni.position(mf) == .floating); // we know `parent_ni` contains at least `first_floating_ni` |
| 1834 | |
| 1835 | // Find the first floating child, if any, which does not overlap the new header space. |
| 1836 | const first_good_floating_oni: Node.Index.Optional = first_good_floating: { |
| 1837 | var floating_ni = first_floating_ni; |
| 1838 | while (true) { |
| 1839 | const floating_offset, _ = floating_ni.location(mf).resolve(mf); |
| 1840 | if (floating_offset >= headers_size + extra_capacity) { |
| 1841 | break :first_good_floating .wrap(floating_ni); |
| 1842 | } |
| 1843 | const next_ni = floating_ni.next(mf).unwrap() orelse { |
| 1844 | break :first_good_floating .none; |
| 1845 | }; |
| 1846 | switch (next_ni.position(mf)) { |
| 1847 | .header => unreachable, // after the last header |
| 1848 | .floating => floating_ni = next_ni, |
| 1849 | .footer => break :first_good_floating .none, |
| 1303 | 1850 | } |
| 1851 | } |
| 1852 | }; |
| 1853 | |
| 1854 | if (first_good_floating_oni == first_floating_ni.toOptional()) { |
| 1855 | // None of the floating children are in our way! That means there's already enough space. |
| 1856 | return; |
| 1857 | } |
| 1858 | |
| 1859 | const last_moving_ni = if (first_good_floating_oni.unwrap()) |first_good_floating_ni| last_moving: { |
| 1860 | break :last_moving first_good_floating_ni.prev(mf).unwrap().?; |
| 1861 | } else last_moving: { |
| 1862 | break :last_moving last_floating_ni; |
| 1863 | }; |
| 1864 | |
| 1865 | // We are going to move all nodes between `first_floating_ni` and `last_moving_ni` to the end of |
| 1866 | // the parent. We'll move all the node data in one big block. |
| 1867 | |
| 1868 | const moving_offset: u64 = first_floating_ni.location(mf).resolve(mf)[0]; |
| 1869 | const moving_size: u64 = size: { |
| 1870 | const last_moving_off, const last_moving_size = last_moving_ni.location(mf).resolve(mf); |
| 1871 | break :size last_moving_off + last_moving_size - moving_offset; |
| 1872 | }; |
| 1873 | |
| 1874 | var moving_alignment: Alignment = .@"1"; |
| 1875 | var moving_has_content = false; // optimization: no need to move data if it's all uninitialized |
| 1876 | { |
| 1877 | var cur_ni = first_floating_ni; |
| 1878 | while (true) { |
| 1879 | moving_alignment = moving_alignment.max(cur_ni.alignment(mf)); |
| 1880 | moving_has_content = moving_has_content or cur_ni.get(mf).flags.has_content; |
| 1881 | if (cur_ni == last_moving_ni) break; |
| 1882 | cur_ni = cur_ni.next(mf).unwrap().?; |
| 1883 | } |
| 1884 | } |
| 1885 | |
| 1886 | const first_free_offset = free_offset: { |
| 1887 | const last_floating_off, const last_floating_size = last_floating_ni.location(mf).resolve(mf); |
| 1888 | break :free_offset @max(last_floating_off + last_floating_size, headers_size + extra_capacity); |
| 1889 | }; |
| 1890 | // Alignment is a little tricky here. We don't necessarily want the new offset to be aligned to |
| 1891 | // `moving_alignment` exactly, because if (e.g.) the first floating node is align(2) and the |
| 1892 | // second is align(4), then the overall range we're moving may not be 4-byte aligned even though |
| 1893 | // one of the nodes is. Instead, the old and new offsets must be congruent modulo the alignment. |
| 1894 | const aligned_dest_offset = moving_alignment.forward(first_free_offset); |
| 1895 | const dest_offset = aligned_dest_offset + (moving_offset - moving_alignment.backward(moving_offset)); |
| 1896 | assert(dest_offset % moving_alignment.toByteUnits() == moving_offset % moving_alignment.toByteUnits()); |
| 1897 | |
| 1898 | // This expression is correct because `dest_offset` is after all floating nodes (except the ones |
| 1899 | // we're moving there of course). |
| 1900 | const min_parent_size = dest_offset + moving_size + footers_size; |
| 1901 | if (parent_size < min_parent_size) { |
| 1902 | const new_parent_size = parent_ni.alignment(mf).forward( |
| 1903 | min_parent_size +| min_parent_size / growth_factor, |
| 1904 | ); |
| 1905 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| 1906 | } |
| 1907 | |
| 1908 | if (moving_has_content) { |
| 1909 | const parent_file_off = parent_ni.fileLocation(mf, false).offset; |
| 1910 | try mf.moveRange( |
| 1911 | parent_file_off + moving_offset, |
| 1912 | parent_file_off + dest_offset, |
| 1913 | moving_size, |
| 1914 | ); |
| 1915 | } |
| 1916 | |
| 1917 | // Remove everything between `first_floating_ni` and `last_moving_ni` from the linked list, then |
| 1918 | // re-insert them in their new position. |
| 1919 | try mf.removeNodesFromChildList(gpa, first_floating_ni, last_moving_ni); |
| 1920 | try mf.addNodesToChildListBefore(gpa, first_footer_oni, first_floating_ni, last_moving_ni); |
| 1921 | |
| 1922 | // Finally, we need to update the locations of all of those nodes. |
| 1923 | var cur_ni = first_floating_ni; |
| 1924 | while (true) { |
| 1925 | assert(cur_ni.position(mf) == .floating); |
| 1926 | const old_offset, const old_size = cur_ni.location(mf).resolve(mf); |
| 1927 | const new_offset = old_offset - moving_offset + dest_offset; |
| 1928 | assert(cur_ni.alignment(mf).check(new_offset)); |
| 1929 | try cur_ni.setLocation(mf, gpa, new_offset, old_size); |
| 1930 | if (cur_ni == last_moving_ni) break; |
| 1931 | cur_ni = cur_ni.next(mf).unwrap().?; |
| 1932 | } |
| 1933 | } |
| 1934 | |
| 1935 | /// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current |
| 1936 | /// footers, so that the footers can grow into that space. |
| 1937 | fn ensureAdditionalFooterCapacity( |
| 1938 | mf: *MappedFile, |
| 1939 | gpa: Allocator, |
| 1940 | parent_ni: Node.Index, |
| 1941 | extra_capacity: u64, |
| 1942 | ) Error!void { |
| 1943 | // This is way easier than the header case, because we don't need to actually move anything; we |
| 1944 | // just need to expand the parent if there isn't space, and that will add padding after the |
| 1945 | // parent's floating children, which is exactly where we want it. |
| 1946 | |
| 1947 | const first_footer_oni = parent_ni.firstFooter(mf); |
| 1948 | |
| 1949 | _, const parent_size = parent_ni.location(mf).resolve(mf); |
| 1950 | |
| 1951 | const footers_size: u64 = footers_size: { |
| 1952 | const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; |
| 1953 | const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); |
| 1954 | break :footers_size parent_size - first_footer_off; |
| 1955 | }; |
| 1956 | |
| 1957 | const header_and_floating_end: u64 = end: { |
| 1958 | const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { |
| 1959 | break :before_footers first_footer_ni.prev(mf); |
| 1960 | } else before_footers: { |
| 1961 | break :before_footers parent_ni.last(mf); |
| 1962 | }; |
| 1963 | const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; |
| 1964 | const offset, const size = before_footers_ni.location(mf).resolve(mf); |
| 1965 | break :end offset + size; |
| 1966 | }; |
| 1304 | 1967 | |
| 1305 | | break :final_offset new_forward_offset; |
| 1306 | | } else new_offset; |
| 1968 | assert(header_and_floating_end + footers_size <= parent_size); |
| 1307 | 1969 | |
| 1308 | | ni.setLocationAssumeCapacity(mf, final_offset, new_size); |
| 1970 | const min_parent_size = header_and_floating_end + footers_size + extra_capacity; |
| 1971 | if (parent_size < min_parent_size) { |
| 1972 | const new_parent_size = parent_ni.alignment(mf).forward( |
| 1973 | min_parent_size +| min_parent_size / growth_factor, |
| 1974 | ); |
| 1975 | try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); |
| 1309 | 1976 | } |
| 1310 | 1977 | } |
| 1311 | 1978 | |
| 1979 | fn removeNodesFromChildList( |
| 1980 | mf: *MappedFile, |
| 1981 | gpa: Allocator, |
| 1982 | first_remove_ni: Node.Index, |
| 1983 | last_remove_ni: Node.Index, |
| 1984 | ) Allocator.Error!void { |
| 1985 | const parent_ni = first_remove_ni.parent(mf).unwrap().?; |
| 1986 | assert(last_remove_ni.parent(mf).unwrap().? == parent_ni); |
| 1987 | |
| 1988 | const prev_oni = first_remove_ni.prev(mf); |
| 1989 | const next_oni = last_remove_ni.next(mf); |
| 1990 | |
| 1991 | if (prev_oni.unwrap()) |prev_ni| { |
| 1992 | assert(prev_ni.next(mf).unwrap().? == first_remove_ni); |
| 1993 | try prev_ni.setNext(gpa, next_oni, mf); |
| 1994 | } else { |
| 1995 | assert(parent_ni.first(mf).unwrap().? == first_remove_ni); |
| 1996 | parent_ni.get(mf).first = next_oni; |
| 1997 | } |
| 1998 | |
| 1999 | if (next_oni.unwrap()) |next_ni| { |
| 2000 | assert(next_ni.prev(mf).unwrap().? == last_remove_ni); |
| 2001 | next_ni.get(mf).prev = prev_oni; |
| 2002 | } else { |
| 2003 | assert(parent_ni.last(mf).unwrap().? == last_remove_ni); |
| 2004 | parent_ni.get(mf).last = prev_oni; |
| 2005 | } |
| 2006 | } |
| 2007 | /// Assumes `first_add_ni` and `last_add_ni` are connected, and that all nodes in between them |
| 2008 | /// already have their `parent` field correctly populated. |
| 2009 | /// |
| 2010 | /// To add a single node, set `first_add_ni` equal to `last_add_ni`. |
| 2011 | fn addNodesToChildListBefore( |
| 2012 | mf: *MappedFile, |
| 2013 | gpa: Allocator, |
| 2014 | /// `null` means to add at the end of the parent. |
| 2015 | next_oni: Node.Index.Optional, |
| 2016 | first_add_ni: Node.Index, |
| 2017 | last_add_ni: Node.Index, |
| 2018 | ) Allocator.Error!void { |
| 2019 | const parent_ni = first_add_ni.parent(mf).unwrap().?; |
| 2020 | assert(last_add_ni.parent(mf).unwrap().? == parent_ni); |
| 2021 | if (next_oni.unwrap()) |next_ni| { |
| 2022 | assert(next_ni.parent(mf).unwrap().? == parent_ni); |
| 2023 | } |
| 2024 | |
| 2025 | const prev_oni: Node.Index.Optional = if (next_oni.unwrap()) |next_ni| prev: { |
| 2026 | break :prev next_ni.prev(mf); |
| 2027 | } else prev: { |
| 2028 | break :prev parent_ni.last(mf); |
| 2029 | }; |
| 2030 | |
| 2031 | first_add_ni.get(mf).prev = prev_oni; |
| 2032 | try last_add_ni.setNext(gpa, next_oni, mf); |
| 2033 | |
| 2034 | if (prev_oni.unwrap()) |prev_ni| { |
| 2035 | assert(prev_ni.next(mf) == next_oni); |
| 2036 | try prev_ni.setNext(gpa, .wrap(first_add_ni), mf); |
| 2037 | } else { |
| 2038 | assert(parent_ni.first(mf) == next_oni); |
| 2039 | parent_ni.get(mf).first = .wrap(first_add_ni); |
| 2040 | } |
| 2041 | |
| 2042 | if (next_oni.unwrap()) |next_ni| { |
| 2043 | assert(next_ni.prev(mf) == prev_oni); |
| 2044 | next_ni.get(mf).prev = .wrap(last_add_ni); |
| 2045 | } else { |
| 2046 | assert(parent_ni.last(mf) == prev_oni); |
| 2047 | parent_ni.get(mf).last = .wrap(last_add_ni); |
| 2048 | } |
| 2049 | } |
| 2050 | fn addNodesToChildListAfter( |
| 2051 | mf: *MappedFile, |
| 2052 | gpa: Allocator, |
| 2053 | /// `null` means to add at the start of the parent. |
| 2054 | prev_oni: Node.Index.Optional, |
| 2055 | first_add_ni: Node.Index, |
| 2056 | last_add_ni: Node.Index, |
| 2057 | ) Allocator.Error!void { |
| 2058 | const next_oni: Node.Index.Optional = next: { |
| 2059 | if (prev_oni.unwrap()) |prev_ni| break :next prev_ni.next(mf); |
| 2060 | const parent_ni = first_add_ni.parent(mf).unwrap().?; |
| 2061 | break :next parent_ni.first(mf); |
| 2062 | }; |
| 2063 | return mf.addNodesToChildListBefore(gpa, next_oni, first_add_ni, last_add_ni); |
| 2064 | } |
| 2065 | |
| 2066 | fn realignNode( |
| 2067 | mf: *MappedFile, |
| 2068 | gpa: Allocator, |
| 2069 | ni: Node.Index, |
| 2070 | new_alignment: Alignment, |
| 2071 | ) Error!void { |
| 2072 | mf.nodes_lock.assertUnlocked(); |
| 2073 | |
| 2074 | const old_offset, const old_size = ni.location(mf).resolve(mf); |
| 2075 | |
| 2076 | if (ni == .root or ni.position(mf) != .floating) { |
| 2077 | // Only this node's size is aligned, not its offset. |
| 2078 | if (!new_alignment.check(old_size)) { |
| 2079 | assert(new_alignment.compare(.gt, ni.alignment(mf))); |
| 2080 | try mf.growNode( |
| 2081 | gpa, |
| 2082 | ni, |
| 2083 | new_alignment.forward(old_size), |
| 2084 | .exact, // because `growNode` is not aware that the size needs to match `new_alignment` |
| 2085 | ); |
| 2086 | } |
| 2087 | } else { |
| 2088 | // This is a floating node, so its size and offset are both aligned. |
| 2089 | if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { |
| 2090 | assert(new_alignment.compare(.gt, ni.alignment(mf))); |
| 2091 | try mf.growFloatingNodeWithAlignment( |
| 2092 | gpa, |
| 2093 | ni, |
| 2094 | new_alignment, |
| 2095 | new_alignment.forward(old_size), |
| 2096 | .minimum, |
| 2097 | ); |
| 2098 | } |
| 2099 | } |
| 2100 | |
| 2101 | ni.get(mf).flags.alignment = new_alignment; |
| 2102 | } |
| 2103 | |
| 1312 | 2104 | fn updateWriters(mf: *MappedFile) void { |
| 1313 | 2105 | var writers_it = mf.writers.first; |
| 1314 | 2106 | while (writers_it) |writer_node| : (writers_it = writer_node.next) { |
| ... | ... | @@ -1317,10 +2109,47 @@ fn updateWriters(mf: *MappedFile) void { |
| 1317 | 2109 | } |
| 1318 | 2110 | } |
| 1319 | 2111 | |
| 1320 | | fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { |
| 1321 | | // make a copy of this node at the new location |
| 1322 | | try mf.copyRange(old_file_offset, new_file_offset, size); |
| 1323 | | // delete the copy of this node at the old location |
| 2112 | fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) Error!void { |
| 2113 | if (old_file_offset == new_file_offset) return; |
| 2114 | |
| 2115 | if (old_file_offset >= new_file_offset + size or |
| 2116 | new_file_offset >= old_file_offset + size) |
| 2117 | { |
| 2118 | const n = try mf.copyFileRange( |
| 2119 | mf.memory_map.file, |
| 2120 | old_file_offset, |
| 2121 | new_file_offset, |
| 2122 | size, |
| 2123 | ); |
| 2124 | @memcpy( |
| 2125 | mf.memory_map.memory[@intCast(new_file_offset + n)..][0..@intCast(size - n)], |
| 2126 | mf.memory_map.memory[@intCast(old_file_offset + n)..][0..@intCast(size - n)], |
| 2127 | ); |
| 2128 | |
| 2129 | try mf.zeroRange(old_file_offset, size); |
| 2130 | |
| 2131 | return; |
| 2132 | } |
| 2133 | |
| 2134 | // TODO: if the non-overlapping region is greater than or equal to a filesystem block, is it |
| 2135 | // ever worth doing multiple `copyFileRange` calls instead of a big `@memmove`? |
| 2136 | |
| 2137 | @memmove( |
| 2138 | mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], |
| 2139 | mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], |
| 2140 | ); |
| 2141 | |
| 2142 | if (new_file_offset > old_file_offset) { |
| 2143 | const clear_size = new_file_offset - old_file_offset; |
| 2144 | assert(clear_size < size); |
| 2145 | try mf.zeroRange(old_file_offset, clear_size); |
| 2146 | } else { |
| 2147 | const clear_size = old_file_offset - new_file_offset; |
| 2148 | assert(clear_size < size); |
| 2149 | try mf.zeroRange(new_file_offset + size, clear_size); |
| 2150 | } |
| 2151 | } |
| 2152 | fn zeroRange(mf: *MappedFile, file_offset: u64, size: u64) Error!void { |
| 1324 | 2153 | if (is_linux and |
| 1325 | 2154 | !mf.flags.fallocate_punch_hole_unsupported and |
| 1326 | 2155 | size >= mf.flags.block_size.toByteUnits() * 2 - 1) |
| ... | ... | @@ -1328,147 +2157,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: |
| 1328 | 2157 | while (true) switch (linux.errno(linux.fallocate( |
| 1329 | 2158 | mf.memory_map.file.handle, |
| 1330 | 2159 | linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, |
| 1331 | | @intCast(old_file_offset), |
| 2160 | @intCast(file_offset), |
| 1332 | 2161 | @intCast(size), |
| 1333 | 2162 | ))) { |
| 1334 | 2163 | .SUCCESS => return, |
| 1335 | 2164 | .INTR => continue, |
| 1336 | | .BADF, .FBIG, .INVAL => unreachable, |
| 1337 | | .IO => return error.InputOutput, |
| 1338 | | .NODEV => return error.NotFile, |
| 1339 | | .NOSPC => return error.NoSpaceLeft, |
| 1340 | 2165 | .NOSYS, .OPNOTSUPP => { |
| 1341 | 2166 | mf.flags.fallocate_punch_hole_unsupported = true; |
| 1342 | 2167 | break; // fall back to slow path |
| 1343 | 2168 | }, |
| 1344 | | .PERM => return error.PermissionDenied, |
| 1345 | | .SPIPE => return error.Unseekable, |
| 1346 | | .TXTBSY => return error.FileBusy, |
| 1347 | | else => |e| return std.posix.unexpectedErrno(e), |
| 2169 | else => |e| { |
| 2170 | mf.io_err = switch (e) { |
| 2171 | .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above |
| 2172 | .BADF => unreachable, |
| 2173 | .FBIG => unreachable, |
| 2174 | .INVAL => unreachable, |
| 2175 | .IO => error.InputOutput, |
| 2176 | .NODEV => error.NotFile, |
| 2177 | .NOSPC => error.NoSpaceLeft, |
| 2178 | .PERM => error.PermissionDenied, |
| 2179 | .SPIPE => error.Unseekable, |
| 2180 | .TXTBSY => error.FileBusy, |
| 2181 | else => std.posix.unexpectedErrno(e), |
| 2182 | }; |
| 2183 | return error.MappedFileIo; |
| 2184 | }, |
| 1348 | 2185 | }; |
| 1349 | 2186 | } |
| 1350 | | @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0); |
| 1351 | | } |
| 1352 | | |
| 1353 | | fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { |
| 1354 | | const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size); |
| 1355 | | if (copy_size < size) @memcpy( |
| 1356 | | mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], |
| 1357 | | mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], |
| 1358 | | ); |
| 2187 | @memset(mf.memory_map.memory[@intCast(file_offset)..][0..@intCast(size)], 0); |
| 1359 | 2188 | } |
| 1360 | | |
| 1361 | 2189 | fn copyFileRange( |
| 1362 | 2190 | mf: *MappedFile, |
| 1363 | 2191 | old_file: Io.File, |
| 1364 | 2192 | old_file_offset: u64, |
| 1365 | 2193 | new_file_offset: u64, |
| 1366 | 2194 | size: u64, |
| 1367 | | ) (Io.Cancelable || IoError)!u64 { |
| 2195 | ) Error!u64 { |
| 2196 | if (!is_linux or mf.flags.copy_file_range_unsupported) { |
| 2197 | return 0; |
| 2198 | } |
| 2199 | |
| 2200 | const min_size = mf.flags.block_size.toByteUnits() * 2 - 1; |
| 2201 | if (size < min_size) return 0; |
| 2202 | |
| 1368 | 2203 | const io = mf.io; |
| 1369 | | mf.memory_map.write(io) catch |err| switch (err) { |
| 1370 | | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking |
| 1371 | | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing |
| 1372 | | else => |e| return e, |
| 2204 | mf.memory_map.write(io) catch |err| { |
| 2205 | mf.io_err = switch (err) { |
| 2206 | error.Canceled => |e| return e, |
| 2207 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 2208 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 2209 | else => |e| e, |
| 2210 | }; |
| 2211 | return error.MappedFileIo; |
| 1373 | 2212 | }; |
| 1374 | 2213 | var remaining_size = size; |
| 1375 | | if (is_linux and !mf.flags.copy_file_range_unsupported) { |
| 1376 | | var old_file_offset_mut: i64 = @intCast(old_file_offset); |
| 1377 | | var new_file_offset_mut: i64 = @intCast(new_file_offset); |
| 1378 | | while (remaining_size >= mf.flags.block_size.toByteUnits() * 2 - 1) { |
| 1379 | | const copy_len = linux.copy_file_range( |
| 1380 | | old_file.handle, |
| 1381 | | &old_file_offset_mut, |
| 1382 | | mf.memory_map.file.handle, |
| 1383 | | &new_file_offset_mut, |
| 1384 | | @intCast(remaining_size), |
| 1385 | | 0, |
| 1386 | | ); |
| 1387 | | switch (linux.errno(copy_len)) { |
| 1388 | | .SUCCESS => { |
| 1389 | | if (copy_len == 0) break; |
| 1390 | | remaining_size -= copy_len; |
| 1391 | | if (remaining_size == 0) break; |
| 1392 | | }, |
| 1393 | | .INTR => continue, |
| 1394 | | .BADF, .FBIG, .INVAL, .OVERFLOW => unreachable, |
| 1395 | | .IO => return error.InputOutput, |
| 1396 | | .ISDIR => return error.IsDir, |
| 1397 | | .NOMEM => return error.SystemResources, |
| 1398 | | .NOSPC => return error.NoSpaceLeft, |
| 1399 | | .NOSYS, .OPNOTSUPP, .XDEV => { |
| 1400 | | mf.flags.copy_file_range_unsupported = true; |
| 1401 | | break; |
| 1402 | | }, |
| 1403 | | .PERM => return error.PermissionDenied, |
| 1404 | | .TXTBSY => return error.FileBusy, |
| 1405 | | else => |e| return std.posix.unexpectedErrno(e), |
| 1406 | | } |
| 2214 | var old_file_offset_mut: i64 = @intCast(old_file_offset); |
| 2215 | var new_file_offset_mut: i64 = @intCast(new_file_offset); |
| 2216 | while (remaining_size >= min_size) { |
| 2217 | const copy_len = linux.copy_file_range( |
| 2218 | old_file.handle, |
| 2219 | &old_file_offset_mut, |
| 2220 | mf.memory_map.file.handle, |
| 2221 | &new_file_offset_mut, |
| 2222 | @intCast(remaining_size), |
| 2223 | 0, |
| 2224 | ); |
| 2225 | switch (linux.errno(copy_len)) { |
| 2226 | .SUCCESS => { |
| 2227 | if (copy_len == 0) break; |
| 2228 | remaining_size -= copy_len; |
| 2229 | if (remaining_size == 0) break; |
| 2230 | }, |
| 2231 | .INTR => continue, |
| 2232 | .NOSYS, .OPNOTSUPP, .XDEV => { |
| 2233 | mf.flags.copy_file_range_unsupported = true; |
| 2234 | break; |
| 2235 | }, |
| 2236 | else => |e| { |
| 2237 | mf.io_err = switch (e) { |
| 2238 | .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP, .XDEV => unreachable, // handled above |
| 2239 | .BADF => unreachable, |
| 2240 | .FBIG => unreachable, |
| 2241 | .INVAL => unreachable, |
| 2242 | .OVERFLOW => unreachable, |
| 2243 | .IO => error.InputOutput, |
| 2244 | .ISDIR => error.IsDir, |
| 2245 | .NOMEM => error.SystemResources, |
| 2246 | .NOSPC => error.NoSpaceLeft, |
| 2247 | .PERM => error.PermissionDenied, |
| 2248 | .TXTBSY => error.FileBusy, |
| 2249 | else => std.posix.unexpectedErrno(e), |
| 2250 | }; |
| 2251 | return error.MappedFileIo; |
| 2252 | }, |
| 1407 | 2253 | } |
| 1408 | 2254 | } |
| 1409 | 2255 | return size - remaining_size; |
| 1410 | 2256 | } |
| 1411 | 2257 | |
| 1412 | | fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: Allocator) Allocator.Error!void { |
| 1413 | | try mf.large.ensureUnusedCapacity(gpa, 2); |
| 1414 | | try mf.updates.ensureUnusedCapacity(gpa, 2); |
| 1415 | | } |
| 1416 | | |
| 1417 | 2258 | pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void { |
| 1418 | | mf.ensureTotalCapacityInner(new_capacity) catch |err| switch (err) { |
| 1419 | | error.OutOfMemory, |
| 1420 | | error.Canceled, |
| 1421 | | => |e| return e, |
| 1422 | | |
| 1423 | | else => |e| { |
| 1424 | | mf.io_err = e; |
| 1425 | | return error.MappedFileIo; |
| 1426 | | }, |
| 1427 | | }; |
| 1428 | | } |
| 1429 | | fn ensureTotalCapacityInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { |
| 1430 | 2259 | if (mf.memory_map.memory.len >= new_capacity) return; |
| 1431 | | try mf.ensureTotalCapacityPreciseInner(new_capacity +| new_capacity / growth_factor); |
| 2260 | try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor); |
| 1432 | 2261 | } |
| 1433 | 2262 | |
| 1434 | 2263 | pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void { |
| 1435 | | mf.ensureTotalCapacityPreciseInner(new_capacity) catch |err| switch (err) { |
| 1436 | | error.OutOfMemory, |
| 1437 | | error.Canceled, |
| 1438 | | => |e| return e, |
| 1439 | | |
| 1440 | | else => |e| { |
| 1441 | | mf.io_err = e; |
| 1442 | | return error.MappedFileIo; |
| 1443 | | }, |
| 1444 | | }; |
| 1445 | | } |
| 1446 | | fn ensureTotalCapacityPreciseInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { |
| 1447 | 2264 | if (mf.memory_map.memory.len >= new_capacity) return; |
| 1448 | 2265 | const io = mf.io; |
| 1449 | | const aligned_capacity = mf.flags.block_size.forward(new_capacity); |
| 2266 | const aligned_capacity: usize = @intCast( |
| 2267 | mf.flags.block_size.forward(new_capacity), |
| 2268 | ); |
| 1450 | 2269 | |
| 1451 | 2270 | if (mf.memory_map.memory.len > 0) { |
| 1452 | 2271 | if (mf.memory_map.setLength(io, aligned_capacity)) |_| { |
| 1453 | 2272 | return; |
| 1454 | 2273 | } else |err| switch (err) { |
| 1455 | 2274 | error.OperationUnsupported => {}, |
| 1456 | | else => |e| return e, |
| 2275 | error.OutOfMemory, error.Canceled => |e| return e, |
| 2276 | else => |e| { |
| 2277 | mf.io_err = e; |
| 2278 | return error.MappedFileIo; |
| 2279 | }, |
| 1457 | 2280 | } |
| 1458 | 2281 | |
| 1459 | | mf.memory_map.write(io) catch |err| switch (err) { |
| 1460 | | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking |
| 1461 | | error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing |
| 1462 | | else => |e| return e, |
| 2282 | mf.memory_map.write(io) catch |err| { |
| 2283 | mf.io_err = switch (err) { |
| 2284 | error.Canceled => |e| return e, |
| 2285 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 2286 | error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing |
| 2287 | else => |e| e, |
| 2288 | }; |
| 2289 | return error.MappedFileIo; |
| 1463 | 2290 | }; |
| 1464 | 2291 | unmap(mf); |
| 1465 | 2292 | } |
| 1466 | 2293 | |
| 1467 | 2294 | const file = mf.memory_map.file; |
| 1468 | | mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| switch (err) { |
| 1469 | | error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking |
| 1470 | | error.NotOpenForReading => return error.Unexpected, // we definitely opened the file for writing |
| 1471 | | else => |e| return e, |
| 2295 | mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| { |
| 2296 | mf.io_err = switch (err) { |
| 2297 | error.OutOfMemory, error.Canceled => |e| return e, |
| 2298 | error.WouldBlock => error.Unexpected, // file was not opened as non-blocking |
| 2299 | error.NotOpenForReading => error.Unexpected, // we definitely opened the file for writing |
| 2300 | else => |e| e, |
| 2301 | }; |
| 2302 | return error.MappedFileIo; |
| 1472 | 2303 | }; |
| 1473 | 2304 | } |
| 1474 | 2305 | |
| ... | ... | @@ -1487,7 +2318,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable || error{MappedFileIo})!void { |
| 1487 | 2318 | |
| 1488 | 2319 | error.WouldBlock, // file was not opened as non-blocking |
| 1489 | 2320 | error.NotOpenForWriting, // we definitely opened the file for writing |
| 1490 | | error.ReadOnlyFileSystem, |
| 2321 | error.ReadOnlyFileSystem, // again, we opened the file for writing |
| 1491 | 2322 | => { |
| 1492 | 2323 | mf.io_err = error.Unexpected; |
| 1493 | 2324 | return error.MappedFileIo; |
| ... | ... | @@ -1512,211 +2343,276 @@ fn verify(mf: *MappedFile) void { |
| 1512 | 2343 | assert(root.next == .none); |
| 1513 | 2344 | mf.verifyNode(.root); |
| 1514 | 2345 | } |
| 1515 | | |
| 1516 | 2346 | fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { |
| 1517 | 2347 | const parent = parent_ni.get(mf); |
| 1518 | | const parent_offset, const parent_size = parent.location().resolve(mf); |
| 1519 | | var prev_ni: Node.Index = .none; |
| 2348 | _, const parent_size = parent.location().resolve(mf); |
| 2349 | |
| 2350 | var prev_oni: Node.Index.Optional = .none; |
| 1520 | 2351 | var prev_end: u64 = 0; |
| 1521 | | var ni = parent.first; |
| 1522 | | while (true) { |
| 1523 | | if (ni == .none) { |
| 1524 | | assert(parent.last == prev_ni); |
| 1525 | | return; |
| 1526 | | } |
| 2352 | var prev_pos: Node.Position = .header; |
| 2353 | var oni = parent.first; |
| 2354 | while (oni.unwrap()) |ni| { |
| 1527 | 2355 | const node = ni.get(mf); |
| 1528 | | assert(node.parent == parent_ni); |
| 2356 | assert(node.parent == parent_ni.toOptional()); |
| 2357 | assert(node.prev == prev_oni); |
| 2358 | |
| 1529 | 2359 | const offset, const size = node.location().resolve(mf); |
| 1530 | | assert(node.flags.alignment.check(@intCast(offset))); |
| 1531 | | assert(node.flags.alignment.check(@intCast(size))); |
| 1532 | 2360 | const end = offset + size; |
| 1533 | | assert(end <= parent_offset + parent_size); |
| 2361 | |
| 2362 | assert(node.flags.alignment.check(size)); |
| 1534 | 2363 | assert(offset >= prev_end); |
| 1535 | | assert(node.prev == prev_ni); |
| 2364 | assert(end <= parent_size); |
| 2365 | |
| 2366 | switch (node.flags.position) { |
| 2367 | .header => { |
| 2368 | assert(prev_pos == .header); |
| 2369 | assert(offset == prev_end); |
| 2370 | }, |
| 2371 | .floating => { |
| 2372 | assert(prev_pos != .footer); |
| 2373 | assert(node.flags.alignment.check(offset)); |
| 2374 | }, |
| 2375 | .footer => { |
| 2376 | if (prev_pos == .footer) assert(offset == prev_end); |
| 2377 | }, |
| 2378 | } |
| 2379 | |
| 1536 | 2380 | mf.verifyNode(ni); |
| 1537 | | prev_ni = ni; |
| 2381 | |
| 2382 | prev_oni = .wrap(ni); |
| 1538 | 2383 | prev_end = end; |
| 1539 | | ni = node.next; |
| 2384 | prev_pos = ni.position(mf); |
| 2385 | |
| 2386 | oni = node.next; |
| 2387 | } |
| 2388 | assert(parent.last == prev_oni); |
| 2389 | if (prev_pos == .footer) { |
| 2390 | assert(prev_end == parent_size); |
| 1540 | 2391 | } |
| 1541 | 2392 | } |
| 1542 | 2393 | |
| 1543 | | const testing = std.testing; |
| 1544 | | fn testVerifyContent(mf: *@This(), ni: Node.Index, value: u8, init_len: usize) !void { |
| 1545 | | // Not using std.mem.allEqual, so we can get useful output |
| 1546 | | const slice = ni.slice(mf); |
| 1547 | | var buf: [256]u8 = undefined; |
| 1548 | | @memset(buf[0..init_len], value); |
| 1549 | | @memset(buf[init_len..], 0); |
| 1550 | | try testing.expectEqualSlices(u8, buf[0..slice.len], slice); |
| 2394 | test "fuzz node operations" { |
| 2395 | try std.testing.fuzz({}, fuzzOneNodeOperations, .{}); |
| 1551 | 2396 | } |
| 2397 | fn fuzzOneNodeOperations(_: void, smith: *std.testing.Smith) anyerror!void { |
| 2398 | const gpa = std.testing.allocator; |
| 2399 | const io = std.testing.io; |
| 1552 | 2400 | |
| 1553 | | test { |
| 1554 | | const gpa = testing.allocator; |
| 1555 | | |
| 1556 | | var tmp_dir = testing.tmpDir(.{}); |
| 2401 | var tmp_dir = std.testing.tmpDir(.{}); |
| 1557 | 2402 | defer tmp_dir.cleanup(); |
| 1558 | 2403 | |
| 1559 | | var file = try tmp_dir.dir.createFile(testing.io, "test.mf", .{ .read = true }); |
| 1560 | | defer file.close(testing.io); |
| 2404 | var tmp_file = try tmp_dir.dir.createFile(io, "test.mf", .{ .read = true }); |
| 2405 | defer tmp_file.close(io); |
| 1561 | 2406 | |
| 1562 | | var mf = try init(file, gpa, testing.io); |
| 2407 | var mf: MappedFile = try .init(tmp_file, gpa, io); |
| 1563 | 2408 | defer mf.deinit(gpa); |
| 1564 | 2409 | |
| 1565 | | const a = try mf.addFirstChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); |
| 1566 | | const c = try mf.addLastChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); |
| 1567 | | const b = try mf.addNodeAfter(gpa, a, .{ .fixed = true, .alignment = .@"16" }); |
| 1568 | | const d = try mf.addNodeAfter(gpa, b, .{ .alignment = .@"4" }); |
| 2410 | var nodes: std.array_hash_map.Auto(MappedFile.Node.Index, struct { |
| 2411 | parent: MappedFile.Node.Index.Optional, |
| 2412 | position: MappedFile.Node.Position, |
| 2413 | num_headers: u32, |
| 2414 | num_footers: u32, |
| 2415 | /// For leaf nodes, this value is whether we have initialized the contents of the node or |
| 2416 | /// not. For non-leaf nodes, this value is unspecified and should be ignored. |
| 2417 | initialized: bool, |
| 2418 | }) = .empty; |
| 2419 | defer nodes.deinit(gpa); |
| 2420 | |
| 2421 | // When initializing a leaf node, we will place its 4-byte node index at the start of its range, |
| 2422 | // and the bitwise NOT of its node index at the end of its range (both little-endian). This is |
| 2423 | // just a simple way to put distinct values we can validate at all node boundaries. |
| 2424 | |
| 2425 | try nodes.putNoClobber(gpa, .root, .{ |
| 2426 | .parent = .none, |
| 2427 | .position = .floating, |
| 2428 | .num_headers = 0, |
| 2429 | .num_footers = 0, |
| 2430 | .initialized = false, |
| 2431 | }); |
| 2432 | |
| 2433 | // Allow a range of alignments, with most nodes having a small alignment of 1--32 bytes (most |
| 2434 | // commonly 1 byte), but with a small chance for some large alignments too. |
| 2435 | const alignment_weights: []const std.testing.Smith.Weight = comptime &.{ |
| 2436 | .value(Alignment, .@"1", 20), |
| 2437 | .value(Alignment, .@"2", 5), |
| 2438 | .value(Alignment, .@"4", 5), |
| 2439 | .value(Alignment, .@"8", 5), |
| 2440 | .value(Alignment, .@"16", 5), |
| 2441 | .value(Alignment, .@"32", 5), |
| 2442 | .value(Alignment, .fromByteUnits(0x200), 1), |
| 2443 | .value(Alignment, .fromByteUnits(0x400), 1), |
| 2444 | .value(Alignment, .fromByteUnits(0x800), 1), |
| 2445 | .value(Alignment, .fromByteUnits(0x1000), 1), |
| 2446 | .value(Alignment, .fromByteUnits(0x2000), 1), |
| 2447 | .value(Alignment, .fromByteUnits(0x4000), 1), |
| 2448 | .value(Alignment, .fromByteUnits(0x8000), 1), |
| 2449 | }; |
| 1569 | 2450 | |
| 1570 | | const a_init_size = 8; |
| 1571 | | const b_init_size = 16; |
| 1572 | | const c_init_size = 24; |
| 1573 | | const d_init_size = 28; |
| 2451 | const min_nonzero_size = 2 * @sizeOf(MappedFile.Node.Index); |
| 2452 | const max_size = 0x10_000; |
| 2453 | const initial_size_weights: []const std.testing.Smith.Weight = comptime &.{ |
| 2454 | // initially, make nodes just as likely to be empty as non-empty |
| 2455 | .value(u64, 0, max_size - min_nonzero_size + 1), |
| 2456 | .rangeAtMost(u64, min_nonzero_size, max_size, 1), |
| 2457 | }; |
| 1574 | 2458 | |
| 1575 | | // Resize without content |
| 1576 | | { |
| 1577 | | // Verify size is aligned forward |
| 1578 | | try d.resize(&mf, gpa, d_init_size - 1); |
| 1579 | | try a.resize(&mf, gpa, a_init_size - 2); |
| 1580 | | try c.resize(&mf, gpa, c_init_size); |
| 1581 | | try b.resize(&mf, gpa, b_init_size); |
| 1582 | | mf.verify(); |
| 1583 | | |
| 1584 | | const a_loc, const a_size = a.location(&mf).resolve(&mf); |
| 1585 | | const b_loc, const b_size = b.location(&mf).resolve(&mf); |
| 1586 | | const c_loc, const c_size = c.location(&mf).resolve(&mf); |
| 1587 | | _, const d_size = d.location(&mf).resolve(&mf); |
| 1588 | | try testing.expect(a_size >= a_init_size); |
| 1589 | | try testing.expect(b_size >= b_init_size); |
| 1590 | | try testing.expect(c_size >= c_init_size); |
| 1591 | | try testing.expect(d_size >= d_init_size); |
| 1592 | | try testing.expect(b_loc >= a_loc + a_size); |
| 1593 | | try testing.expect(c_loc >= b_loc + b_size); |
| 1594 | | } |
| 2459 | while (!smith.eos()) switch (smith.value(enum { add, resize, realign })) { |
| 2460 | .add => { |
| 2461 | const parent_ni = nodes.keys()[smith.index(nodes.count())]; |
| 1595 | 2462 | |
| 1596 | | const a_exp_size = 24; |
| 1597 | | const b_exp_size = 28; |
| 1598 | | const c_exp_size = 48; |
| 1599 | | const d_exp_size = 32; |
| 2463 | const alignment = smith.valueWeighted(Alignment, alignment_weights); |
| 2464 | const size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); |
| 1600 | 2465 | |
| 1601 | | // Resize with content |
| 1602 | | { |
| 1603 | | @memset(a.slice(&mf)[0..a_init_size], 0xaa); |
| 1604 | | @memset(b.slice(&mf)[0..b_init_size], 0xbb); |
| 1605 | | @memset(c.slice(&mf)[0..c_init_size], 0xcc); |
| 1606 | | @memset(d.slice(&mf)[0..d_init_size], 0xdd); |
| 1607 | | |
| 1608 | | try a.resize(&mf, gpa, a_exp_size); |
| 1609 | | try b.resize(&mf, gpa, b_exp_size); |
| 1610 | | try c.resize(&mf, gpa, c_exp_size); |
| 1611 | | try d.resize(&mf, gpa, d_exp_size); |
| 1612 | | mf.verify(); |
| 1613 | | |
| 1614 | | const a_loc, const a_size = a.location(&mf).resolve(&mf); |
| 1615 | | const b_loc, const b_size = b.location(&mf).resolve(&mf); |
| 1616 | | const c_loc, const c_size = c.location(&mf).resolve(&mf); |
| 1617 | | _, const d_size = d.location(&mf).resolve(&mf); |
| 1618 | | try testing.expect(a_size >= a_exp_size); |
| 1619 | | try testing.expect(b_size >= b_exp_size); |
| 1620 | | try testing.expect(c_size >= c_exp_size); |
| 1621 | | try testing.expect(d_size >= d_exp_size); |
| 1622 | | try testing.expect(b_loc >= a_loc + a_size); |
| 1623 | | try testing.expect(c_loc >= b_loc + b_size); |
| 1624 | | |
| 1625 | | try testVerifyContent(&mf, a, 0xaa, a_init_size); |
| 1626 | | try testVerifyContent(&mf, b, 0xbb, b_init_size); |
| 1627 | | try testVerifyContent(&mf, c, 0xcc, c_init_size); |
| 1628 | | try testVerifyContent(&mf, d, 0xdd, d_init_size); |
| 1629 | | } |
| 2466 | const position = smith.valueWeighted(Node.Position, comptime &.{ |
| 2467 | // make floating nodes more common than header and footer nodes |
| 2468 | .value(Node.Position, .header, 1), |
| 2469 | .value(Node.Position, .footer, 1), |
| 2470 | .value(Node.Position, .floating, 4), |
| 2471 | }); |
| 2472 | const new_ni: Node.Index = switch (position) { |
| 2473 | .header => new_ni: { |
| 2474 | const parent_info = nodes.getPtr(parent_ni).?; |
| 2475 | const prev_oni: Node.Index.Optional = prev_oni: { |
| 2476 | const n = smith.valueRangeAtMost(u32, 0, parent_info.num_headers); |
| 2477 | if (n == 0) break :prev_oni .none; |
| 2478 | var cur_ni = parent_ni.first(&mf).unwrap().?; |
| 2479 | for (1..n) |_| cur_ni = cur_ni.next(&mf).unwrap().?; |
| 2480 | break :prev_oni .wrap(cur_ni); |
| 2481 | }; |
| 2482 | const new_ni = try parent_ni.addHeaderChildAfter(&mf, gpa, prev_oni, .{ |
| 2483 | .size = size, |
| 2484 | .alignment = alignment, |
| 2485 | }); |
| 2486 | parent_info.num_headers += 1; |
| 2487 | break :new_ni new_ni; |
| 2488 | }, |
| 1630 | 2489 | |
| 1631 | | const child_init: []const struct { Alignment, usize } = &.{ |
| 1632 | | .{ .@"16", 16 }, |
| 1633 | | .{ .@"1", 1 }, |
| 1634 | | .{ .@"1", 19 }, |
| 1635 | | .{ .@"1", 3 }, |
| 1636 | | .{ .@"8", 30 }, |
| 1637 | | .{ .@"2", 5 }, |
| 1638 | | .{ .@"1", 60 }, |
| 1639 | | .{ .@"2", 2 }, |
| 1640 | | .{ .@"16", 32 }, |
| 1641 | | }; |
| 2490 | .floating => try parent_ni.addFloatingChild(&mf, gpa, .{ |
| 2491 | .size = size, |
| 2492 | .alignment = alignment, |
| 2493 | }), |
| 2494 | |
| 2495 | .footer => new_ni: { |
| 2496 | const parent_info = nodes.getPtr(parent_ni).?; |
| 2497 | const next_oni: Node.Index.Optional = next_oni: { |
| 2498 | const n = smith.valueRangeAtMost(u32, 0, parent_info.num_footers); |
| 2499 | if (n == 0) break :next_oni .none; |
| 2500 | var cur_ni = parent_ni.last(&mf).unwrap().?; |
| 2501 | for (1..n) |_| cur_ni = cur_ni.prev(&mf).unwrap().?; |
| 2502 | break :next_oni .wrap(cur_ni); |
| 2503 | }; |
| 2504 | const new_ni = try parent_ni.addFooterChildBefore(&mf, gpa, next_oni, .{ |
| 2505 | .size = size, |
| 2506 | .alignment = alignment, |
| 2507 | }); |
| 2508 | parent_info.num_footers += 1; |
| 2509 | break :new_ni new_ni; |
| 2510 | }, |
| 2511 | }; |
| 1642 | 2512 | |
| 1643 | | var children: [child_init.len]Node.Index = undefined; |
| 2513 | const initialize = size > 0 and smith.value(bool); |
| 2514 | if (initialize) { |
| 2515 | const slice = new_ni.slice(&mf); |
| 2516 | std.mem.writeInt(u32, slice[0..4], @backingInt(new_ni), .little); |
| 2517 | std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(new_ni), .little); |
| 2518 | } |
| 1644 | 2519 | |
| 1645 | | // Differently-aligned fixed sibling nodes |
| 1646 | | { |
| 1647 | | for (children[0 .. children.len - 1], child_init[0 .. children.len - 1], 0..) |*ni, opts, i| { |
| 1648 | | ni.* = try mf.addLastChildNode(gpa, b, .{ |
| 1649 | | .alignment = opts.@"0", |
| 1650 | | .size = opts.@"1", |
| 1651 | | .fixed = true, |
| 2520 | try nodes.putNoClobber(gpa, new_ni, .{ |
| 2521 | .parent = .wrap(parent_ni), |
| 2522 | .position = position, |
| 2523 | .num_headers = 0, |
| 2524 | .num_footers = 0, |
| 2525 | .initialized = initialize, |
| 1652 | 2526 | }); |
| 2527 | }, |
| 1653 | 2528 | |
| 1654 | | @memset(ni.slice(&mf)[0..opts.@"1"], @intCast(i + 1)); |
| 1655 | | } |
| 1656 | | // Shift differently-aligned nodes by inserting a node |
| 1657 | | children[children.len - 1] = try mf.addNodeAfter(gpa, children[3], .{ |
| 1658 | | .alignment = child_init[children.len - 1].@"0", |
| 1659 | | .size = child_init[children.len - 1].@"1", |
| 1660 | | .fixed = true, |
| 1661 | | }); |
| 1662 | | @memset(children[children.len - 1].slice(&mf), @intCast(children.len)); |
| 1663 | | |
| 1664 | | mf.verify(); |
| 1665 | | for (children, child_init, 0..) |ni, opts, i| { |
| 1666 | | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); |
| 1667 | | } |
| 1668 | | } |
| 2529 | .resize => { |
| 2530 | const ni = nodes.keys()[smith.index(nodes.count())]; |
| 2531 | const node_info = nodes.getPtr(ni).?; |
| 1669 | 2532 | |
| 1670 | | // Shifting child nodes forward due via resize of parent.prev |
| 1671 | | { |
| 1672 | | try testing.expect(a.location(&mf).resolve(&mf)[1] < 64); |
| 1673 | | try a.resize(&mf, gpa, 64); |
| 2533 | const alignment = ni.alignment(&mf); |
| 1674 | 2534 | |
| 1675 | | try testVerifyContent(&mf, a, 0xaa, a_init_size); |
| 1676 | | try testVerifyContent(&mf, c, 0xcc, c_init_size); |
| 1677 | | try testVerifyContent(&mf, d, 0xdd, d_init_size); |
| 1678 | | for (children, child_init, 0..) |ni, opts, i| { |
| 1679 | | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); |
| 1680 | | } |
| 1681 | | } |
| 2535 | if (ni.first(&mf) == .none and smith.value(bool)) { |
| 2536 | // Since this is a leaf node, we can use `resizeLeaf`. |
| 2537 | const new_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); |
| 2538 | try ni.resizeLeaf(&mf, gpa, new_size); |
| 2539 | if (new_size == 0) { |
| 2540 | node_info.initialized = false; |
| 2541 | } |
| 2542 | } else { |
| 2543 | const min_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); |
| 2544 | try ni.ensureMinimumSize(&mf, gpa, min_size); |
| 2545 | } |
| 1682 | 2546 | |
| 1683 | | // Re-align last node into trailing free space within parent |
| 1684 | | { |
| 1685 | | try b.resize(&mf, gpa, b.location(&mf).resolve(&mf)[1] + 64); |
| 2547 | if (ni.first(&mf) == .none) { |
| 2548 | // This is a leaf node, so it can contain data. |
| 2549 | if (node_info.initialized) { |
| 2550 | // It's already initialized, so we'll write the expected footer at the new end. |
| 2551 | const slice = ni.slice(&mf); |
| 2552 | std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); |
| 2553 | } else if (ni.location(&mf).resolve(&mf)[1] > 0) { |
| 2554 | // It was uninitialized, but it has a non-zero size, so maybe we'd like to |
| 2555 | // initialize it now? |
| 2556 | if (smith.value(bool)) { |
| 2557 | node_info.initialized = true; |
| 2558 | const slice = ni.slice(&mf); |
| 2559 | std.mem.writeInt(u32, slice[0..4], @backingInt(ni), .little); |
| 2560 | std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); |
| 2561 | } |
| 2562 | } |
| 2563 | } |
| 2564 | }, |
| 2565 | .realign => { |
| 2566 | const ni = nodes.keys()[smith.index(nodes.count())]; |
| 2567 | const new_alignment = smith.valueWeighted(Alignment, alignment_weights); |
| 2568 | if (new_alignment.compare(.gt, ni.alignment(&mf))) { |
| 2569 | _, const old_size = ni.location(&mf).resolve(&mf); |
| 2570 | try ni.realign(&mf, gpa, new_alignment); |
| 2571 | if (ni.first(&mf) == .none and nodes.get(ni).?.initialized) { |
| 2572 | const slice = ni.slice(&mf); |
| 2573 | @memmove(slice[slice.len - 4 ..][0..4], slice[old_size - 4 ..][0..4]); |
| 2574 | } |
| 2575 | } |
| 2576 | }, |
| 2577 | }; |
| 1686 | 2578 | |
| 1687 | | const last = children[children.len - 2]; |
| 1688 | | try last.realign(&mf, gpa, .@"4", true); |
| 1689 | | mf.verify(); |
| 2579 | mf.verify(); |
| 1690 | 2580 | |
| 1691 | | for (children, child_init, 0..) |ni, opts, i| |
| 1692 | | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); |
| 1693 | | try testVerifyContent(&mf, c, 0xcc, c_init_size); |
| 1694 | | } |
| 2581 | for (nodes.keys(), nodes.values()) |ni, expected| { |
| 2582 | try std.testing.expectEqual(expected.parent, ni.parent(&mf)); |
| 2583 | if (ni != .root) { |
| 2584 | try std.testing.expectEqual(expected.position, ni.position(&mf)); |
| 2585 | } |
| 1695 | 2586 | |
| 1696 | | // Re-align, shifting sibling nodes |
| 1697 | | { |
| 1698 | | try children[1].realign(&mf, gpa, .@"8", true); |
| 1699 | | mf.verify(); |
| 2587 | { |
| 2588 | var num_headers: u32 = 0; |
| 2589 | var header_oni = ni.lastHeader(&mf); |
| 2590 | while (header_oni.unwrap()) |header_ni| { |
| 2591 | num_headers += 1; |
| 2592 | header_oni = header_ni.prev(&mf); |
| 2593 | } |
| 2594 | try std.testing.expectEqual(expected.num_headers, num_headers); |
| 2595 | } |
| 1700 | 2596 | |
| 1701 | | for (children, child_init, 0..) |ni, opts, i| |
| 1702 | | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); |
| 1703 | | try testVerifyContent(&mf, c, 0xcc, c_init_size); |
| 1704 | | } |
| 2597 | { |
| 2598 | var num_footers: u32 = 0; |
| 2599 | var footer_oni = ni.firstFooter(&mf); |
| 2600 | while (footer_oni.unwrap()) |footer_ni| { |
| 2601 | num_footers += 1; |
| 2602 | footer_oni = footer_ni.next(&mf); |
| 2603 | } |
| 2604 | try std.testing.expectEqual(expected.num_footers, num_footers); |
| 2605 | } |
| 1705 | 2606 | |
| 1706 | | // Shrink and shift start of trailing node into free space |
| 1707 | | { |
| 1708 | | try mf.shrinkNode(gpa, a, 16, true); |
| 1709 | | mf.verify(); |
| 1710 | | |
| 1711 | | const a_loc, const a_size = a.location(&mf).resolve(&mf); |
| 1712 | | const b_loc, _ = b.location(&mf).resolve(&mf); |
| 1713 | | try testing.expectEqual(b_loc, a_loc + a_size); |
| 1714 | | |
| 1715 | | try testVerifyContent(&mf, a, 0xaa, a_init_size); |
| 1716 | | try testVerifyContent(&mf, c, 0xcc, c_init_size); |
| 1717 | | try testVerifyContent(&mf, d, 0xdd, d_init_size); |
| 1718 | | for (children, child_init, 0..) |ni, opts, i| { |
| 1719 | | try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); |
| 2607 | if (ni.first(&mf) == .none and expected.initialized) { |
| 2608 | const slice = ni.sliceConst(&mf); |
| 2609 | if (slice.len > 0) { |
| 2610 | try std.testing.expect(slice.len >= min_nonzero_size); |
| 2611 | const header = std.mem.readInt(u32, slice[0..4], .little); |
| 2612 | const footer = std.mem.readInt(u32, slice[slice.len - 4 ..][0..4], .little); |
| 2613 | try std.testing.expectEqual(@backingInt(ni), header); |
| 2614 | try std.testing.expectEqual(~@backingInt(ni), footer); |
| 2615 | } |
| 1720 | 2616 | } |
| 1721 | 2617 | } |
| 1722 | 2618 | } |
Matthew Lugg <mlugg@mlugg.co.uk> 2026-08-23 11:21:31+01:00
Commit is signed but in an unrecognized format.