authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-03-08 21:59:07+00:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-03-08 21:59:07+00:00
log9cf28d1e9bc940b13cd47efb778fd42a4b3b88de
treea335d9f3435c157fd65b01b4b29311ab188deadd
parentf3227598ebe9ac7e330fea0259d4290ee31e96b9
parent265f42d472fa110c1e211e5f82fdf540ee198aa8
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #19214 from mlugg/fuck-usingnamespace

std: fuck usingnamespace

10 files changed, 5565 insertions(+), 5661 deletions(-)

CMakeLists.txt+2-1
...@@ -291,7 +291,8 @@ set(ZIG_STAGE2_SOURCES...@@ -291,7 +291,8 @@ set(ZIG_STAGE2_SOURCES
291 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/errno/generic.zig"291 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/errno/generic.zig"
292 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/x86_64.zig"292 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/x86_64.zig"
293 "${CMAKE_SOURCE_DIR}/lib/std/os/linux.zig"293 "${CMAKE_SOURCE_DIR}/lib/std/os/linux.zig"
294 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/io_uring.zig"294 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/IoUring.zig"
295 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/io_uring_sqe.zig"
295 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/x86_64.zig"296 "${CMAKE_SOURCE_DIR}/lib/std/os/linux/x86_64.zig"
296 "${CMAKE_SOURCE_DIR}/lib/std/os/windows.zig"297 "${CMAKE_SOURCE_DIR}/lib/std/os/windows.zig"
297 "${CMAKE_SOURCE_DIR}/lib/std/os/windows/ntstatus.zig"298 "${CMAKE_SOURCE_DIR}/lib/std/os/windows/ntstatus.zig"
lib/std/c.zig+28-35
...@@ -1494,38 +1494,33 @@ pub const speed_t = switch (native_os) {...@@ -1494,38 +1494,33 @@ pub const speed_t = switch (native_os) {
1494pub const whence_t = if (native_os == .wasi) std.os.wasi.whence_t else c_int;1494pub const whence_t = if (native_os == .wasi) std.os.wasi.whence_t else c_int;
14951495
1496// Unix-like systems1496// Unix-like systems
1497pub usingnamespace switch (native_os) {1497pub const DIR = opaque {};
1498 .netbsd, .windows => struct {},1498pub extern "c" fn opendir(pathname: [*:0]const u8) ?*DIR;
1499 else => struct {1499pub extern "c" fn fdopendir(fd: c_int) ?*DIR;
1500 pub const DIR = opaque {};1500pub extern "c" fn rewinddir(dp: *DIR) void;
1501 pub extern "c" fn opendir(pathname: [*:0]const u8) ?*DIR;1501pub extern "c" fn closedir(dp: *DIR) c_int;
1502 pub extern "c" fn fdopendir(fd: c_int) ?*DIR;1502pub extern "c" fn telldir(dp: *DIR) c_long;
1503 pub extern "c" fn rewinddir(dp: *DIR) void;1503pub extern "c" fn seekdir(dp: *DIR, loc: c_long) void;
1504 pub extern "c" fn closedir(dp: *DIR) c_int;
1505 pub extern "c" fn telldir(dp: *DIR) c_long;
1506 pub extern "c" fn seekdir(dp: *DIR, loc: c_long) void;
15071504
1508 pub extern "c" fn clock_gettime(clk_id: c_int, tp: *c.timespec) c_int;1505pub extern "c" fn clock_gettime(clk_id: c_int, tp: *c.timespec) c_int;
1509 pub extern "c" fn clock_getres(clk_id: c_int, tp: *c.timespec) c_int;1506pub extern "c" fn clock_getres(clk_id: c_int, tp: *c.timespec) c_int;
1510 pub extern "c" fn gettimeofday(noalias tv: ?*c.timeval, noalias tz: ?*c.timezone) c_int;1507pub extern "c" fn gettimeofday(noalias tv: ?*c.timeval, noalias tz: ?*c.timezone) c_int;
1511 pub extern "c" fn nanosleep(rqtp: *const c.timespec, rmtp: ?*c.timespec) c_int;1508pub extern "c" fn nanosleep(rqtp: *const c.timespec, rmtp: ?*c.timespec) c_int;
15121509
1513 pub extern "c" fn getrusage(who: c_int, usage: *c.rusage) c_int;1510pub extern "c" fn getrusage(who: c_int, usage: *c.rusage) c_int;
15141511
1515 pub extern "c" fn sched_yield() c_int;1512pub extern "c" fn sched_yield() c_int;
15161513
1517 pub extern "c" fn sigaction(sig: c_int, noalias act: ?*const c.Sigaction, noalias oact: ?*c.Sigaction) c_int;1514pub extern "c" fn sigaction(sig: c_int, noalias act: ?*const c.Sigaction, noalias oact: ?*c.Sigaction) c_int;
1518 pub extern "c" fn sigprocmask(how: c_int, noalias set: ?*const c.sigset_t, noalias oset: ?*c.sigset_t) c_int;1515pub extern "c" fn sigprocmask(how: c_int, noalias set: ?*const c.sigset_t, noalias oset: ?*c.sigset_t) c_int;
1519 pub extern "c" fn sigfillset(set: ?*c.sigset_t) void;1516pub extern "c" fn sigfillset(set: ?*c.sigset_t) void;
1520 pub extern "c" fn sigwait(set: ?*c.sigset_t, sig: ?*c_int) c_int;1517pub extern "c" fn sigwait(set: ?*c.sigset_t, sig: ?*c_int) c_int;
15211518
1522 pub extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int;1519pub extern "c" fn socket(domain: c_uint, sock_type: c_uint, protocol: c_uint) c_int;
15231520
1524 pub extern "c" fn alarm(seconds: c_uint) c_uint;1521pub extern "c" fn alarm(seconds: c_uint) c_uint;
15251522
1526 pub extern "c" fn msync(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int;1523pub extern "c" fn msync(addr: *align(page_size) const anyopaque, len: usize, flags: c_int) c_int;
1527 },
1528};
15291524
1530pub const fstat = switch (native_os) {1525pub const fstat = switch (native_os) {
1531 .macos => switch (native_arch) {1526 .macos => switch (native_arch) {
...@@ -1863,16 +1858,14 @@ pub extern "c" fn setlogmask(maskpri: c_int) c_int;...@@ -1863,16 +1858,14 @@ pub extern "c" fn setlogmask(maskpri: c_int) c_int;
18631858
1864pub extern "c" fn if_nametoindex([*:0]const u8) c_int;1859pub extern "c" fn if_nametoindex([*:0]const u8) c_int;
18651860
1866pub usingnamespace if (builtin.target.isAndroid()) struct {1861pub const getcontext = if (builtin.target.isAndroid())
1867 // android bionic libc does not implement getcontext,1862 @compileError("android bionic libc does not implement getcontext")
1868 // and std.os.linux.getcontext also cannot be built for1863else if (native_os == .linux and builtin.target.isMusl())
1869 // bionic libc currently.1864 std.os.linux.getcontext
1870} else if (native_os == .linux and builtin.target.isMusl()) struct {1865else
1871 // musl does not implement getcontext1866 struct {
1872 pub const getcontext = std.os.linux.getcontext;1867 extern fn getcontext(ucp: *std.os.ucontext_t) c_int;
1873} else struct {1868 }.getcontext;
1874 pub extern "c" fn getcontext(ucp: *std.os.ucontext_t) c_int;
1875};
18761869
1877pub const max_align_t = if (native_abi == .msvc)1870pub const max_align_t = if (native_abi == .msvc)
1878 f641871 f64
lib/std/c/openbsd.zig-5
...@@ -894,11 +894,6 @@ comptime {...@@ -894,11 +894,6 @@ comptime {
894 std.debug.assert(@sizeOf(siginfo_t) == 136);894 std.debug.assert(@sizeOf(siginfo_t) == 136);
895}895}
896896
897pub usingnamespace switch (builtin.cpu.arch) {
898 .x86_64 => struct {},
899 else => struct {},
900};
901
902pub const ucontext_t = switch (builtin.cpu.arch) {897pub const ucontext_t = switch (builtin.cpu.arch) {
903 .x86_64 => extern struct {898 .x86_64 => extern struct {
904 sc_rdi: c_long,899 sc_rdi: c_long,
lib/std/enums.zig+715-819
...@@ -241,957 +241,794 @@ test nameCast {...@@ -241,957 +241,794 @@ test nameCast {
241/// to dense indices. This type does no dynamic allocation and241/// to dense indices. This type does no dynamic allocation and
242/// can be copied by value.242/// can be copied by value.
243pub fn EnumSet(comptime E: type) type {243pub fn EnumSet(comptime E: type) type {
244 const mixin = struct {244 return struct {
245 fn EnumSetExt(comptime Self: type) type {245 const Self = @This();
246 const Indexer = Self.Indexer;
247 return struct {
248 /// Initializes the set using a struct of bools
249 pub fn init(init_values: EnumFieldStruct(E, bool, false)) Self {
250 var result = Self{};
251 comptime var i: usize = 0;
252 inline while (i < Self.len) : (i += 1) {
253 const key = comptime Indexer.keyForIndex(i);
254 const tag = comptime @tagName(key);
255 if (@field(init_values, tag)) {
256 result.bits.set(i);
257 }
258 }
259 return result;
260 }
261 };
262 }
263 };
264 return IndexedSet(EnumIndexer(E), mixin.EnumSetExt);
265}
266246
267/// A map keyed by an enum, backed by a bitfield and a dense array.247 /// The indexing rules for converting between keys and indices.
268/// If the enum is not dense, a mapping will be constructed from248 pub const Indexer = EnumIndexer(E);
269/// enum values to dense indices. This type does no dynamic249 /// The element type for this set.
270/// allocation and can be copied by value.250 pub const Key = Indexer.Key;
271pub fn EnumMap(comptime E: type, comptime V: type) type {
272 const mixin = struct {
273 fn EnumMapExt(comptime Self: type) type {
274 const Indexer = Self.Indexer;
275 return struct {
276 /// Initializes the map using a sparse struct of optionals
277 pub fn init(init_values: EnumFieldStruct(E, ?V, @as(?V, null))) Self {
278 var result = Self{};
279 comptime var i: usize = 0;
280 inline while (i < Self.len) : (i += 1) {
281 const key = comptime Indexer.keyForIndex(i);
282 const tag = comptime @tagName(key);
283 if (@field(init_values, tag)) |*v| {
284 result.bits.set(i);
285 result.values[i] = v.*;
286 }
287 }
288 return result;
289 }
290 /// Initializes a full mapping with all keys set to value.
291 /// Consider using EnumArray instead if the map will remain full.
292 pub fn initFull(value: V) Self {
293 var result = Self{
294 .bits = Self.BitSet.initFull(),
295 .values = undefined,
296 };
297 @memset(&result.values, value);
298 return result;
299 }
300 /// Initializes a full mapping with supplied values.
301 /// Consider using EnumArray instead if the map will remain full.
302 pub fn initFullWith(init_values: EnumFieldStruct(E, V, @as(?V, null))) Self {
303 return initFullWithDefault(@as(?V, null), init_values);
304 }
305 /// Initializes a full mapping with a provided default.
306 /// Consider using EnumArray instead if the map will remain full.
307 pub fn initFullWithDefault(comptime default: ?V, init_values: EnumFieldStruct(E, V, default)) Self {
308 var result = Self{
309 .bits = Self.BitSet.initFull(),
310 .values = undefined,
311 };
312 comptime var i: usize = 0;
313 inline while (i < Self.len) : (i += 1) {
314 const key = comptime Indexer.keyForIndex(i);
315 const tag = comptime @tagName(key);
316 result.values[i] = @field(init_values, tag);
317 }
318 return result;
319 }
320 };
321 }
322 };
323 return IndexedMap(EnumIndexer(E), V, mixin.EnumMapExt);
324}
325251
326/// A multiset of enum elements up to a count of usize. Backed252 const BitSet = std.StaticBitSet(Indexer.count);
327/// by an EnumArray. This type does no dynamic allocation and can
328/// be copied by value.
329pub fn EnumMultiset(comptime E: type) type {
330 return BoundedEnumMultiset(E, usize);
331}
332253
333/// A multiset of enum elements up to CountSize. Backed by an254 /// The maximum number of items in this set.
334/// EnumArray. This type does no dynamic allocation and can be255 pub const len = Indexer.count;
335/// copied by value.
336pub fn BoundedEnumMultiset(comptime E: type, comptime CountSize: type) type {
337 return struct {
338 const Self = @This();
339256
340 counts: EnumArray(E, CountSize),257 bits: BitSet = BitSet.initEmpty(),
341258
342 /// Initializes the multiset using a struct of counts.259 /// Initializes the set using a struct of bools
343 pub fn init(init_counts: EnumFieldStruct(E, CountSize, 0)) Self {260 pub fn init(init_values: EnumFieldStruct(E, bool, false)) Self {
344 var self = initWithCount(0);261 var result: Self = .{};
345 inline for (@typeInfo(E).Enum.fields) |field| {262 inline for (0..Self.len) |i| {
346 const c = @field(init_counts, field.name);263 const key = comptime Indexer.keyForIndex(i);
347 const key = @as(E, @enumFromInt(field.value));264 const tag = @tagName(key);
348 self.counts.set(key, c);265 if (@field(init_values, tag)) {
266 result.bits.set(i);
267 }
349 }268 }
350 return self;269 return result;
351 }270 }
352271
353 /// Initializes the multiset with a count of zero.272 /// Returns a set containing no keys.
354 pub fn initEmpty() Self {273 pub fn initEmpty() Self {
355 return initWithCount(0);274 return .{ .bits = BitSet.initEmpty() };
356 }275 }
357276
358 /// Initializes the multiset with all keys at the277 /// Returns a set containing all possible keys.
359 /// same count.278 pub fn initFull() Self {
360 pub fn initWithCount(comptime c: CountSize) Self {279 return .{ .bits = BitSet.initFull() };
361 return .{
362 .counts = EnumArray(E, CountSize).initDefault(c, .{}),
363 };
364 }280 }
365281
366 /// Returns the total number of key counts in the multiset.282 /// Returns a set containing multiple keys.
367 pub fn count(self: Self) usize {283 pub fn initMany(keys: []const Key) Self {
368 var sum: usize = 0;284 var set = initEmpty();
369 for (self.counts.values) |c| {285 for (keys) |key| set.insert(key);
370 sum += c;286 return set;
371 }
372 return sum;
373 }287 }
374288
375 /// Checks if at least one key in multiset.289 /// Returns a set containing a single key.
376 pub fn contains(self: Self, key: E) bool {290 pub fn initOne(key: Key) Self {
377 return self.counts.get(key) > 0;291 return initMany(&[_]Key{key});
378 }292 }
379293
380 /// Removes all instance of a key from multiset. Same as294 /// Returns the number of keys in the set.
381 /// setCount(key, 0).295 pub fn count(self: Self) usize {
382 pub fn removeAll(self: *Self, key: E) void {296 return self.bits.count();
383 return self.counts.set(key, 0);
384 }297 }
385298
386 /// Increases the key count by given amount. Caller asserts299 /// Checks if a key is in the set.
387 /// operation will not overflow.300 pub fn contains(self: Self, key: Key) bool {
388 pub fn addAssertSafe(self: *Self, key: E, c: CountSize) void {301 return self.bits.isSet(Indexer.indexOf(key));
389 self.counts.getPtr(key).* += c;
390 }302 }
391303
392 /// Increases the key count by given amount.304 /// Puts a key in the set.
393 pub fn add(self: *Self, key: E, c: CountSize) error{Overflow}!void {305 pub fn insert(self: *Self, key: Key) void {
394 self.counts.set(key, try std.math.add(CountSize, self.counts.get(key), c));306 self.bits.set(Indexer.indexOf(key));
395 }307 }
396308
397 /// Decreases the key count by given amount. If amount is309 /// Removes a key from the set.
398 /// greater than the number of keys in multset, then key count310 pub fn remove(self: *Self, key: Key) void {
399 /// will be set to zero.311 self.bits.unset(Indexer.indexOf(key));
400 pub fn remove(self: *Self, key: E, c: CountSize) void {
401 self.counts.getPtr(key).* -= @min(self.getCount(key), c);
402 }312 }
403313
404 /// Returns the count for a key.314 /// Changes the presence of a key in the set to match the passed bool.
405 pub fn getCount(self: Self, key: E) CountSize {315 pub fn setPresent(self: *Self, key: Key, present: bool) void {
406 return self.counts.get(key);316 self.bits.setValue(Indexer.indexOf(key), present);
407 }317 }
408318
409 /// Set the count for a key.319 /// Toggles the presence of a key in the set. If the key is in
410 pub fn setCount(self: *Self, key: E, c: CountSize) void {320 /// the set, removes it. Otherwise adds it.
411 self.counts.set(key, c);321 pub fn toggle(self: *Self, key: Key) void {
322 self.bits.toggle(Indexer.indexOf(key));
412 }323 }
413324
414 /// Increases the all key counts by given multiset. Caller325 /// Toggles the presence of all keys in the passed set.
415 /// asserts operation will not overflow any key.326 pub fn toggleSet(self: *Self, other: Self) void {
416 pub fn addSetAssertSafe(self: *Self, other: Self) void {327 self.bits.toggleSet(other.bits);
417 inline for (@typeInfo(E).Enum.fields) |field| {
418 const key = @as(E, @enumFromInt(field.value));
419 self.addAssertSafe(key, other.getCount(key));
420 }
421 }328 }
422329
423 /// Increases the all key counts by given multiset.330 /// Toggles all possible keys in the set.
424 pub fn addSet(self: *Self, other: Self) error{Overflow}!void {331 pub fn toggleAll(self: *Self) void {
425 inline for (@typeInfo(E).Enum.fields) |field| {332 self.bits.toggleAll();
426 const key = @as(E, @enumFromInt(field.value));
427 try self.add(key, other.getCount(key));
428 }
429 }333 }
430334
431 /// Decreases the all key counts by given multiset. If335 /// Adds all keys in the passed set to this set.
432 /// the given multiset has more key counts than this,336 pub fn setUnion(self: *Self, other: Self) void {
433 /// then that key will have a key count of zero.337 self.bits.setUnion(other.bits);
434 pub fn removeSet(self: *Self, other: Self) void {
435 inline for (@typeInfo(E).Enum.fields) |field| {
436 const key = @as(E, @enumFromInt(field.value));
437 self.remove(key, other.getCount(key));
438 }
439 }338 }
440339
441 /// Returns true iff all key counts are the same as340 /// Removes all keys which are not in the passed set.
442 /// given multiset.341 pub fn setIntersection(self: *Self, other: Self) void {
342 self.bits.setIntersection(other.bits);
343 }
344
345 /// Returns true iff both sets have the same keys.
443 pub fn eql(self: Self, other: Self) bool {346 pub fn eql(self: Self, other: Self) bool {
444 inline for (@typeInfo(E).Enum.fields) |field| {347 return self.bits.eql(other.bits);
445 const key = @as(E, @enumFromInt(field.value));
446 if (self.getCount(key) != other.getCount(key)) {
447 return false;
448 }
449 }
450 return true;
451 }348 }
452349
453 /// Returns true iff all key counts less than or350 /// Returns true iff all the keys in this set are
454 /// equal to the given multiset.351 /// in the other set. The other set may have keys
352 /// not found in this set.
455 pub fn subsetOf(self: Self, other: Self) bool {353 pub fn subsetOf(self: Self, other: Self) bool {
456 inline for (@typeInfo(E).Enum.fields) |field| {354 return self.bits.subsetOf(other.bits);
457 const key = @as(E, @enumFromInt(field.value));
458 if (self.getCount(key) > other.getCount(key)) {
459 return false;
460 }
461 }
462 return true;
463 }355 }
464356
465 /// Returns true iff all key counts greater than or357 /// Returns true iff this set contains all the keys
466 /// equal to the given multiset.358 /// in the other set. This set may have keys not
359 /// found in the other set.
467 pub fn supersetOf(self: Self, other: Self) bool {360 pub fn supersetOf(self: Self, other: Self) bool {
468 inline for (@typeInfo(E).Enum.fields) |field| {361 return self.bits.supersetOf(other.bits);
469 const key = @as(E, @enumFromInt(field.value));
470 if (self.getCount(key) < other.getCount(key)) {
471 return false;
472 }
473 }
474 return true;
475 }362 }
476363
477 /// Returns a multiset with the total key count of this364 /// Returns a set with all the keys not in this set.
478 /// multiset and the other multiset. Caller asserts365 pub fn complement(self: Self) Self {
479 /// operation will not overflow any key.366 return .{ .bits = self.bits.complement() };
480 pub fn plusAssertSafe(self: Self, other: Self) Self {
481 var result = self;
482 result.addSetAssertSafe(other);
483 return result;
484 }367 }
485368
486 /// Returns a multiset with the total key count of this369 /// Returns a set with keys that are in either this
487 /// multiset and the other multiset.370 /// set or the other set.
488 pub fn plus(self: Self, other: Self) error{Overflow}!Self {371 pub fn unionWith(self: Self, other: Self) Self {
489 var result = self;372 return .{ .bits = self.bits.unionWith(other.bits) };
490 try result.addSet(other);
491 return result;
492 }373 }
493374
494 /// Returns a multiset with the key count of this375 /// Returns a set with keys that are in both this
495 /// multiset minus the corresponding key count in the376 /// set and the other set.
496 /// other multiset. If the other multiset contains377 pub fn intersectWith(self: Self, other: Self) Self {
497 /// more key count than this set, that key will have378 return .{ .bits = self.bits.intersectWith(other.bits) };
498 /// a count of zero.
499 pub fn minus(self: Self, other: Self) Self {
500 var result = self;
501 result.removeSet(other);
502 return result;
503 }379 }
504380
505 pub const Entry = EnumArray(E, CountSize).Entry;381 /// Returns a set with keys that are in either this
506 pub const Iterator = EnumArray(E, CountSize).Iterator;382 /// set or the other set, but not both.
507383 pub fn xorWith(self: Self, other: Self) Self {
508 /// Returns an iterator over this multiset. Keys with zero384 return .{ .bits = self.bits.xorWith(other.bits) };
509 /// counts are included. Modifications to the set during
510 /// iteration may or may not be observed by the iterator,
511 /// but will not invalidate it.
512 pub fn iterator(self: *Self) Iterator {
513 return self.counts.iterator();
514 }385 }
515 };
516}
517386
518test EnumMultiset {387 /// Returns a set with keys that are in this set
519 const Ball = enum { red, green, blue };388 /// except for keys in the other set.
389 pub fn differenceWith(self: Self, other: Self) Self {
390 return .{ .bits = self.bits.differenceWith(other.bits) };
391 }
520392
521 const empty = EnumMultiset(Ball).initEmpty();393 /// Returns an iterator over this set, which iterates in
522 const r0_g1_b2 = EnumMultiset(Ball).init(.{394 /// index order. Modifications to the set during iteration
523 .red = 0,395 /// may or may not be observed by the iterator, but will
524 .green = 1,396 /// not invalidate it.
525 .blue = 2,397 pub fn iterator(self: *const Self) Iterator {
526 });398 return .{ .inner = self.bits.iterator(.{}) };
527 const ten_of_each = EnumMultiset(Ball).initWithCount(10);399 }
528
529 try testing.expectEqual(empty.count(), 0);
530 try testing.expectEqual(r0_g1_b2.count(), 3);
531 try testing.expectEqual(ten_of_each.count(), 30);
532
533 try testing.expect(!empty.contains(.red));
534 try testing.expect(!empty.contains(.green));
535 try testing.expect(!empty.contains(.blue));
536
537 try testing.expect(!r0_g1_b2.contains(.red));
538 try testing.expect(r0_g1_b2.contains(.green));
539 try testing.expect(r0_g1_b2.contains(.blue));
540
541 try testing.expect(ten_of_each.contains(.red));
542 try testing.expect(ten_of_each.contains(.green));
543 try testing.expect(ten_of_each.contains(.blue));
544
545 {
546 var copy = ten_of_each;
547 copy.removeAll(.red);
548 try testing.expect(!copy.contains(.red));
549
550 // removeAll second time does nothing
551 copy.removeAll(.red);
552 try testing.expect(!copy.contains(.red));
553 }
554400
555 {401 pub const Iterator = struct {
556 var copy = ten_of_each;402 inner: BitSet.Iterator(.{}),
557 copy.addAssertSafe(.red, 6);
558 try testing.expectEqual(copy.getCount(.red), 16);
559 }
560403
561 {404 pub fn next(self: *Iterator) ?Key {
562 var copy = ten_of_each;405 return if (self.inner.next()) |index|
563 try copy.add(.red, 6);406 Indexer.keyForIndex(index)
564 try testing.expectEqual(copy.getCount(.red), 16);407 else
408 null;
409 }
410 };
411 };
412}
565413
566 try testing.expectError(error.Overflow, copy.add(.red, std.math.maxInt(usize)));414/// A map keyed by an enum, backed by a bitfield and a dense array.
567 }415/// If the enum is not dense, a mapping will be constructed from
416/// enum values to dense indices. This type does no dynamic
417/// allocation and can be copied by value.
418pub fn EnumMap(comptime E: type, comptime V: type) type {
419 return struct {
420 const Self = @This();
568421
569 {422 /// The index mapping for this map
570 var copy = ten_of_each;423 pub const Indexer = EnumIndexer(E);
571 copy.remove(.red, 4);424 /// The key type used to index this map
572 try testing.expectEqual(copy.getCount(.red), 6);425 pub const Key = Indexer.Key;
426 /// The value type stored in this map
427 pub const Value = V;
428 /// The number of possible keys in the map
429 pub const len = Indexer.count;
573430
574 // subtracting more it contains does not underflow431 const BitSet = std.StaticBitSet(Indexer.count);
575 copy.remove(.green, 14);
576 try testing.expectEqual(copy.getCount(.green), 0);
577 }
578432
579 try testing.expectEqual(empty.getCount(.green), 0);433 /// Bits determining whether items are in the map
580 try testing.expectEqual(r0_g1_b2.getCount(.green), 1);434 bits: BitSet = BitSet.initEmpty(),
581 try testing.expectEqual(ten_of_each.getCount(.green), 10);435 /// Values of items in the map. If the associated
436 /// bit is zero, the value is undefined.
437 values: [Indexer.count]Value = undefined,
582438
583 {439 /// Initializes the map using a sparse struct of optionals
584 var copy = empty;440 pub fn init(init_values: EnumFieldStruct(E, ?Value, null)) Self {
585 copy.setCount(.red, 6);441 var result: Self = .{};
586 try testing.expectEqual(copy.getCount(.red), 6);442 inline for (0..Self.len) |i| {
587 }443 const key = comptime Indexer.keyForIndex(i);
444 const tag = @tagName(key);
445 if (@field(init_values, tag)) |*v| {
446 result.bits.set(i);
447 result.values[i] = v.*;
448 }
449 }
450 }
588451
589 {452 /// Initializes a full mapping with all keys set to value.
590 var copy = r0_g1_b2;453 /// Consider using EnumArray instead if the map will remain full.
591 copy.addSetAssertSafe(ten_of_each);454 pub fn initFull(value: Value) Self {
592 try testing.expectEqual(copy.getCount(.red), 10);455 var result: Self = .{
593 try testing.expectEqual(copy.getCount(.green), 11);456 .bits = Self.BitSet.initFull(),
594 try testing.expectEqual(copy.getCount(.blue), 12);457 .values = undefined,
595 }458 };
459 @memset(&result.values, value);
460 return result;
461 }
596462
597 {463 /// Initializes a full mapping with supplied values.
598 var copy = r0_g1_b2;464 /// Consider using EnumArray instead if the map will remain full.
599 try copy.addSet(ten_of_each);465 pub fn initFullWith(init_values: EnumFieldStruct(E, Value, null)) Self {
600 try testing.expectEqual(copy.getCount(.red), 10);466 return initFullWithDefault(null, init_values);
601 try testing.expectEqual(copy.getCount(.green), 11);467 }
602 try testing.expectEqual(copy.getCount(.blue), 12);
603468
604 const full = EnumMultiset(Ball).initWithCount(std.math.maxInt(usize));469 /// Initializes a full mapping with a provided default.
605 try testing.expectError(error.Overflow, copy.addSet(full));470 /// Consider using EnumArray instead if the map will remain full.
606 }471 pub fn initFullWithDefault(comptime default: ?Value, init_values: EnumFieldStruct(E, Value, default)) Self {
472 var result: Self = .{
473 .bits = Self.BitSet.initFull(),
474 .values = undefined,
475 };
476 inline for (0..Self.len) |i| {
477 const key = comptime Indexer.keyForIndex(i);
478 const tag = @tagName(key);
479 result.values[i] = @field(init_values, tag);
480 }
481 return result;
482 }
607483
608 {484 /// The number of items in the map.
609 var copy = ten_of_each;485 pub fn count(self: Self) usize {
610 copy.removeSet(r0_g1_b2);486 return self.bits.count();
611 try testing.expectEqual(copy.getCount(.red), 10);487 }
612 try testing.expectEqual(copy.getCount(.green), 9);
613 try testing.expectEqual(copy.getCount(.blue), 8);
614488
615 copy.removeSet(ten_of_each);489 /// Checks if the map contains an item.
616 try testing.expectEqual(copy.getCount(.red), 0);490 pub fn contains(self: Self, key: Key) bool {
617 try testing.expectEqual(copy.getCount(.green), 0);491 return self.bits.isSet(Indexer.indexOf(key));
618 try testing.expectEqual(copy.getCount(.blue), 0);492 }
619 }
620493
621 try testing.expect(empty.eql(empty));494 /// Gets the value associated with a key.
622 try testing.expect(r0_g1_b2.eql(r0_g1_b2));495 /// If the key is not in the map, returns null.
623 try testing.expect(ten_of_each.eql(ten_of_each));496 pub fn get(self: Self, key: Key) ?Value {
624 try testing.expect(!empty.eql(r0_g1_b2));497 const index = Indexer.indexOf(key);
625 try testing.expect(!r0_g1_b2.eql(ten_of_each));498 return if (self.bits.isSet(index)) self.values[index] else null;
626 try testing.expect(!ten_of_each.eql(empty));499 }
627500
628 try testing.expect(empty.subsetOf(empty));501 /// Gets the value associated with a key, which must
629 try testing.expect(r0_g1_b2.subsetOf(r0_g1_b2));502 /// exist in the map.
630 try testing.expect(empty.subsetOf(r0_g1_b2));503 pub fn getAssertContains(self: Self, key: Key) Value {
631 try testing.expect(r0_g1_b2.subsetOf(ten_of_each));504 const index = Indexer.indexOf(key);
632 try testing.expect(!ten_of_each.subsetOf(r0_g1_b2));505 assert(self.bits.isSet(index));
633 try testing.expect(!r0_g1_b2.subsetOf(empty));506 return self.values[index];
507 }
634508
635 try testing.expect(empty.supersetOf(empty));509 /// Gets the address of the value associated with a key.
636 try testing.expect(r0_g1_b2.supersetOf(r0_g1_b2));510 /// If the key is not in the map, returns null.
637 try testing.expect(r0_g1_b2.supersetOf(empty));511 pub fn getPtr(self: *Self, key: Key) ?*Value {
638 try testing.expect(ten_of_each.supersetOf(r0_g1_b2));512 const index = Indexer.indexOf(key);
639 try testing.expect(!r0_g1_b2.supersetOf(ten_of_each));513 return if (self.bits.isSet(index)) &self.values[index] else null;
640 try testing.expect(!empty.supersetOf(r0_g1_b2));514 }
641515
642 {516 /// Gets the address of the const value associated with a key.
643 // with multisets it could be the case where two517 /// If the key is not in the map, returns null.
644 // multisets are neither subset nor superset of each518 pub fn getPtrConst(self: *const Self, key: Key) ?*const Value {
645 // other.519 const index = Indexer.indexOf(key);
520 return if (self.bits.isSet(index)) &self.values[index] else null;
521 }
646522
647 const r10 = EnumMultiset(Ball).init(.{523 /// Gets the address of the value associated with a key.
648 .red = 10,524 /// The key must be present in the map.
649 });525 pub fn getPtrAssertContains(self: *Self, key: Key) *Value {
650 const b10 = EnumMultiset(Ball).init(.{526 const index = Indexer.indexOf(key);
651 .blue = 10,527 assert(self.bits.isSet(index));
652 });528 return &self.values[index];
529 }
653530
654 try testing.expect(!r10.subsetOf(b10));531 /// Gets the address of the const value associated with a key.
655 try testing.expect(!b10.subsetOf(r10));532 /// The key must be present in the map.
656 try testing.expect(!r10.supersetOf(b10));533 pub fn getPtrConstAssertContains(self: *const Self, key: Key) *const Value {
657 try testing.expect(!b10.supersetOf(r10));534 const index = Indexer.indexOf(key);
658 }535 assert(self.bits.isSet(index));
536 return &self.values[index];
537 }
659538
660 {539 /// Adds the key to the map with the supplied value.
661 const result = r0_g1_b2.plusAssertSafe(ten_of_each);540 /// If the key is already in the map, overwrites the value.
662 try testing.expectEqual(result.getCount(.red), 10);541 pub fn put(self: *Self, key: Key, value: Value) void {
663 try testing.expectEqual(result.getCount(.green), 11);542 const index = Indexer.indexOf(key);
664 try testing.expectEqual(result.getCount(.blue), 12);543 self.bits.set(index);
665 }544 self.values[index] = value;
545 }
666546
667 {547 /// Adds the key to the map with an undefined value.
668 const result = try r0_g1_b2.plus(ten_of_each);548 /// If the key is already in the map, the value becomes undefined.
669 try testing.expectEqual(result.getCount(.red), 10);549 /// A pointer to the value is returned, which should be
670 try testing.expectEqual(result.getCount(.green), 11);550 /// used to initialize the value.
671 try testing.expectEqual(result.getCount(.blue), 12);551 pub fn putUninitialized(self: *Self, key: Key) *Value {
552 const index = Indexer.indexOf(key);
553 self.bits.set(index);
554 self.values[index] = undefined;
555 return &self.values[index];
556 }
672557
673 const full = EnumMultiset(Ball).initWithCount(std.math.maxInt(usize));558 /// Sets the value associated with the key in the map,
674 try testing.expectError(error.Overflow, result.plus(full));559 /// and returns the old value. If the key was not in
675 }560 /// the map, returns null.
561 pub fn fetchPut(self: *Self, key: Key, value: Value) ?Value {
562 const index = Indexer.indexOf(key);
563 const result: ?Value = if (self.bits.isSet(index)) self.values[index] else null;
564 self.bits.set(index);
565 self.values[index] = value;
566 return result;
567 }
676568
677 {569 /// Removes a key from the map. If the key was not in the map,
678 const result = ten_of_each.minus(r0_g1_b2);570 /// does nothing.
679 try testing.expectEqual(result.getCount(.red), 10);571 pub fn remove(self: *Self, key: Key) void {
680 try testing.expectEqual(result.getCount(.green), 9);572 const index = Indexer.indexOf(key);
681 try testing.expectEqual(result.getCount(.blue), 8);573 self.bits.unset(index);
682 }574 self.values[index] = undefined;
575 }
683576
684 {577 /// Removes a key from the map, and returns the old value.
685 const result = ten_of_each.minus(r0_g1_b2).minus(ten_of_each);578 /// If the key was not in the map, returns null.
686 try testing.expectEqual(result.getCount(.red), 0);579 pub fn fetchRemove(self: *Self, key: Key) ?Value {
687 try testing.expectEqual(result.getCount(.green), 0);580 const index = Indexer.indexOf(key);
688 try testing.expectEqual(result.getCount(.blue), 0);581 const result: ?Value = if (self.bits.isSet(index)) self.values[index] else null;
689 }582 self.bits.unset(index);
583 self.values[index] = undefined;
584 return result;
585 }
690586
691 {587 /// Returns an iterator over the map, which visits items in index order.
692 var copy = empty;588 /// Modifications to the underlying map may or may not be observed by
693 var it = copy.iterator();589 /// the iterator, but will not invalidate it.
694 var entry = it.next().?;590 pub fn iterator(self: *Self) Iterator {
695 try testing.expectEqual(entry.key, .red);591 return .{
696 try testing.expectEqual(entry.value.*, 0);592 .inner = self.bits.iterator(.{}),
697 entry = it.next().?;593 .values = &self.values,
698 try testing.expectEqual(entry.key, .green);594 };
699 try testing.expectEqual(entry.value.*, 0);595 }
700 entry = it.next().?;
701 try testing.expectEqual(entry.key, .blue);
702 try testing.expectEqual(entry.value.*, 0);
703 try testing.expectEqual(it.next(), null);
704 }
705596
706 {597 /// An entry in the map.
707 var copy = r0_g1_b2;598 pub const Entry = struct {
708 var it = copy.iterator();599 /// The key associated with this entry.
709 var entry = it.next().?;600 /// Modifying this key will not change the map.
710 try testing.expectEqual(entry.key, .red);601 key: Key,
711 try testing.expectEqual(entry.value.*, 0);
712 entry = it.next().?;
713 try testing.expectEqual(entry.key, .green);
714 try testing.expectEqual(entry.value.*, 1);
715 entry = it.next().?;
716 try testing.expectEqual(entry.key, .blue);
717 try testing.expectEqual(entry.value.*, 2);
718 try testing.expectEqual(it.next(), null);
719 }
720}
721602
722/// An array keyed by an enum, backed by a dense array.603 /// A pointer to the value in the map associated
723/// If the enum is not dense, a mapping will be constructed from604 /// with this key. Modifications through this
724/// enum values to dense indices. This type does no dynamic605 /// pointer will modify the underlying data.
725/// allocation and can be copied by value.606 value: *Value,
726pub fn EnumArray(comptime E: type, comptime V: type) type {607 };
727 const mixin = struct {608
728 fn EnumArrayExt(comptime Self: type) type {609 pub const Iterator = struct {
729 const Indexer = Self.Indexer;610 inner: BitSet.Iterator(.{}),
730 return struct {611 values: *[Indexer.count]Value,
731 /// Initializes all values in the enum array
732 pub fn init(init_values: EnumFieldStruct(E, V, @as(?V, null))) Self {
733 return initDefault(@as(?V, null), init_values);
734 }
735612
736 /// Initializes values in the enum array, with the specified default.613 pub fn next(self: *Iterator) ?Entry {
737 pub fn initDefault(comptime default: ?V, init_values: EnumFieldStruct(E, V, default)) Self {614 return if (self.inner.next()) |index|
738 var result = Self{ .values = undefined };615 Entry{
739 comptime var i: usize = 0;616 .key = Indexer.keyForIndex(index),
740 inline while (i < Self.len) : (i += 1) {617 .value = &self.values[index],
741 const key = comptime Indexer.keyForIndex(i);
742 const tag = @tagName(key);
743 result.values[i] = @field(init_values, tag);
744 }618 }
745 return result;619 else
746 }620 null;
747 };621 }
748 }622 };
749 };623 };
750 return IndexedArray(EnumIndexer(E), V, mixin.EnumArrayExt);
751}624}
752625
753fn NoExtension(comptime Self: type) type {626/// A multiset of enum elements up to a count of usize. Backed
754 _ = Self;627/// by an EnumArray. This type does no dynamic allocation and can
755 return NoExt;628/// be copied by value.
629pub fn EnumMultiset(comptime E: type) type {
630 return BoundedEnumMultiset(E, usize);
756}631}
757const NoExt = struct {};
758632
759/// A set type with an Indexer mapping from keys to indices.633/// A multiset of enum elements up to CountSize. Backed by an
760/// Presence or absence is stored as a dense bitfield. This634/// EnumArray. This type does no dynamic allocation and can be
761/// type does no allocation and can be copied by value.635/// copied by value.
762pub fn IndexedSet(comptime I: type, comptime Ext: ?fn (type) type) type {636pub fn BoundedEnumMultiset(comptime E: type, comptime CountSize: type) type {
763 comptime ensureIndexer(I);
764 return struct {637 return struct {
765 const Self = @This();638 const Self = @This();
766639
767 pub usingnamespace (Ext orelse NoExtension)(Self);640 counts: EnumArray(E, CountSize),
768
769 /// The indexing rules for converting between keys and indices.
770 pub const Indexer = I;
771 /// The element type for this set.
772 pub const Key = Indexer.Key;
773
774 const BitSet = std.StaticBitSet(Indexer.count);
775
776 /// The maximum number of items in this set.
777 pub const len = Indexer.count;
778
779 bits: BitSet = BitSet.initEmpty(),
780641
781 /// Returns a set containing no keys.642 /// Initializes the multiset using a struct of counts.
782 pub fn initEmpty() Self {643 pub fn init(init_counts: EnumFieldStruct(E, CountSize, 0)) Self {
783 return .{ .bits = BitSet.initEmpty() };644 var self = initWithCount(0);
645 inline for (@typeInfo(E).Enum.fields) |field| {
646 const c = @field(init_counts, field.name);
647 const key = @as(E, @enumFromInt(field.value));
648 self.counts.set(key, c);
649 }
650 return self;
784 }651 }
785652
786 /// Returns a set containing all possible keys.653 /// Initializes the multiset with a count of zero.
787 pub fn initFull() Self {654 pub fn initEmpty() Self {
788 return .{ .bits = BitSet.initFull() };655 return initWithCount(0);
789 }656 }
790657
791 /// Returns a set containing multiple keys.658 /// Initializes the multiset with all keys at the
792 pub fn initMany(keys: []const Key) Self {659 /// same count.
793 var set = initEmpty();660 pub fn initWithCount(comptime c: CountSize) Self {
794 for (keys) |key| set.insert(key);661 return .{
795 return set;662 .counts = EnumArray(E, CountSize).initDefault(c, .{}),
663 };
796 }664 }
797665
798 /// Returns a set containing a single key.666 /// Returns the total number of key counts in the multiset.
799 pub fn initOne(key: Key) Self {667 pub fn count(self: Self) usize {
800 return initMany(&[_]Key{key});668 var sum: usize = 0;
669 for (self.counts.values) |c| {
670 sum += c;
671 }
672 return sum;
801 }673 }
802674
803 /// Returns the number of keys in the set.675 /// Checks if at least one key in multiset.
804 pub fn count(self: Self) usize {676 pub fn contains(self: Self, key: E) bool {
805 return self.bits.count();677 return self.counts.get(key) > 0;
806 }678 }
807679
808 /// Checks if a key is in the set.680 /// Removes all instance of a key from multiset. Same as
809 pub fn contains(self: Self, key: Key) bool {681 /// setCount(key, 0).
810 return self.bits.isSet(Indexer.indexOf(key));682 pub fn removeAll(self: *Self, key: E) void {
683 return self.counts.set(key, 0);
811 }684 }
812685
813 /// Puts a key in the set.686 /// Increases the key count by given amount. Caller asserts
814 pub fn insert(self: *Self, key: Key) void {687 /// operation will not overflow.
815 self.bits.set(Indexer.indexOf(key));688 pub fn addAssertSafe(self: *Self, key: E, c: CountSize) void {
689 self.counts.getPtr(key).* += c;
816 }690 }
817691
818 /// Removes a key from the set.692 /// Increases the key count by given amount.
819 pub fn remove(self: *Self, key: Key) void {693 pub fn add(self: *Self, key: E, c: CountSize) error{Overflow}!void {
820 self.bits.unset(Indexer.indexOf(key));694 self.counts.set(key, try std.math.add(CountSize, self.counts.get(key), c));
821 }695 }
822696
823 /// Changes the presence of a key in the set to match the passed bool.697 /// Decreases the key count by given amount. If amount is
824 pub fn setPresent(self: *Self, key: Key, present: bool) void {698 /// greater than the number of keys in multset, then key count
825 self.bits.setValue(Indexer.indexOf(key), present);699 /// will be set to zero.
700 pub fn remove(self: *Self, key: E, c: CountSize) void {
701 self.counts.getPtr(key).* -= @min(self.getCount(key), c);
826 }702 }
827703
828 /// Toggles the presence of a key in the set. If the key is in704 /// Returns the count for a key.
829 /// the set, removes it. Otherwise adds it.705 pub fn getCount(self: Self, key: E) CountSize {
830 pub fn toggle(self: *Self, key: Key) void {706 return self.counts.get(key);
831 self.bits.toggle(Indexer.indexOf(key));
832 }707 }
833708
834 /// Toggles the presence of all keys in the passed set.709 /// Set the count for a key.
835 pub fn toggleSet(self: *Self, other: Self) void {710 pub fn setCount(self: *Self, key: E, c: CountSize) void {
836 self.bits.toggleSet(other.bits);711 self.counts.set(key, c);
837 }712 }
838713
839 /// Toggles all possible keys in the set.714 /// Increases the all key counts by given multiset. Caller
840 pub fn toggleAll(self: *Self) void {715 /// asserts operation will not overflow any key.
841 self.bits.toggleAll();716 pub fn addSetAssertSafe(self: *Self, other: Self) void {
717 inline for (@typeInfo(E).Enum.fields) |field| {
718 const key = @as(E, @enumFromInt(field.value));
719 self.addAssertSafe(key, other.getCount(key));
720 }
842 }721 }
843722
844 /// Adds all keys in the passed set to this set.723 /// Increases the all key counts by given multiset.
845 pub fn setUnion(self: *Self, other: Self) void {724 pub fn addSet(self: *Self, other: Self) error{Overflow}!void {
846 self.bits.setUnion(other.bits);725 inline for (@typeInfo(E).Enum.fields) |field| {
726 const key = @as(E, @enumFromInt(field.value));
727 try self.add(key, other.getCount(key));
728 }
847 }729 }
848730
849 /// Removes all keys which are not in the passed set.731 /// Decreases the all key counts by given multiset. If
850 pub fn setIntersection(self: *Self, other: Self) void {732 /// the given multiset has more key counts than this,
851 self.bits.setIntersection(other.bits);733 /// then that key will have a key count of zero.
734 pub fn removeSet(self: *Self, other: Self) void {
735 inline for (@typeInfo(E).Enum.fields) |field| {
736 const key = @as(E, @enumFromInt(field.value));
737 self.remove(key, other.getCount(key));
738 }
852 }739 }
853740
854 /// Returns true iff both sets have the same keys.741 /// Returns true iff all key counts are the same as
742 /// given multiset.
855 pub fn eql(self: Self, other: Self) bool {743 pub fn eql(self: Self, other: Self) bool {
856 return self.bits.eql(other.bits);744 inline for (@typeInfo(E).Enum.fields) |field| {
745 const key = @as(E, @enumFromInt(field.value));
746 if (self.getCount(key) != other.getCount(key)) {
747 return false;
748 }
749 }
750 return true;
857 }751 }
858752
859 /// Returns true iff all the keys in this set are753 /// Returns true iff all key counts less than or
860 /// in the other set. The other set may have keys754 /// equal to the given multiset.
861 /// not found in this set.
862 pub fn subsetOf(self: Self, other: Self) bool {755 pub fn subsetOf(self: Self, other: Self) bool {
863 return self.bits.subsetOf(other.bits);756 inline for (@typeInfo(E).Enum.fields) |field| {
757 const key = @as(E, @enumFromInt(field.value));
758 if (self.getCount(key) > other.getCount(key)) {
759 return false;
760 }
761 }
762 return true;
864 }763 }
865764
866 /// Returns true iff this set contains all the keys765 /// Returns true iff all key counts greater than or
867 /// in the other set. This set may have keys not766 /// equal to the given multiset.
868 /// found in the other set.
869 pub fn supersetOf(self: Self, other: Self) bool {767 pub fn supersetOf(self: Self, other: Self) bool {
870 return self.bits.supersetOf(other.bits);768 inline for (@typeInfo(E).Enum.fields) |field| {
871 }769 const key = @as(E, @enumFromInt(field.value));
872770 if (self.getCount(key) < other.getCount(key)) {
873 /// Returns a set with all the keys not in this set.771 return false;
874 pub fn complement(self: Self) Self {772 }
875 return .{ .bits = self.bits.complement() };773 }
876 }774 return true;
877
878 /// Returns a set with keys that are in either this
879 /// set or the other set.
880 pub fn unionWith(self: Self, other: Self) Self {
881 return .{ .bits = self.bits.unionWith(other.bits) };
882 }
883
884 /// Returns a set with keys that are in both this
885 /// set and the other set.
886 pub fn intersectWith(self: Self, other: Self) Self {
887 return .{ .bits = self.bits.intersectWith(other.bits) };
888 }775 }
889776
890 /// Returns a set with keys that are in either this777 /// Returns a multiset with the total key count of this
891 /// set or the other set, but not both.778 /// multiset and the other multiset. Caller asserts
892 pub fn xorWith(self: Self, other: Self) Self {779 /// operation will not overflow any key.
893 return .{ .bits = self.bits.xorWith(other.bits) };780 pub fn plusAssertSafe(self: Self, other: Self) Self {
781 var result = self;
782 result.addSetAssertSafe(other);
783 return result;
894 }784 }
895785
896 /// Returns a set with keys that are in this set786 /// Returns a multiset with the total key count of this
897 /// except for keys in the other set.787 /// multiset and the other multiset.
898 pub fn differenceWith(self: Self, other: Self) Self {788 pub fn plus(self: Self, other: Self) error{Overflow}!Self {
899 return .{ .bits = self.bits.differenceWith(other.bits) };789 var result = self;
790 try result.addSet(other);
791 return result;
900 }792 }
901793
902 /// Returns an iterator over this set, which iterates in794 /// Returns a multiset with the key count of this
903 /// index order. Modifications to the set during iteration795 /// multiset minus the corresponding key count in the
904 /// may or may not be observed by the iterator, but will796 /// other multiset. If the other multiset contains
905 /// not invalidate it.797 /// more key count than this set, that key will have
906 pub fn iterator(self: *const Self) Iterator {798 /// a count of zero.
907 return .{ .inner = self.bits.iterator(.{}) };799 pub fn minus(self: Self, other: Self) Self {
800 var result = self;
801 result.removeSet(other);
802 return result;
908 }803 }
909804
910 pub const Iterator = struct {805 pub const Entry = EnumArray(E, CountSize).Entry;
911 inner: BitSet.Iterator(.{}),806 pub const Iterator = EnumArray(E, CountSize).Iterator;
912807
913 pub fn next(self: *Iterator) ?Key {808 /// Returns an iterator over this multiset. Keys with zero
914 return if (self.inner.next()) |index|809 /// counts are included. Modifications to the set during
915 Indexer.keyForIndex(index)810 /// iteration may or may not be observed by the iterator,
916 else811 /// but will not invalidate it.
917 null;812 pub fn iterator(self: *Self) Iterator {
918 }813 return self.counts.iterator();
919 };814 }
920 };815 };
921}816}
922817
923test "pure EnumSet fns" {818test EnumMultiset {
924 const Suit = enum { spades, hearts, clubs, diamonds };819 const Ball = enum { red, green, blue };
925
926 const empty = EnumSet(Suit).initEmpty();
927 const full = EnumSet(Suit).initFull();
928 const black = EnumSet(Suit).initMany(&[_]Suit{ .spades, .clubs });
929 const red = EnumSet(Suit).initMany(&[_]Suit{ .hearts, .diamonds });
930
931 try testing.expect(empty.eql(empty));
932 try testing.expect(full.eql(full));
933 try testing.expect(!empty.eql(full));
934 try testing.expect(!full.eql(empty));
935 try testing.expect(!empty.eql(black));
936 try testing.expect(!full.eql(red));
937 try testing.expect(!red.eql(empty));
938 try testing.expect(!black.eql(full));
939
940 try testing.expect(empty.subsetOf(empty));
941 try testing.expect(empty.subsetOf(full));
942 try testing.expect(full.subsetOf(full));
943 try testing.expect(!black.subsetOf(red));
944 try testing.expect(!red.subsetOf(black));
945
946 try testing.expect(full.supersetOf(full));
947 try testing.expect(full.supersetOf(empty));
948 try testing.expect(empty.supersetOf(empty));
949 try testing.expect(!black.supersetOf(red));
950 try testing.expect(!red.supersetOf(black));
951
952 try testing.expect(empty.complement().eql(full));
953 try testing.expect(full.complement().eql(empty));
954 try testing.expect(black.complement().eql(red));
955 try testing.expect(red.complement().eql(black));
956820
957 try testing.expect(empty.unionWith(empty).eql(empty));821 const empty = EnumMultiset(Ball).initEmpty();
958 try testing.expect(empty.unionWith(full).eql(full));822 const r0_g1_b2 = EnumMultiset(Ball).init(.{
959 try testing.expect(full.unionWith(full).eql(full));823 .red = 0,
960 try testing.expect(full.unionWith(empty).eql(full));824 .green = 1,
961 try testing.expect(black.unionWith(red).eql(full));825 .blue = 2,
962 try testing.expect(red.unionWith(black).eql(full));826 });
827 const ten_of_each = EnumMultiset(Ball).initWithCount(10);
963828
964 try testing.expect(empty.intersectWith(empty).eql(empty));829 try testing.expectEqual(empty.count(), 0);
965 try testing.expect(empty.intersectWith(full).eql(empty));830 try testing.expectEqual(r0_g1_b2.count(), 3);
966 try testing.expect(full.intersectWith(full).eql(full));831 try testing.expectEqual(ten_of_each.count(), 30);
967 try testing.expect(full.intersectWith(empty).eql(empty));
968 try testing.expect(black.intersectWith(red).eql(empty));
969 try testing.expect(red.intersectWith(black).eql(empty));
970832
971 try testing.expect(empty.xorWith(empty).eql(empty));833 try testing.expect(!empty.contains(.red));
972 try testing.expect(empty.xorWith(full).eql(full));834 try testing.expect(!empty.contains(.green));
973 try testing.expect(full.xorWith(full).eql(empty));835 try testing.expect(!empty.contains(.blue));
974 try testing.expect(full.xorWith(empty).eql(full));
975 try testing.expect(black.xorWith(red).eql(full));
976 try testing.expect(red.xorWith(black).eql(full));
977836
978 try testing.expect(empty.differenceWith(empty).eql(empty));837 try testing.expect(!r0_g1_b2.contains(.red));
979 try testing.expect(empty.differenceWith(full).eql(empty));838 try testing.expect(r0_g1_b2.contains(.green));
980 try testing.expect(full.differenceWith(full).eql(empty));839 try testing.expect(r0_g1_b2.contains(.blue));
981 try testing.expect(full.differenceWith(empty).eql(full));
982 try testing.expect(full.differenceWith(red).eql(black));
983 try testing.expect(full.differenceWith(black).eql(red));
984}
985840
986test "EnumSet empty" {841 try testing.expect(ten_of_each.contains(.red));
987 const E = enum {};842 try testing.expect(ten_of_each.contains(.green));
988 const empty = EnumSet(E).initEmpty();843 try testing.expect(ten_of_each.contains(.blue));
989 const full = EnumSet(E).initFull();
990844
991 try std.testing.expect(empty.eql(full));845 {
992 try std.testing.expect(empty.complement().eql(full));846 var copy = ten_of_each;
993 try std.testing.expect(empty.complement().eql(full.complement()));847 copy.removeAll(.red);
994 try std.testing.expect(empty.eql(full.complement()));848 try testing.expect(!copy.contains(.red));
995}
996849
997test "EnumSet const iterator" {850 // removeAll second time does nothing
998 const Direction = enum { up, down, left, right };851 copy.removeAll(.red);
999 const diag_move = init: {852 try testing.expect(!copy.contains(.red));
1000 var move = EnumSet(Direction).initEmpty();853 }
1001 move.insert(.right);
1002 move.insert(.up);
1003 break :init move;
1004 };
1005854
1006 var result = EnumSet(Direction).initEmpty();855 {
1007 var it = diag_move.iterator();856 var copy = ten_of_each;
1008 while (it.next()) |dir| {857 copy.addAssertSafe(.red, 6);
1009 result.insert(dir);858 try testing.expectEqual(copy.getCount(.red), 16);
1010 }859 }
1011860
1012 try testing.expect(result.eql(diag_move));861 {
1013}862 var copy = ten_of_each;
863 try copy.add(.red, 6);
864 try testing.expectEqual(copy.getCount(.red), 16);
1014865
1015/// A map from keys to values, using an index lookup. Uses a866 try testing.expectError(error.Overflow, copy.add(.red, std.math.maxInt(usize)));
1016/// bitfield to track presence and a dense array of values.867 }
1017/// This type does no allocation and can be copied by value.
1018pub fn IndexedMap(comptime I: type, comptime V: type, comptime Ext: ?fn (type) type) type {
1019 comptime ensureIndexer(I);
1020 return struct {
1021 const Self = @This();
1022868
1023 pub usingnamespace (Ext orelse NoExtension)(Self);869 {
870 var copy = ten_of_each;
871 copy.remove(.red, 4);
872 try testing.expectEqual(copy.getCount(.red), 6);
1024873
1025 /// The index mapping for this map874 // subtracting more it contains does not underflow
1026 pub const Indexer = I;875 copy.remove(.green, 14);
1027 /// The key type used to index this map876 try testing.expectEqual(copy.getCount(.green), 0);
1028 pub const Key = Indexer.Key;877 }
1029 /// The value type stored in this map
1030 pub const Value = V;
1031 /// The number of possible keys in the map
1032 pub const len = Indexer.count;
1033878
1034 const BitSet = std.StaticBitSet(Indexer.count);879 try testing.expectEqual(empty.getCount(.green), 0);
880 try testing.expectEqual(r0_g1_b2.getCount(.green), 1);
881 try testing.expectEqual(ten_of_each.getCount(.green), 10);
1035882
1036 /// Bits determining whether items are in the map883 {
1037 bits: BitSet = BitSet.initEmpty(),884 var copy = empty;
1038 /// Values of items in the map. If the associated885 copy.setCount(.red, 6);
1039 /// bit is zero, the value is undefined.886 try testing.expectEqual(copy.getCount(.red), 6);
1040 values: [Indexer.count]Value = undefined,887 }
1041888
1042 /// The number of items in the map.889 {
1043 pub fn count(self: Self) usize {890 var copy = r0_g1_b2;
1044 return self.bits.count();891 copy.addSetAssertSafe(ten_of_each);
1045 }892 try testing.expectEqual(copy.getCount(.red), 10);
893 try testing.expectEqual(copy.getCount(.green), 11);
894 try testing.expectEqual(copy.getCount(.blue), 12);
895 }
1046896
1047 /// Checks if the map contains an item.897 {
1048 pub fn contains(self: Self, key: Key) bool {898 var copy = r0_g1_b2;
1049 return self.bits.isSet(Indexer.indexOf(key));899 try copy.addSet(ten_of_each);
1050 }900 try testing.expectEqual(copy.getCount(.red), 10);
901 try testing.expectEqual(copy.getCount(.green), 11);
902 try testing.expectEqual(copy.getCount(.blue), 12);
1051903
1052 /// Gets the value associated with a key.904 const full = EnumMultiset(Ball).initWithCount(std.math.maxInt(usize));
1053 /// If the key is not in the map, returns null.905 try testing.expectError(error.Overflow, copy.addSet(full));
1054 pub fn get(self: Self, key: Key) ?Value {906 }
1055 const index = Indexer.indexOf(key);
1056 return if (self.bits.isSet(index)) self.values[index] else null;
1057 }
1058907
1059 /// Gets the value associated with a key, which must908 {
1060 /// exist in the map.909 var copy = ten_of_each;
1061 pub fn getAssertContains(self: Self, key: Key) Value {910 copy.removeSet(r0_g1_b2);
1062 const index = Indexer.indexOf(key);911 try testing.expectEqual(copy.getCount(.red), 10);
1063 assert(self.bits.isSet(index));912 try testing.expectEqual(copy.getCount(.green), 9);
1064 return self.values[index];913 try testing.expectEqual(copy.getCount(.blue), 8);
1065 }
1066914
1067 /// Gets the address of the value associated with a key.915 copy.removeSet(ten_of_each);
1068 /// If the key is not in the map, returns null.916 try testing.expectEqual(copy.getCount(.red), 0);
1069 pub fn getPtr(self: *Self, key: Key) ?*Value {917 try testing.expectEqual(copy.getCount(.green), 0);
1070 const index = Indexer.indexOf(key);918 try testing.expectEqual(copy.getCount(.blue), 0);
1071 return if (self.bits.isSet(index)) &self.values[index] else null;919 }
1072 }
1073920
1074 /// Gets the address of the const value associated with a key.921 try testing.expect(empty.eql(empty));
1075 /// If the key is not in the map, returns null.922 try testing.expect(r0_g1_b2.eql(r0_g1_b2));
1076 pub fn getPtrConst(self: *const Self, key: Key) ?*const Value {923 try testing.expect(ten_of_each.eql(ten_of_each));
1077 const index = Indexer.indexOf(key);924 try testing.expect(!empty.eql(r0_g1_b2));
1078 return if (self.bits.isSet(index)) &self.values[index] else null;925 try testing.expect(!r0_g1_b2.eql(ten_of_each));
1079 }926 try testing.expect(!ten_of_each.eql(empty));
1080927
1081 /// Gets the address of the value associated with a key.928 try testing.expect(empty.subsetOf(empty));
1082 /// The key must be present in the map.929 try testing.expect(r0_g1_b2.subsetOf(r0_g1_b2));
1083 pub fn getPtrAssertContains(self: *Self, key: Key) *Value {930 try testing.expect(empty.subsetOf(r0_g1_b2));
1084 const index = Indexer.indexOf(key);931 try testing.expect(r0_g1_b2.subsetOf(ten_of_each));
1085 assert(self.bits.isSet(index));932 try testing.expect(!ten_of_each.subsetOf(r0_g1_b2));
1086 return &self.values[index];933 try testing.expect(!r0_g1_b2.subsetOf(empty));
1087 }
1088934
1089 /// Gets the address of the const value associated with a key.935 try testing.expect(empty.supersetOf(empty));
1090 /// The key must be present in the map.936 try testing.expect(r0_g1_b2.supersetOf(r0_g1_b2));
1091 pub fn getPtrConstAssertContains(self: *const Self, key: Key) *const Value {937 try testing.expect(r0_g1_b2.supersetOf(empty));
1092 const index = Indexer.indexOf(key);938 try testing.expect(ten_of_each.supersetOf(r0_g1_b2));
1093 assert(self.bits.isSet(index));939 try testing.expect(!r0_g1_b2.supersetOf(ten_of_each));
1094 return &self.values[index];940 try testing.expect(!empty.supersetOf(r0_g1_b2));
1095 }
1096941
1097 /// Adds the key to the map with the supplied value.942 {
1098 /// If the key is already in the map, overwrites the value.943 // with multisets it could be the case where two
1099 pub fn put(self: *Self, key: Key, value: Value) void {944 // multisets are neither subset nor superset of each
1100 const index = Indexer.indexOf(key);945 // other.
1101 self.bits.set(index);
1102 self.values[index] = value;
1103 }
1104946
1105 /// Adds the key to the map with an undefined value.947 const r10 = EnumMultiset(Ball).init(.{
1106 /// If the key is already in the map, the value becomes undefined.948 .red = 10,
1107 /// A pointer to the value is returned, which should be949 });
1108 /// used to initialize the value.950 const b10 = EnumMultiset(Ball).init(.{
1109 pub fn putUninitialized(self: *Self, key: Key) *Value {951 .blue = 10,
1110 const index = Indexer.indexOf(key);952 });
1111 self.bits.set(index);
1112 self.values[index] = undefined;
1113 return &self.values[index];
1114 }
1115953
1116 /// Sets the value associated with the key in the map,954 try testing.expect(!r10.subsetOf(b10));
1117 /// and returns the old value. If the key was not in955 try testing.expect(!b10.subsetOf(r10));
1118 /// the map, returns null.956 try testing.expect(!r10.supersetOf(b10));
1119 pub fn fetchPut(self: *Self, key: Key, value: Value) ?Value {957 try testing.expect(!b10.supersetOf(r10));
1120 const index = Indexer.indexOf(key);958 }
1121 const result: ?Value = if (self.bits.isSet(index)) self.values[index] else null;
1122 self.bits.set(index);
1123 self.values[index] = value;
1124 return result;
1125 }
1126959
1127 /// Removes a key from the map. If the key was not in the map,960 {
1128 /// does nothing.961 const result = r0_g1_b2.plusAssertSafe(ten_of_each);
1129 pub fn remove(self: *Self, key: Key) void {962 try testing.expectEqual(result.getCount(.red), 10);
1130 const index = Indexer.indexOf(key);963 try testing.expectEqual(result.getCount(.green), 11);
1131 self.bits.unset(index);964 try testing.expectEqual(result.getCount(.blue), 12);
1132 self.values[index] = undefined;965 }
1133 }
1134966
1135 /// Removes a key from the map, and returns the old value.967 {
1136 /// If the key was not in the map, returns null.968 const result = try r0_g1_b2.plus(ten_of_each);
1137 pub fn fetchRemove(self: *Self, key: Key) ?Value {969 try testing.expectEqual(result.getCount(.red), 10);
1138 const index = Indexer.indexOf(key);970 try testing.expectEqual(result.getCount(.green), 11);
1139 const result: ?Value = if (self.bits.isSet(index)) self.values[index] else null;971 try testing.expectEqual(result.getCount(.blue), 12);
1140 self.bits.unset(index);
1141 self.values[index] = undefined;
1142 return result;
1143 }
1144972
1145 /// Returns an iterator over the map, which visits items in index order.973 const full = EnumMultiset(Ball).initWithCount(std.math.maxInt(usize));
1146 /// Modifications to the underlying map may or may not be observed by974 try testing.expectError(error.Overflow, result.plus(full));
1147 /// the iterator, but will not invalidate it.975 }
1148 pub fn iterator(self: *Self) Iterator {
1149 return .{
1150 .inner = self.bits.iterator(.{}),
1151 .values = &self.values,
1152 };
1153 }
1154976
1155 /// An entry in the map.977 {
1156 pub const Entry = struct {978 const result = ten_of_each.minus(r0_g1_b2);
1157 /// The key associated with this entry.979 try testing.expectEqual(result.getCount(.red), 10);
1158 /// Modifying this key will not change the map.980 try testing.expectEqual(result.getCount(.green), 9);
1159 key: Key,981 try testing.expectEqual(result.getCount(.blue), 8);
982 }
1160983
1161 /// A pointer to the value in the map associated984 {
1162 /// with this key. Modifications through this985 const result = ten_of_each.minus(r0_g1_b2).minus(ten_of_each);
1163 /// pointer will modify the underlying data.986 try testing.expectEqual(result.getCount(.red), 0);
1164 value: *Value,987 try testing.expectEqual(result.getCount(.green), 0);
1165 };988 try testing.expectEqual(result.getCount(.blue), 0);
989 }
1166990
1167 pub const Iterator = struct {991 {
1168 inner: BitSet.Iterator(.{}),992 var copy = empty;
1169 values: *[Indexer.count]Value,993 var it = copy.iterator();
994 var entry = it.next().?;
995 try testing.expectEqual(entry.key, .red);
996 try testing.expectEqual(entry.value.*, 0);
997 entry = it.next().?;
998 try testing.expectEqual(entry.key, .green);
999 try testing.expectEqual(entry.value.*, 0);
1000 entry = it.next().?;
1001 try testing.expectEqual(entry.key, .blue);
1002 try testing.expectEqual(entry.value.*, 0);
1003 try testing.expectEqual(it.next(), null);
1004 }
11701005
1171 pub fn next(self: *Iterator) ?Entry {1006 {
1172 return if (self.inner.next()) |index|1007 var copy = r0_g1_b2;
1173 Entry{1008 var it = copy.iterator();
1174 .key = Indexer.keyForIndex(index),1009 var entry = it.next().?;
1175 .value = &self.values[index],1010 try testing.expectEqual(entry.key, .red);
1176 }1011 try testing.expectEqual(entry.value.*, 0);
1177 else1012 entry = it.next().?;
1178 null;1013 try testing.expectEqual(entry.key, .green);
1179 }1014 try testing.expectEqual(entry.value.*, 1);
1180 };1015 entry = it.next().?;
1181 };1016 try testing.expectEqual(entry.key, .blue);
1017 try testing.expectEqual(entry.value.*, 2);
1018 try testing.expectEqual(it.next(), null);
1019 }
1182}1020}
11831021
1184/// A dense array of values, using an indexed lookup.1022/// An array keyed by an enum, backed by a dense array.
1185/// This type does no allocation and can be copied by value.1023/// If the enum is not dense, a mapping will be constructed from
1186pub fn IndexedArray(comptime I: type, comptime V: type, comptime Ext: ?fn (type) type) type {1024/// enum values to dense indices. This type does no dynamic
1187 comptime ensureIndexer(I);1025/// allocation and can be copied by value.
1026pub fn EnumArray(comptime E: type, comptime V: type) type {
1188 return struct {1027 return struct {
1189 const Self = @This();1028 const Self = @This();
11901029
1191 pub usingnamespace (Ext orelse NoExtension)(Self);
1192
1193 /// The index mapping for this map1030 /// The index mapping for this map
1194 pub const Indexer = I;1031 pub const Indexer = EnumIndexer(E);
1195 /// The key type used to index this map1032 /// The key type used to index this map
1196 pub const Key = Indexer.Key;1033 pub const Key = Indexer.Key;
1197 /// The value type stored in this map1034 /// The value type stored in this map
...@@ -1201,6 +1038,21 @@ pub fn IndexedArray(comptime I: type, comptime V: type, comptime Ext: ?fn (type)...@@ -1201,6 +1038,21 @@ pub fn IndexedArray(comptime I: type, comptime V: type, comptime Ext: ?fn (type)
12011038
1202 values: [Indexer.count]Value,1039 values: [Indexer.count]Value,
12031040
1041 pub fn init(init_values: EnumFieldStruct(E, Value, null)) Self {
1042 return initDefault(null, init_values);
1043 }
1044
1045 /// Initializes values in the enum array, with the specified default.
1046 pub fn initDefault(comptime default: ?Value, init_values: EnumFieldStruct(E, Value, default)) Self {
1047 var result: Self = .{ .values = undefined };
1048 inline for (0..Self.len) |i| {
1049 const key = comptime Indexer.keyForIndex(i);
1050 const tag = @tagName(key);
1051 result.values[i] = @field(init_values, tag);
1052 }
1053 return result;
1054 }
1055
1204 pub fn initUndefined() Self {1056 pub fn initUndefined() Self {
1205 return Self{ .values = undefined };1057 return Self{ .values = undefined };
1206 }1058 }
...@@ -1269,46 +1121,96 @@ pub fn IndexedArray(comptime I: type, comptime V: type, comptime Ext: ?fn (type)...@@ -1269,46 +1121,96 @@ pub fn IndexedArray(comptime I: type, comptime V: type, comptime Ext: ?fn (type)
1269 };1121 };
1270}1122}
12711123
1272/// Verifies that a type is a valid Indexer, providing a helpful1124test "pure EnumSet fns" {
1273/// compile error if not. An Indexer maps a comptime-known set1125 const Suit = enum { spades, hearts, clubs, diamonds };
1274/// of keys to a dense set of zero-based indices.1126
1275/// The indexer interface must look like this:1127 const empty = EnumSet(Suit).initEmpty();
1276/// ```1128 const full = EnumSet(Suit).initFull();
1277/// struct {1129 const black = EnumSet(Suit).initMany(&[_]Suit{ .spades, .clubs });
1278/// /// The key type which this indexer converts to indices1130 const red = EnumSet(Suit).initMany(&[_]Suit{ .hearts, .diamonds });
1279/// pub const Key: type,1131
1280/// /// The number of indexes in the dense mapping1132 try testing.expect(empty.eql(empty));
1281/// pub const count: comptime_int,1133 try testing.expect(full.eql(full));
1282/// /// Converts from a key to an index1134 try testing.expect(!empty.eql(full));
1283/// pub fn indexOf(Key) usize;1135 try testing.expect(!full.eql(empty));
1284/// /// Converts from an index to a key1136 try testing.expect(!empty.eql(black));
1285/// pub fn keyForIndex(usize) Key;1137 try testing.expect(!full.eql(red));
1286/// }1138 try testing.expect(!red.eql(empty));
1287/// ```1139 try testing.expect(!black.eql(full));
1288pub fn ensureIndexer(comptime T: type) void {1140
1289 comptime {1141 try testing.expect(empty.subsetOf(empty));
1290 if (!@hasDecl(T, "Key")) @compileError("Indexer must have decl Key: type.");1142 try testing.expect(empty.subsetOf(full));
1291 if (@TypeOf(T.Key) != type) @compileError("Indexer.Key must be a type.");1143 try testing.expect(full.subsetOf(full));
1292 if (!@hasDecl(T, "count")) @compileError("Indexer must have decl count: comptime_int.");1144 try testing.expect(!black.subsetOf(red));
1293 if (@TypeOf(T.count) != comptime_int) @compileError("Indexer.count must be a comptime_int.");1145 try testing.expect(!red.subsetOf(black));
1294 if (!@hasDecl(T, "indexOf")) @compileError("Indexer.indexOf must be a fn (Key) usize.");1146
1295 if (@TypeOf(T.indexOf) != fn (T.Key) usize) @compileError("Indexer must have decl indexOf: fn (Key) usize.");1147 try testing.expect(full.supersetOf(full));
1296 if (!@hasDecl(T, "keyForIndex")) @compileError("Indexer must have decl keyForIndex: fn (usize) Key.");1148 try testing.expect(full.supersetOf(empty));
1297 if (@TypeOf(T.keyForIndex) != fn (usize) T.Key) @compileError("Indexer.keyForIndex must be a fn (usize) Key.");1149 try testing.expect(empty.supersetOf(empty));
1298 }1150 try testing.expect(!black.supersetOf(red));
1151 try testing.expect(!red.supersetOf(black));
1152
1153 try testing.expect(empty.complement().eql(full));
1154 try testing.expect(full.complement().eql(empty));
1155 try testing.expect(black.complement().eql(red));
1156 try testing.expect(red.complement().eql(black));
1157
1158 try testing.expect(empty.unionWith(empty).eql(empty));
1159 try testing.expect(empty.unionWith(full).eql(full));
1160 try testing.expect(full.unionWith(full).eql(full));
1161 try testing.expect(full.unionWith(empty).eql(full));
1162 try testing.expect(black.unionWith(red).eql(full));
1163 try testing.expect(red.unionWith(black).eql(full));
1164
1165 try testing.expect(empty.intersectWith(empty).eql(empty));
1166 try testing.expect(empty.intersectWith(full).eql(empty));
1167 try testing.expect(full.intersectWith(full).eql(full));
1168 try testing.expect(full.intersectWith(empty).eql(empty));
1169 try testing.expect(black.intersectWith(red).eql(empty));
1170 try testing.expect(red.intersectWith(black).eql(empty));
1171
1172 try testing.expect(empty.xorWith(empty).eql(empty));
1173 try testing.expect(empty.xorWith(full).eql(full));
1174 try testing.expect(full.xorWith(full).eql(empty));
1175 try testing.expect(full.xorWith(empty).eql(full));
1176 try testing.expect(black.xorWith(red).eql(full));
1177 try testing.expect(red.xorWith(black).eql(full));
1178
1179 try testing.expect(empty.differenceWith(empty).eql(empty));
1180 try testing.expect(empty.differenceWith(full).eql(empty));
1181 try testing.expect(full.differenceWith(full).eql(empty));
1182 try testing.expect(full.differenceWith(empty).eql(full));
1183 try testing.expect(full.differenceWith(red).eql(black));
1184 try testing.expect(full.differenceWith(black).eql(red));
1299}1185}
13001186
1301test ensureIndexer {1187test "EnumSet empty" {
1302 ensureIndexer(struct {1188 const E = enum {};
1303 pub const Key = u32;1189 const empty = EnumSet(E).initEmpty();
1304 pub const count: comptime_int = 8;1190 const full = EnumSet(E).initFull();
1305 pub fn indexOf(k: Key) usize {1191
1306 return @as(usize, @intCast(k));1192 try std.testing.expect(empty.eql(full));
1307 }1193 try std.testing.expect(empty.complement().eql(full));
1308 pub fn keyForIndex(index: usize) Key {1194 try std.testing.expect(empty.complement().eql(full.complement()));
1309 return @as(Key, @intCast(index));1195 try std.testing.expect(empty.eql(full.complement()));
1310 }1196}
1311 });1197
1198test "EnumSet const iterator" {
1199 const Direction = enum { up, down, left, right };
1200 const diag_move = init: {
1201 var move = EnumSet(Direction).initEmpty();
1202 move.insert(.right);
1203 move.insert(.up);
1204 break :init move;
1205 };
1206
1207 var result = EnumSet(Direction).initEmpty();
1208 var it = diag_move.iterator();
1209 while (it.next()) |dir| {
1210 result.insert(dir);
1211 }
1212
1213 try testing.expect(result.eql(diag_move));
1312}1214}
13131215
1314pub fn EnumIndexer(comptime E: type) type {1216pub fn EnumIndexer(comptime E: type) type {
...@@ -1438,7 +1340,6 @@ test "EnumIndexer non-exhaustive" {...@@ -1438,7 +1340,6 @@ test "EnumIndexer non-exhaustive" {
1438 _,1340 _,
1439 };1341 };
1440 const Indexer = EnumIndexer(E);1342 const Indexer = EnumIndexer(E);
1441 ensureIndexer(Indexer);
14421343
1443 const min_tag: E = @enumFromInt(std.math.minInt(BackingInt));1344 const min_tag: E = @enumFromInt(std.math.minInt(BackingInt));
1444 const max_tag: E = @enumFromInt(std.math.maxInt(BackingInt));1345 const max_tag: E = @enumFromInt(std.math.maxInt(BackingInt));
...@@ -1466,7 +1367,6 @@ test "EnumIndexer non-exhaustive" {...@@ -1466,7 +1367,6 @@ test "EnumIndexer non-exhaustive" {
1466test "EnumIndexer dense zeroed" {1367test "EnumIndexer dense zeroed" {
1467 const E = enum(u2) { b = 1, a = 0, c = 2 };1368 const E = enum(u2) { b = 1, a = 0, c = 2 };
1468 const Indexer = EnumIndexer(E);1369 const Indexer = EnumIndexer(E);
1469 ensureIndexer(Indexer);
1470 try testing.expectEqual(E, Indexer.Key);1370 try testing.expectEqual(E, Indexer.Key);
1471 try testing.expectEqual(3, Indexer.count);1371 try testing.expectEqual(3, Indexer.count);
14721372
...@@ -1482,7 +1382,6 @@ test "EnumIndexer dense zeroed" {...@@ -1482,7 +1382,6 @@ test "EnumIndexer dense zeroed" {
1482test "EnumIndexer dense positive" {1382test "EnumIndexer dense positive" {
1483 const E = enum(u4) { c = 6, a = 4, b = 5 };1383 const E = enum(u4) { c = 6, a = 4, b = 5 };
1484 const Indexer = EnumIndexer(E);1384 const Indexer = EnumIndexer(E);
1485 ensureIndexer(Indexer);
1486 try testing.expectEqual(E, Indexer.Key);1385 try testing.expectEqual(E, Indexer.Key);
1487 try testing.expectEqual(3, Indexer.count);1386 try testing.expectEqual(3, Indexer.count);
14881387
...@@ -1498,7 +1397,6 @@ test "EnumIndexer dense positive" {...@@ -1498,7 +1397,6 @@ test "EnumIndexer dense positive" {
1498test "EnumIndexer dense negative" {1397test "EnumIndexer dense negative" {
1499 const E = enum(i4) { a = -6, c = -4, b = -5 };1398 const E = enum(i4) { a = -6, c = -4, b = -5 };
1500 const Indexer = EnumIndexer(E);1399 const Indexer = EnumIndexer(E);
1501 ensureIndexer(Indexer);
1502 try testing.expectEqual(E, Indexer.Key);1400 try testing.expectEqual(E, Indexer.Key);
1503 try testing.expectEqual(3, Indexer.count);1401 try testing.expectEqual(3, Indexer.count);
15041402
...@@ -1514,7 +1412,6 @@ test "EnumIndexer dense negative" {...@@ -1514,7 +1412,6 @@ test "EnumIndexer dense negative" {
1514test "EnumIndexer sparse" {1412test "EnumIndexer sparse" {
1515 const E = enum(i4) { a = -2, c = 6, b = 4 };1413 const E = enum(i4) { a = -2, c = 6, b = 4 };
1516 const Indexer = EnumIndexer(E);1414 const Indexer = EnumIndexer(E);
1517 ensureIndexer(Indexer);
1518 try testing.expectEqual(E, Indexer.Key);1415 try testing.expectEqual(E, Indexer.Key);
1519 try testing.expectEqual(3, Indexer.count);1416 try testing.expectEqual(3, Indexer.count);
15201417
...@@ -1530,7 +1427,6 @@ test "EnumIndexer sparse" {...@@ -1530,7 +1427,6 @@ test "EnumIndexer sparse" {
1530test "EnumIndexer empty" {1427test "EnumIndexer empty" {
1531 const E = enum {};1428 const E = enum {};
1532 const Indexer = EnumIndexer(E);1429 const Indexer = EnumIndexer(E);
1533 ensureIndexer(Indexer);
1534 try testing.expectEqual(E, Indexer.Key);1430 try testing.expectEqual(E, Indexer.Key);
1535 try testing.expectEqual(0, Indexer.count);1431 try testing.expectEqual(0, Indexer.count);
1536}1432}
lib/std/os/linux.zig+289-305
...@@ -383,25 +383,24 @@ pub const O = switch (native_arch) {...@@ -383,25 +383,24 @@ pub const O = switch (native_arch) {
383 else => @compileError("missing std.os.linux.O constants for this architecture"),383 else => @compileError("missing std.os.linux.O constants for this architecture"),
384};384};
385385
386pub usingnamespace @import("linux/io_uring.zig");
387
388/// Set by startup code, used by `getauxval`.386/// Set by startup code, used by `getauxval`.
389pub var elf_aux_maybe: ?[*]std.elf.Auxv = null;387pub var elf_aux_maybe: ?[*]std.elf.Auxv = null;
390388
391pub usingnamespace if (switch (builtin.zig_backend) {389const extern_getauxval = switch (builtin.zig_backend) {
392 // Calling extern functions is not yet supported with these backends390 // Calling extern functions is not yet supported with these backends
393 .stage2_aarch64, .stage2_arm, .stage2_riscv64, .stage2_sparc64 => false,391 .stage2_aarch64, .stage2_arm, .stage2_riscv64, .stage2_sparc64 => false,
394 else => !builtin.link_libc,392 else => !builtin.link_libc,
395}) struct {393};
396 /// See `std.elf` for the constants.394
397 /// This matches the libc getauxval function.395comptime {
398 pub extern fn getauxval(index: usize) usize;396 if (extern_getauxval) {
399 comptime {
400 @export(getauxvalImpl, .{ .name = "getauxval", .linkage = .Weak });397 @export(getauxvalImpl, .{ .name = "getauxval", .linkage = .Weak });
401 }398 }
402} else struct {399}
403 pub const getauxval = getauxvalImpl;400
404};401pub const getauxval = if (extern_getauxval) struct {
402 extern fn getauxval(index: usize) usize;
403}.getauxval else getauxvalImpl;
405404
406fn getauxvalImpl(index: usize) callconv(.C) usize {405fn getauxvalImpl(index: usize) callconv(.C) usize {
407 const auxv = elf_aux_maybe orelse return 0;406 const auxv = elf_aux_maybe orelse return 0;
...@@ -2823,284 +2822,282 @@ pub const AF = struct {...@@ -2823,284 +2822,282 @@ pub const AF = struct {
2823 pub const MAX = PF.MAX;2822 pub const MAX = PF.MAX;
2824};2823};
28252824
2826pub const SO = struct {2825pub const SO = if (is_mips) struct {
2827 pub usingnamespace if (is_mips) struct {2826 pub const DEBUG = 1;
2828 pub const DEBUG = 1;2827 pub const REUSEADDR = 0x0004;
2829 pub const REUSEADDR = 0x0004;2828 pub const KEEPALIVE = 0x0008;
2830 pub const KEEPALIVE = 0x0008;2829 pub const DONTROUTE = 0x0010;
2831 pub const DONTROUTE = 0x0010;2830 pub const BROADCAST = 0x0020;
2832 pub const BROADCAST = 0x0020;2831 pub const LINGER = 0x0080;
2833 pub const LINGER = 0x0080;2832 pub const OOBINLINE = 0x0100;
2834 pub const OOBINLINE = 0x0100;2833 pub const REUSEPORT = 0x0200;
2835 pub const REUSEPORT = 0x0200;2834 pub const SNDBUF = 0x1001;
2836 pub const SNDBUF = 0x1001;2835 pub const RCVBUF = 0x1002;
2837 pub const RCVBUF = 0x1002;2836 pub const SNDLOWAT = 0x1003;
2838 pub const SNDLOWAT = 0x1003;2837 pub const RCVLOWAT = 0x1004;
2839 pub const RCVLOWAT = 0x1004;2838 pub const RCVTIMEO = 0x1006;
2840 pub const RCVTIMEO = 0x1006;2839 pub const SNDTIMEO = 0x1005;
2841 pub const SNDTIMEO = 0x1005;2840 pub const ERROR = 0x1007;
2842 pub const ERROR = 0x1007;2841 pub const TYPE = 0x1008;
2843 pub const TYPE = 0x1008;2842 pub const ACCEPTCONN = 0x1009;
2844 pub const ACCEPTCONN = 0x1009;2843 pub const PROTOCOL = 0x1028;
2845 pub const PROTOCOL = 0x1028;2844 pub const DOMAIN = 0x1029;
2846 pub const DOMAIN = 0x1029;2845 pub const NO_CHECK = 11;
2847 pub const NO_CHECK = 11;2846 pub const PRIORITY = 12;
2848 pub const PRIORITY = 12;2847 pub const BSDCOMPAT = 14;
2849 pub const BSDCOMPAT = 14;2848 pub const PASSCRED = 17;
2850 pub const PASSCRED = 17;2849 pub const PEERCRED = 18;
2851 pub const PEERCRED = 18;2850 pub const PEERSEC = 30;
2852 pub const PEERSEC = 30;2851 pub const SNDBUFFORCE = 31;
2853 pub const SNDBUFFORCE = 31;2852 pub const RCVBUFFORCE = 33;
2854 pub const RCVBUFFORCE = 33;2853 pub const SECURITY_AUTHENTICATION = 22;
2855 pub const SECURITY_AUTHENTICATION = 22;2854 pub const SECURITY_ENCRYPTION_TRANSPORT = 23;
2856 pub const SECURITY_ENCRYPTION_TRANSPORT = 23;2855 pub const SECURITY_ENCRYPTION_NETWORK = 24;
2857 pub const SECURITY_ENCRYPTION_NETWORK = 24;2856 pub const BINDTODEVICE = 25;
2858 pub const BINDTODEVICE = 25;2857 pub const ATTACH_FILTER = 26;
2859 pub const ATTACH_FILTER = 26;2858 pub const DETACH_FILTER = 27;
2860 pub const DETACH_FILTER = 27;2859 pub const GET_FILTER = ATTACH_FILTER;
2861 pub const GET_FILTER = ATTACH_FILTER;2860 pub const PEERNAME = 28;
2862 pub const PEERNAME = 28;2861 pub const TIMESTAMP_OLD = 29;
2863 pub const TIMESTAMP_OLD = 29;2862 pub const PASSSEC = 34;
2864 pub const PASSSEC = 34;2863 pub const TIMESTAMPNS_OLD = 35;
2865 pub const TIMESTAMPNS_OLD = 35;2864 pub const MARK = 36;
2866 pub const MARK = 36;2865 pub const TIMESTAMPING_OLD = 37;
2867 pub const TIMESTAMPING_OLD = 37;2866 pub const RXQ_OVFL = 40;
2868 pub const RXQ_OVFL = 40;2867 pub const WIFI_STATUS = 41;
2869 pub const WIFI_STATUS = 41;2868 pub const PEEK_OFF = 42;
2870 pub const PEEK_OFF = 42;2869 pub const NOFCS = 43;
2871 pub const NOFCS = 43;2870 pub const LOCK_FILTER = 44;
2872 pub const LOCK_FILTER = 44;2871 pub const SELECT_ERR_QUEUE = 45;
2873 pub const SELECT_ERR_QUEUE = 45;2872 pub const BUSY_POLL = 46;
2874 pub const BUSY_POLL = 46;2873 pub const MAX_PACING_RATE = 47;
2875 pub const MAX_PACING_RATE = 47;2874 pub const BPF_EXTENSIONS = 48;
2876 pub const BPF_EXTENSIONS = 48;2875 pub const INCOMING_CPU = 49;
2877 pub const INCOMING_CPU = 49;2876 pub const ATTACH_BPF = 50;
2878 pub const ATTACH_BPF = 50;2877 pub const DETACH_BPF = DETACH_FILTER;
2879 pub const DETACH_BPF = DETACH_FILTER;2878 pub const ATTACH_REUSEPORT_CBPF = 51;
2880 pub const ATTACH_REUSEPORT_CBPF = 51;2879 pub const ATTACH_REUSEPORT_EBPF = 52;
2881 pub const ATTACH_REUSEPORT_EBPF = 52;2880 pub const CNX_ADVICE = 53;
2882 pub const CNX_ADVICE = 53;2881 pub const MEMINFO = 55;
2883 pub const MEMINFO = 55;2882 pub const INCOMING_NAPI_ID = 56;
2884 pub const INCOMING_NAPI_ID = 56;2883 pub const COOKIE = 57;
2885 pub const COOKIE = 57;2884 pub const PEERGROUPS = 59;
2886 pub const PEERGROUPS = 59;2885 pub const ZEROCOPY = 60;
2887 pub const ZEROCOPY = 60;2886 pub const TXTIME = 61;
2888 pub const TXTIME = 61;2887 pub const BINDTOIFINDEX = 62;
2889 pub const BINDTOIFINDEX = 62;2888 pub const TIMESTAMP_NEW = 63;
2890 pub const TIMESTAMP_NEW = 63;2889 pub const TIMESTAMPNS_NEW = 64;
2891 pub const TIMESTAMPNS_NEW = 64;2890 pub const TIMESTAMPING_NEW = 65;
2892 pub const TIMESTAMPING_NEW = 65;2891 pub const RCVTIMEO_NEW = 66;
2893 pub const RCVTIMEO_NEW = 66;2892 pub const SNDTIMEO_NEW = 67;
2894 pub const SNDTIMEO_NEW = 67;2893 pub const DETACH_REUSEPORT_BPF = 68;
2895 pub const DETACH_REUSEPORT_BPF = 68;2894} else if (is_ppc or is_ppc64) struct {
2896 } else if (is_ppc or is_ppc64) struct {2895 pub const DEBUG = 1;
2897 pub const DEBUG = 1;2896 pub const REUSEADDR = 2;
2898 pub const REUSEADDR = 2;2897 pub const TYPE = 3;
2899 pub const TYPE = 3;2898 pub const ERROR = 4;
2900 pub const ERROR = 4;2899 pub const DONTROUTE = 5;
2901 pub const DONTROUTE = 5;2900 pub const BROADCAST = 6;
2902 pub const BROADCAST = 6;2901 pub const SNDBUF = 7;
2903 pub const SNDBUF = 7;2902 pub const RCVBUF = 8;
2904 pub const RCVBUF = 8;2903 pub const KEEPALIVE = 9;
2905 pub const KEEPALIVE = 9;2904 pub const OOBINLINE = 10;
2906 pub const OOBINLINE = 10;2905 pub const NO_CHECK = 11;
2907 pub const NO_CHECK = 11;2906 pub const PRIORITY = 12;
2908 pub const PRIORITY = 12;2907 pub const LINGER = 13;
2909 pub const LINGER = 13;2908 pub const BSDCOMPAT = 14;
2910 pub const BSDCOMPAT = 14;2909 pub const REUSEPORT = 15;
2911 pub const REUSEPORT = 15;2910 pub const RCVLOWAT = 16;
2912 pub const RCVLOWAT = 16;2911 pub const SNDLOWAT = 17;
2913 pub const SNDLOWAT = 17;2912 pub const RCVTIMEO = 18;
2914 pub const RCVTIMEO = 18;2913 pub const SNDTIMEO = 19;
2915 pub const SNDTIMEO = 19;2914 pub const PASSCRED = 20;
2916 pub const PASSCRED = 20;2915 pub const PEERCRED = 21;
2917 pub const PEERCRED = 21;2916 pub const ACCEPTCONN = 30;
2918 pub const ACCEPTCONN = 30;2917 pub const PEERSEC = 31;
2919 pub const PEERSEC = 31;2918 pub const SNDBUFFORCE = 32;
2920 pub const SNDBUFFORCE = 32;2919 pub const RCVBUFFORCE = 33;
2921 pub const RCVBUFFORCE = 33;2920 pub const PROTOCOL = 38;
2922 pub const PROTOCOL = 38;2921 pub const DOMAIN = 39;
2923 pub const DOMAIN = 39;2922 pub const SECURITY_AUTHENTICATION = 22;
2924 pub const SECURITY_AUTHENTICATION = 22;2923 pub const SECURITY_ENCRYPTION_TRANSPORT = 23;
2925 pub const SECURITY_ENCRYPTION_TRANSPORT = 23;2924 pub const SECURITY_ENCRYPTION_NETWORK = 24;
2926 pub const SECURITY_ENCRYPTION_NETWORK = 24;2925 pub const BINDTODEVICE = 25;
2927 pub const BINDTODEVICE = 25;2926 pub const ATTACH_FILTER = 26;
2928 pub const ATTACH_FILTER = 26;2927 pub const DETACH_FILTER = 27;
2929 pub const DETACH_FILTER = 27;2928 pub const GET_FILTER = ATTACH_FILTER;
2930 pub const GET_FILTER = ATTACH_FILTER;2929 pub const PEERNAME = 28;
2931 pub const PEERNAME = 28;2930 pub const TIMESTAMP_OLD = 29;
2932 pub const TIMESTAMP_OLD = 29;2931 pub const PASSSEC = 34;
2933 pub const PASSSEC = 34;2932 pub const TIMESTAMPNS_OLD = 35;
2934 pub const TIMESTAMPNS_OLD = 35;2933 pub const MARK = 36;
2935 pub const MARK = 36;2934 pub const TIMESTAMPING_OLD = 37;
2936 pub const TIMESTAMPING_OLD = 37;2935 pub const RXQ_OVFL = 40;
2937 pub const RXQ_OVFL = 40;2936 pub const WIFI_STATUS = 41;
2938 pub const WIFI_STATUS = 41;2937 pub const PEEK_OFF = 42;
2939 pub const PEEK_OFF = 42;2938 pub const NOFCS = 43;
2940 pub const NOFCS = 43;2939 pub const LOCK_FILTER = 44;
2941 pub const LOCK_FILTER = 44;2940 pub const SELECT_ERR_QUEUE = 45;
2942 pub const SELECT_ERR_QUEUE = 45;2941 pub const BUSY_POLL = 46;
2943 pub const BUSY_POLL = 46;2942 pub const MAX_PACING_RATE = 47;
2944 pub const MAX_PACING_RATE = 47;2943 pub const BPF_EXTENSIONS = 48;
2945 pub const BPF_EXTENSIONS = 48;2944 pub const INCOMING_CPU = 49;
2946 pub const INCOMING_CPU = 49;2945 pub const ATTACH_BPF = 50;
2947 pub const ATTACH_BPF = 50;2946 pub const DETACH_BPF = DETACH_FILTER;
2948 pub const DETACH_BPF = DETACH_FILTER;2947 pub const ATTACH_REUSEPORT_CBPF = 51;
2949 pub const ATTACH_REUSEPORT_CBPF = 51;2948 pub const ATTACH_REUSEPORT_EBPF = 52;
2950 pub const ATTACH_REUSEPORT_EBPF = 52;2949 pub const CNX_ADVICE = 53;
2951 pub const CNX_ADVICE = 53;2950 pub const MEMINFO = 55;
2952 pub const MEMINFO = 55;2951 pub const INCOMING_NAPI_ID = 56;
2953 pub const INCOMING_NAPI_ID = 56;2952 pub const COOKIE = 57;
2954 pub const COOKIE = 57;2953 pub const PEERGROUPS = 59;
2955 pub const PEERGROUPS = 59;2954 pub const ZEROCOPY = 60;
2956 pub const ZEROCOPY = 60;2955 pub const TXTIME = 61;
2957 pub const TXTIME = 61;2956 pub const BINDTOIFINDEX = 62;
2958 pub const BINDTOIFINDEX = 62;2957 pub const TIMESTAMP_NEW = 63;
2959 pub const TIMESTAMP_NEW = 63;2958 pub const TIMESTAMPNS_NEW = 64;
2960 pub const TIMESTAMPNS_NEW = 64;2959 pub const TIMESTAMPING_NEW = 65;
2961 pub const TIMESTAMPING_NEW = 65;2960 pub const RCVTIMEO_NEW = 66;
2962 pub const RCVTIMEO_NEW = 66;2961 pub const SNDTIMEO_NEW = 67;
2963 pub const SNDTIMEO_NEW = 67;2962 pub const DETACH_REUSEPORT_BPF = 68;
2964 pub const DETACH_REUSEPORT_BPF = 68;2963} else if (is_sparc) struct {
2965 } else if (is_sparc) struct {2964 pub const DEBUG = 1;
2966 pub const DEBUG = 1;2965 pub const REUSEADDR = 4;
2967 pub const REUSEADDR = 4;2966 pub const TYPE = 4104;
2968 pub const TYPE = 4104;2967 pub const ERROR = 4103;
2969 pub const ERROR = 4103;2968 pub const DONTROUTE = 16;
2970 pub const DONTROUTE = 16;2969 pub const BROADCAST = 32;
2971 pub const BROADCAST = 32;2970 pub const SNDBUF = 4097;
2972 pub const SNDBUF = 4097;2971 pub const RCVBUF = 4098;
2973 pub const RCVBUF = 4098;2972 pub const KEEPALIVE = 8;
2974 pub const KEEPALIVE = 8;2973 pub const OOBINLINE = 256;
2975 pub const OOBINLINE = 256;2974 pub const NO_CHECK = 11;
2976 pub const NO_CHECK = 11;2975 pub const PRIORITY = 12;
2977 pub const PRIORITY = 12;2976 pub const LINGER = 128;
2978 pub const LINGER = 128;2977 pub const BSDCOMPAT = 1024;
2979 pub const BSDCOMPAT = 1024;2978 pub const REUSEPORT = 512;
2980 pub const REUSEPORT = 512;2979 pub const PASSCRED = 2;
2981 pub const PASSCRED = 2;2980 pub const PEERCRED = 64;
2982 pub const PEERCRED = 64;2981 pub const RCVLOWAT = 2048;
2983 pub const RCVLOWAT = 2048;2982 pub const SNDLOWAT = 4096;
2984 pub const SNDLOWAT = 4096;2983 pub const RCVTIMEO = 8192;
2985 pub const RCVTIMEO = 8192;2984 pub const SNDTIMEO = 16384;
2986 pub const SNDTIMEO = 16384;2985 pub const ACCEPTCONN = 32768;
2987 pub const ACCEPTCONN = 32768;2986 pub const PEERSEC = 30;
2988 pub const PEERSEC = 30;2987 pub const SNDBUFFORCE = 4106;
2989 pub const SNDBUFFORCE = 4106;2988 pub const RCVBUFFORCE = 4107;
2990 pub const RCVBUFFORCE = 4107;2989 pub const PROTOCOL = 4136;
2991 pub const PROTOCOL = 4136;2990 pub const DOMAIN = 4137;
2992 pub const DOMAIN = 4137;2991 pub const SECURITY_AUTHENTICATION = 20481;
2993 pub const SECURITY_AUTHENTICATION = 20481;2992 pub const SECURITY_ENCRYPTION_TRANSPORT = 20482;
2994 pub const SECURITY_ENCRYPTION_TRANSPORT = 20482;2993 pub const SECURITY_ENCRYPTION_NETWORK = 20484;
2995 pub const SECURITY_ENCRYPTION_NETWORK = 20484;2994 pub const BINDTODEVICE = 13;
2996 pub const BINDTODEVICE = 13;2995 pub const ATTACH_FILTER = 26;
2997 pub const ATTACH_FILTER = 26;2996 pub const DETACH_FILTER = 27;
2998 pub const DETACH_FILTER = 27;2997 pub const GET_FILTER = 26;
2999 pub const GET_FILTER = 26;2998 pub const PEERNAME = 28;
3000 pub const PEERNAME = 28;2999 pub const TIMESTAMP_OLD = 29;
3001 pub const TIMESTAMP_OLD = 29;3000 pub const PASSSEC = 31;
3002 pub const PASSSEC = 31;3001 pub const TIMESTAMPNS_OLD = 33;
3003 pub const TIMESTAMPNS_OLD = 33;3002 pub const MARK = 34;
3004 pub const MARK = 34;3003 pub const TIMESTAMPING_OLD = 35;
3005 pub const TIMESTAMPING_OLD = 35;3004 pub const RXQ_OVFL = 36;
3006 pub const RXQ_OVFL = 36;3005 pub const WIFI_STATUS = 37;
3007 pub const WIFI_STATUS = 37;3006 pub const PEEK_OFF = 38;
3008 pub const PEEK_OFF = 38;3007 pub const NOFCS = 39;
3009 pub const NOFCS = 39;3008 pub const LOCK_FILTER = 40;
3010 pub const LOCK_FILTER = 40;3009 pub const SELECT_ERR_QUEUE = 41;
3011 pub const SELECT_ERR_QUEUE = 41;3010 pub const BUSY_POLL = 48;
3012 pub const BUSY_POLL = 48;3011 pub const MAX_PACING_RATE = 49;
3013 pub const MAX_PACING_RATE = 49;3012 pub const BPF_EXTENSIONS = 50;
3014 pub const BPF_EXTENSIONS = 50;3013 pub const INCOMING_CPU = 51;
3015 pub const INCOMING_CPU = 51;3014 pub const ATTACH_BPF = 52;
3016 pub const ATTACH_BPF = 52;3015 pub const DETACH_BPF = 27;
3017 pub const DETACH_BPF = 27;3016 pub const ATTACH_REUSEPORT_CBPF = 53;
3018 pub const ATTACH_REUSEPORT_CBPF = 53;3017 pub const ATTACH_REUSEPORT_EBPF = 54;
3019 pub const ATTACH_REUSEPORT_EBPF = 54;3018 pub const CNX_ADVICE = 55;
3020 pub const CNX_ADVICE = 55;3019 pub const MEMINFO = 57;
3021 pub const MEMINFO = 57;3020 pub const INCOMING_NAPI_ID = 58;
3022 pub const INCOMING_NAPI_ID = 58;3021 pub const COOKIE = 59;
3023 pub const COOKIE = 59;3022 pub const PEERGROUPS = 61;
3024 pub const PEERGROUPS = 61;3023 pub const ZEROCOPY = 62;
3025 pub const ZEROCOPY = 62;3024 pub const TXTIME = 63;
3026 pub const TXTIME = 63;3025 pub const BINDTOIFINDEX = 65;
3027 pub const BINDTOIFINDEX = 65;3026 pub const TIMESTAMP_NEW = 70;
3028 pub const TIMESTAMP_NEW = 70;3027 pub const TIMESTAMPNS_NEW = 66;
3029 pub const TIMESTAMPNS_NEW = 66;3028 pub const TIMESTAMPING_NEW = 67;
3030 pub const TIMESTAMPING_NEW = 67;3029 pub const RCVTIMEO_NEW = 68;
3031 pub const RCVTIMEO_NEW = 68;3030 pub const SNDTIMEO_NEW = 69;
3032 pub const SNDTIMEO_NEW = 69;3031 pub const DETACH_REUSEPORT_BPF = 71;
3033 pub const DETACH_REUSEPORT_BPF = 71;3032} else struct {
3034 } else struct {3033 pub const DEBUG = 1;
3035 pub const DEBUG = 1;3034 pub const REUSEADDR = 2;
3036 pub const REUSEADDR = 2;3035 pub const TYPE = 3;
3037 pub const TYPE = 3;3036 pub const ERROR = 4;
3038 pub const ERROR = 4;3037 pub const DONTROUTE = 5;
3039 pub const DONTROUTE = 5;3038 pub const BROADCAST = 6;
3040 pub const BROADCAST = 6;3039 pub const SNDBUF = 7;
3041 pub const SNDBUF = 7;3040 pub const RCVBUF = 8;
3042 pub const RCVBUF = 8;3041 pub const KEEPALIVE = 9;
3043 pub const KEEPALIVE = 9;3042 pub const OOBINLINE = 10;
3044 pub const OOBINLINE = 10;3043 pub const NO_CHECK = 11;
3045 pub const NO_CHECK = 11;3044 pub const PRIORITY = 12;
3046 pub const PRIORITY = 12;3045 pub const LINGER = 13;
3047 pub const LINGER = 13;3046 pub const BSDCOMPAT = 14;
3048 pub const BSDCOMPAT = 14;3047 pub const REUSEPORT = 15;
3049 pub const REUSEPORT = 15;3048 pub const PASSCRED = 16;
3050 pub const PASSCRED = 16;3049 pub const PEERCRED = 17;
3051 pub const PEERCRED = 17;3050 pub const RCVLOWAT = 18;
3052 pub const RCVLOWAT = 18;3051 pub const SNDLOWAT = 19;
3053 pub const SNDLOWAT = 19;3052 pub const RCVTIMEO = 20;
3054 pub const RCVTIMEO = 20;3053 pub const SNDTIMEO = 21;
3055 pub const SNDTIMEO = 21;3054 pub const ACCEPTCONN = 30;
3056 pub const ACCEPTCONN = 30;3055 pub const PEERSEC = 31;
3057 pub const PEERSEC = 31;3056 pub const SNDBUFFORCE = 32;
3058 pub const SNDBUFFORCE = 32;3057 pub const RCVBUFFORCE = 33;
3059 pub const RCVBUFFORCE = 33;3058 pub const PROTOCOL = 38;
3060 pub const PROTOCOL = 38;3059 pub const DOMAIN = 39;
3061 pub const DOMAIN = 39;3060 pub const SECURITY_AUTHENTICATION = 22;
3062 pub const SECURITY_AUTHENTICATION = 22;3061 pub const SECURITY_ENCRYPTION_TRANSPORT = 23;
3063 pub const SECURITY_ENCRYPTION_TRANSPORT = 23;3062 pub const SECURITY_ENCRYPTION_NETWORK = 24;
3064 pub const SECURITY_ENCRYPTION_NETWORK = 24;3063 pub const BINDTODEVICE = 25;
3065 pub const BINDTODEVICE = 25;3064 pub const ATTACH_FILTER = 26;
3066 pub const ATTACH_FILTER = 26;3065 pub const DETACH_FILTER = 27;
3067 pub const DETACH_FILTER = 27;3066 pub const GET_FILTER = ATTACH_FILTER;
3068 pub const GET_FILTER = ATTACH_FILTER;3067 pub const PEERNAME = 28;
3069 pub const PEERNAME = 28;3068 pub const TIMESTAMP_OLD = 29;
3070 pub const TIMESTAMP_OLD = 29;3069 pub const PASSSEC = 34;
3071 pub const PASSSEC = 34;3070 pub const TIMESTAMPNS_OLD = 35;
3072 pub const TIMESTAMPNS_OLD = 35;3071 pub const MARK = 36;
3073 pub const MARK = 36;3072 pub const TIMESTAMPING_OLD = 37;
3074 pub const TIMESTAMPING_OLD = 37;3073 pub const RXQ_OVFL = 40;
3075 pub const RXQ_OVFL = 40;3074 pub const WIFI_STATUS = 41;
3076 pub const WIFI_STATUS = 41;3075 pub const PEEK_OFF = 42;
3077 pub const PEEK_OFF = 42;3076 pub const NOFCS = 43;
3078 pub const NOFCS = 43;3077 pub const LOCK_FILTER = 44;
3079 pub const LOCK_FILTER = 44;3078 pub const SELECT_ERR_QUEUE = 45;
3080 pub const SELECT_ERR_QUEUE = 45;3079 pub const BUSY_POLL = 46;
3081 pub const BUSY_POLL = 46;3080 pub const MAX_PACING_RATE = 47;
3082 pub const MAX_PACING_RATE = 47;3081 pub const BPF_EXTENSIONS = 48;
3083 pub const BPF_EXTENSIONS = 48;3082 pub const INCOMING_CPU = 49;
3084 pub const INCOMING_CPU = 49;3083 pub const ATTACH_BPF = 50;
3085 pub const ATTACH_BPF = 50;3084 pub const DETACH_BPF = DETACH_FILTER;
3086 pub const DETACH_BPF = DETACH_FILTER;3085 pub const ATTACH_REUSEPORT_CBPF = 51;
3087 pub const ATTACH_REUSEPORT_CBPF = 51;3086 pub const ATTACH_REUSEPORT_EBPF = 52;
3088 pub const ATTACH_REUSEPORT_EBPF = 52;3087 pub const CNX_ADVICE = 53;
3089 pub const CNX_ADVICE = 53;3088 pub const MEMINFO = 55;
3090 pub const MEMINFO = 55;3089 pub const INCOMING_NAPI_ID = 56;
3091 pub const INCOMING_NAPI_ID = 56;3090 pub const COOKIE = 57;
3092 pub const COOKIE = 57;3091 pub const PEERGROUPS = 59;
3093 pub const PEERGROUPS = 59;3092 pub const ZEROCOPY = 60;
3094 pub const ZEROCOPY = 60;3093 pub const TXTIME = 61;
3095 pub const TXTIME = 61;3094 pub const BINDTOIFINDEX = 62;
3096 pub const BINDTOIFINDEX = 62;3095 pub const TIMESTAMP_NEW = 63;
3097 pub const TIMESTAMP_NEW = 63;3096 pub const TIMESTAMPNS_NEW = 64;
3098 pub const TIMESTAMPNS_NEW = 64;3097 pub const TIMESTAMPING_NEW = 65;
3099 pub const TIMESTAMPING_NEW = 65;3098 pub const RCVTIMEO_NEW = 66;
3100 pub const RCVTIMEO_NEW = 66;3099 pub const SNDTIMEO_NEW = 67;
3101 pub const SNDTIMEO_NEW = 67;3100 pub const DETACH_REUSEPORT_BPF = 68;
3102 pub const DETACH_REUSEPORT_BPF = 68;
3103 };
3104};3101};
31053102
3106pub const SCM = struct {3103pub const SCM = struct {
...@@ -4189,22 +4186,9 @@ pub const IORING_SETUP_SINGLE_ISSUER = 1 << 12;...@@ -4189,22 +4186,9 @@ pub const IORING_SETUP_SINGLE_ISSUER = 1 << 12;
4189pub const IORING_SETUP_DEFER_TASKRUN = 1 << 13;4186pub const IORING_SETUP_DEFER_TASKRUN = 1 << 13;
41904187
4191/// IO submission data structure (Submission Queue Entry)4188/// IO submission data structure (Submission Queue Entry)
4192pub const io_uring_sqe = extern struct {4189pub const io_uring_sqe = @import("linux/io_uring_sqe.zig").io_uring_sqe;
4193 opcode: IORING_OP,4190
4194 flags: u8,4191pub const IoUring = @import("linux/IoUring.zig");
4195 ioprio: u16,
4196 fd: i32,
4197 off: u64,
4198 addr: u64,
4199 len: u32,
4200 rw_flags: u32,
4201 user_data: u64,
4202 buf_index: u16,
4203 personality: u16,
4204 splice_fd_in: i32,
4205 addr3: u64,
4206 resv: u64,
4207};
42084192
4209/// If sqe->file_index is set to this for opcodes that instantiate a new4193/// If sqe->file_index is set to this for opcodes that instantiate a new
4210/// direct descriptor (like openat/openat2/accept), then io_uring will allocate4194/// direct descriptor (like openat/openat2/accept), then io_uring will allocate
lib/std/os/linux/IoUring.zig created+3670
...@@ -0,0 +1,3670 @@
1const IoUring = @This();
2const std = @import("../../std.zig");
3const builtin = @import("builtin");
4const assert = std.debug.assert;
5const mem = std.mem;
6const net = std.net;
7const os = std.os;
8const posix = std.posix;
9const linux = os.linux;
10const testing = std.testing;
11
12fd: os.fd_t = -1,
13sq: SubmissionQueue,
14cq: CompletionQueue,
15flags: u32,
16features: u32,
17
18/// A friendly way to setup an io_uring, with default linux.io_uring_params.
19/// `entries` must be a power of two between 1 and 32768, although the kernel will make the final
20/// call on how many entries the submission and completion queues will ultimately have,
21/// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
22/// Matches the interface of io_uring_queue_init() in liburing.
23pub fn init(entries: u16, flags: u32) !IoUring {
24 var params = mem.zeroInit(linux.io_uring_params, .{
25 .flags = flags,
26 .sq_thread_idle = 1000,
27 });
28 return try IoUring.init_params(entries, &params);
29}
30
31/// A powerful way to setup an io_uring, if you want to tweak linux.io_uring_params such as submission
32/// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
33/// `params` is passed by reference because the kernel needs to modify the parameters.
34/// Matches the interface of io_uring_queue_init_params() in liburing.
35pub fn init_params(entries: u16, p: *linux.io_uring_params) !IoUring {
36 if (entries == 0) return error.EntriesZero;
37 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
38
39 assert(p.sq_entries == 0);
40 assert(p.cq_entries == 0 or p.flags & linux.IORING_SETUP_CQSIZE != 0);
41 assert(p.features == 0);
42 assert(p.wq_fd == 0 or p.flags & linux.IORING_SETUP_ATTACH_WQ != 0);
43 assert(p.resv[0] == 0);
44 assert(p.resv[1] == 0);
45 assert(p.resv[2] == 0);
46
47 const res = linux.io_uring_setup(entries, p);
48 switch (linux.getErrno(res)) {
49 .SUCCESS => {},
50 .FAULT => return error.ParamsOutsideAccessibleAddressSpace,
51 // The resv array contains non-zero data, p.flags contains an unsupported flag,
52 // entries out of bounds, IORING_SETUP_SQ_AFF was specified without IORING_SETUP_SQPOLL,
53 // or IORING_SETUP_CQSIZE was specified but linux.io_uring_params.cq_entries was invalid:
54 .INVAL => return error.ArgumentsInvalid,
55 .MFILE => return error.ProcessFdQuotaExceeded,
56 .NFILE => return error.SystemFdQuotaExceeded,
57 .NOMEM => return error.SystemResources,
58 // IORING_SETUP_SQPOLL was specified but effective user ID lacks sufficient privileges,
59 // or a container seccomp policy prohibits io_uring syscalls:
60 .PERM => return error.PermissionDenied,
61 .NOSYS => return error.SystemOutdated,
62 else => |errno| return os.unexpectedErrno(errno),
63 }
64 const fd = @as(os.fd_t, @intCast(res));
65 assert(fd >= 0);
66 errdefer os.close(fd);
67
68 // Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
69 // This is not an optional feature for us... if the kernel does it, we have to do it.
70 // The thinking on this by the kernel developers was that both the submission and the
71 // completion queue rings have sizes just over a power of two, but the submission queue ring
72 // is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
73 // gets the submission queue ring for free.
74 // See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
75 // We do not support the double mmap() done before 5.4, because we want to keep the
76 // init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
77 if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
78 return error.SystemOutdated;
79 }
80
81 // Check that the kernel has actually set params and that "impossible is nothing".
82 assert(p.sq_entries != 0);
83 assert(p.cq_entries != 0);
84 assert(p.cq_entries >= p.sq_entries);
85
86 // From here on, we only need to read from params, so pass `p` by value as immutable.
87 // The completion queue shares the mmap with the submission queue, so pass `sq` there too.
88 var sq = try SubmissionQueue.init(fd, p.*);
89 errdefer sq.deinit();
90 var cq = try CompletionQueue.init(fd, p.*, sq);
91 errdefer cq.deinit();
92
93 // Check that our starting state is as we expect.
94 assert(sq.head.* == 0);
95 assert(sq.tail.* == 0);
96 assert(sq.mask == p.sq_entries - 1);
97 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
98 assert(sq.dropped.* == 0);
99 assert(sq.array.len == p.sq_entries);
100 assert(sq.sqes.len == p.sq_entries);
101 assert(sq.sqe_head == 0);
102 assert(sq.sqe_tail == 0);
103
104 assert(cq.head.* == 0);
105 assert(cq.tail.* == 0);
106 assert(cq.mask == p.cq_entries - 1);
107 assert(cq.overflow.* == 0);
108 assert(cq.cqes.len == p.cq_entries);
109
110 return IoUring{
111 .fd = fd,
112 .sq = sq,
113 .cq = cq,
114 .flags = p.flags,
115 .features = p.features,
116 };
117}
118
119pub fn deinit(self: *IoUring) void {
120 assert(self.fd >= 0);
121 // The mmaps depend on the fd, so the order of these calls is important:
122 self.cq.deinit();
123 self.sq.deinit();
124 os.close(self.fd);
125 self.fd = -1;
126}
127
128/// Returns a pointer to a vacant SQE, or an error if the submission queue is full.
129/// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
130/// However, instead of a null we return an error to force safe handling.
131/// Any situation where the submission queue is full tends more towards a control flow error,
132/// and the null return in liburing is more a C idiom than anything else, for lack of a better
133/// alternative. In Zig, we have first-class error handling... so let's use it.
134/// Matches the implementation of io_uring_get_sqe() in liburing.
135pub fn get_sqe(self: *IoUring) !*linux.io_uring_sqe {
136 const head = @atomicLoad(u32, self.sq.head, .Acquire);
137 // Remember that these head and tail offsets wrap around every four billion operations.
138 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.
139 const next = self.sq.sqe_tail +% 1;
140 if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
141 const sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
142 self.sq.sqe_tail = next;
143 return sqe;
144}
145
146/// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
147/// called get_sqe() multiple times to setup multiple I/O requests.
148/// Returns the number of SQEs submitted, if not used alongside IORING_SETUP_SQPOLL.
149/// If the io_uring instance is uses IORING_SETUP_SQPOLL, the value returned on success is not
150/// guaranteed to match the amount of actually submitted sqes during this call. A value higher
151/// or lower, including 0, may be returned.
152/// Matches the implementation of io_uring_submit() in liburing.
153pub fn submit(self: *IoUring) !u32 {
154 return self.submit_and_wait(0);
155}
156
157/// Like submit(), but allows waiting for events as well.
158/// Returns the number of SQEs submitted.
159/// Matches the implementation of io_uring_submit_and_wait() in liburing.
160pub fn submit_and_wait(self: *IoUring, wait_nr: u32) !u32 {
161 const submitted = self.flush_sq();
162 var flags: u32 = 0;
163 if (self.sq_ring_needs_enter(&flags) or wait_nr > 0) {
164 if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
165 flags |= linux.IORING_ENTER_GETEVENTS;
166 }
167 return try self.enter(submitted, wait_nr, flags);
168 }
169 return submitted;
170}
171
172/// Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
173/// Returns the number of SQEs submitted.
174pub fn enter(self: *IoUring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
175 assert(self.fd >= 0);
176 const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
177 switch (linux.getErrno(res)) {
178 .SUCCESS => {},
179 // The kernel was unable to allocate memory or ran out of resources for the request.
180 // The application should wait for some completions and try again:
181 .AGAIN => return error.SystemResources,
182 // The SQE `fd` is invalid, or IOSQE_FIXED_FILE was set but no files were registered:
183 .BADF => return error.FileDescriptorInvalid,
184 // The file descriptor is valid, but the ring is not in the right state.
185 // See io_uring_register(2) for how to enable the ring.
186 .BADFD => return error.FileDescriptorInBadState,
187 // The application attempted to overcommit the number of requests it can have pending.
188 // The application should wait for some completions and try again:
189 .BUSY => return error.CompletionQueueOvercommitted,
190 // The SQE is invalid, or valid but the ring was setup with IORING_SETUP_IOPOLL:
191 .INVAL => return error.SubmissionQueueEntryInvalid,
192 // The buffer is outside the process' accessible address space, or IORING_OP_READ_FIXED
193 // or IORING_OP_WRITE_FIXED was specified but no buffers were registered, or the range
194 // described by `addr` and `len` is not within the buffer registered at `buf_index`:
195 .FAULT => return error.BufferInvalid,
196 .NXIO => return error.RingShuttingDown,
197 // The kernel believes our `self.fd` does not refer to an io_uring instance,
198 // or the opcode is valid but not supported by this kernel (more likely):
199 .OPNOTSUPP => return error.OpcodeNotSupported,
200 // The operation was interrupted by a delivery of a signal before it could complete.
201 // This can happen while waiting for events with IORING_ENTER_GETEVENTS:
202 .INTR => return error.SignalInterrupt,
203 else => |errno| return os.unexpectedErrno(errno),
204 }
205 return @as(u32, @intCast(res));
206}
207
208/// Sync internal state with kernel ring state on the SQ side.
209/// Returns the number of all pending events in the SQ ring, for the shared ring.
210/// This return value includes previously flushed SQEs, as per liburing.
211/// The rationale is to suggest that an io_uring_enter() call is needed rather than not.
212/// Matches the implementation of __io_uring_flush_sq() in liburing.
213pub fn flush_sq(self: *IoUring) u32 {
214 if (self.sq.sqe_head != self.sq.sqe_tail) {
215 // Fill in SQEs that we have queued up, adding them to the kernel ring.
216 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
217 var tail = self.sq.tail.*;
218 var i: usize = 0;
219 while (i < to_submit) : (i += 1) {
220 self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
221 tail +%= 1;
222 self.sq.sqe_head +%= 1;
223 }
224 // Ensure that the kernel can actually see the SQE updates when it sees the tail update.
225 @atomicStore(u32, self.sq.tail, tail, .Release);
226 }
227 return self.sq_ready();
228}
229
230/// Returns true if we are not using an SQ thread (thus nobody submits but us),
231/// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
232/// For the latter case, we set the SQ thread wakeup flag.
233/// Matches the implementation of sq_ring_needs_enter() in liburing.
234pub fn sq_ring_needs_enter(self: *IoUring, flags: *u32) bool {
235 assert(flags.* == 0);
236 if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0) return true;
237 if ((@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
238 flags.* |= linux.IORING_ENTER_SQ_WAKEUP;
239 return true;
240 }
241 return false;
242}
243
244/// Returns the number of flushed and unflushed SQEs pending in the submission queue.
245/// In other words, this is the number of SQEs in the submission queue, i.e. its length.
246/// These are SQEs that the kernel is yet to consume.
247/// Matches the implementation of io_uring_sq_ready in liburing.
248pub fn sq_ready(self: *IoUring) u32 {
249 // Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
250 // see https://github.com/axboe/liburing/issues/92.
251 return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .Acquire);
252}
253
254/// Returns the number of CQEs in the completion queue, i.e. its length.
255/// These are CQEs that the application is yet to consume.
256/// Matches the implementation of io_uring_cq_ready in liburing.
257pub fn cq_ready(self: *IoUring) u32 {
258 return @atomicLoad(u32, self.cq.tail, .Acquire) -% self.cq.head.*;
259}
260
261/// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
262/// If none are available, enters into the kernel to wait for at most `wait_nr` CQEs.
263/// Returns the number of CQEs copied, advancing the CQ ring.
264/// Provides all the wait/peek methods found in liburing, but with batching and a single method.
265/// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
266/// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
267/// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
268/// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
269/// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
270/// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
271pub fn copy_cqes(self: *IoUring, cqes: []linux.io_uring_cqe, wait_nr: u32) !u32 {
272 const count = self.copy_cqes_ready(cqes);
273 if (count > 0) return count;
274 if (self.cq_ring_needs_flush() or wait_nr > 0) {
275 _ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
276 return self.copy_cqes_ready(cqes);
277 }
278 return 0;
279}
280
281fn copy_cqes_ready(self: *IoUring, cqes: []linux.io_uring_cqe) u32 {
282 const ready = self.cq_ready();
283 const count = @min(cqes.len, ready);
284 const head = self.cq.head.* & self.cq.mask;
285 const tail = (self.cq.head.* +% count) & self.cq.mask;
286
287 if (head <= tail) {
288 // head behind tail -> no wrapping
289 @memcpy(cqes[0..count], self.cq.cqes[head..tail]);
290 } else {
291 // head in front of tail -> buffer wraps
292 const two_copies_required: bool = self.cq.cqes.len - head < count;
293 const amount_to_copy_in_first = if (two_copies_required) self.cq.cqes.len - head else count;
294 @memcpy(cqes[0..amount_to_copy_in_first], self.cq.cqes[head .. head + amount_to_copy_in_first]);
295 if (two_copies_required) {
296 @memcpy(cqes[amount_to_copy_in_first..count], self.cq.cqes[0..tail]);
297 }
298 }
299
300 self.cq_advance(count);
301 return count;
302}
303
304/// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
305/// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
306pub fn copy_cqe(ring: *IoUring) !linux.io_uring_cqe {
307 var cqes: [1]linux.io_uring_cqe = undefined;
308 while (true) {
309 const count = try ring.copy_cqes(&cqes, 1);
310 if (count > 0) return cqes[0];
311 }
312}
313
314/// Matches the implementation of cq_ring_needs_flush() in liburing.
315pub fn cq_ring_needs_flush(self: *IoUring) bool {
316 return (@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_CQ_OVERFLOW) != 0;
317}
318
319/// For advanced use cases only that implement custom completion queue methods.
320/// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
321/// Must be called exactly once after a zero-copy CQE has been processed by your application.
322/// Not idempotent, calling more than once will result in other CQEs being lost.
323/// Matches the implementation of cqe_seen() in liburing.
324pub fn cqe_seen(self: *IoUring, cqe: *linux.io_uring_cqe) void {
325 _ = cqe;
326 self.cq_advance(1);
327}
328
329/// For advanced use cases only that implement custom completion queue methods.
330/// Matches the implementation of cq_advance() in liburing.
331pub fn cq_advance(self: *IoUring, count: u32) void {
332 if (count > 0) {
333 // Ensure the kernel only sees the new head value after the CQEs have been read.
334 @atomicStore(u32, self.cq.head, self.cq.head.* +% count, .Release);
335 }
336}
337
338/// Queues (but does not submit) an SQE to perform an `fsync(2)`.
339/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
340/// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `rw_flags`.
341/// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
342/// operations execute in parallel and completions are unordered. Therefore, an application that
343/// submits a write followed by an fsync in the submission queue cannot expect the fsync to
344/// apply to the write, since the fsync may complete before the write is issued to the disk.
345/// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
346/// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
347pub fn fsync(self: *IoUring, user_data: u64, fd: os.fd_t, flags: u32) !*linux.io_uring_sqe {
348 const sqe = try self.get_sqe();
349 sqe.prep_fsync(fd, flags);
350 sqe.user_data = user_data;
351 return sqe;
352}
353
354/// Queues (but does not submit) an SQE to perform a no-op.
355/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
356/// A no-op is more useful than may appear at first glance.
357/// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
358/// know when the ring is idle before acting on a kill signal.
359pub fn nop(self: *IoUring, user_data: u64) !*linux.io_uring_sqe {
360 const sqe = try self.get_sqe();
361 sqe.prep_nop();
362 sqe.user_data = user_data;
363 return sqe;
364}
365
366/// Used to select how the read should be handled.
367pub const ReadBuffer = union(enum) {
368 /// io_uring will read directly into this buffer
369 buffer: []u8,
370
371 /// io_uring will read directly into these buffers using readv.
372 iovecs: []const os.iovec,
373
374 /// io_uring will select a buffer that has previously been provided with `provide_buffers`.
375 /// The buffer group reference by `group_id` must contain at least one buffer for the read to work.
376 /// `len` controls the number of bytes to read into the selected buffer.
377 buffer_selection: struct {
378 group_id: u16,
379 len: usize,
380 },
381};
382
383/// Queues (but does not submit) an SQE to perform a `read(2)` or `preadv(2)` depending on the buffer type.
384/// * Reading into a `ReadBuffer.buffer` uses `read(2)`
385/// * Reading into a `ReadBuffer.iovecs` uses `preadv(2)`
386/// If you want to do a `preadv2(2)` then set `rw_flags` on the returned SQE. See https://man7.org/linux/man-pages/man2/preadv2.2.html
387///
388/// Returns a pointer to the SQE.
389pub fn read(
390 self: *IoUring,
391 user_data: u64,
392 fd: os.fd_t,
393 buffer: ReadBuffer,
394 offset: u64,
395) !*linux.io_uring_sqe {
396 const sqe = try self.get_sqe();
397 switch (buffer) {
398 .buffer => |slice| sqe.prep_read(fd, slice, offset),
399 .iovecs => |vecs| sqe.prep_readv(fd, vecs, offset),
400 .buffer_selection => |selection| {
401 sqe.prep_rw(.READ, fd, 0, selection.len, offset);
402 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
403 sqe.buf_index = selection.group_id;
404 },
405 }
406 sqe.user_data = user_data;
407 return sqe;
408}
409
410/// Queues (but does not submit) an SQE to perform a `write(2)`.
411/// Returns a pointer to the SQE.
412pub fn write(
413 self: *IoUring,
414 user_data: u64,
415 fd: os.fd_t,
416 buffer: []const u8,
417 offset: u64,
418) !*linux.io_uring_sqe {
419 const sqe = try self.get_sqe();
420 sqe.prep_write(fd, buffer, offset);
421 sqe.user_data = user_data;
422 return sqe;
423}
424
425/// Queues (but does not submit) an SQE to perform a `splice(2)`
426/// Either `fd_in` or `fd_out` must be a pipe.
427/// If `fd_in` refers to a pipe, `off_in` is ignored and must be set to std.math.maxInt(u64).
428/// If `fd_in` does not refer to a pipe and `off_in` is maxInt(u64), then `len` are read
429/// from `fd_in` starting from the file offset, which is incremented by the number of bytes read.
430/// If `fd_in` does not refer to a pipe and `off_in` is not maxInt(u64), then the starting offset of `fd_in` will be `off_in`.
431/// This splice operation can be used to implement sendfile by splicing to an intermediate pipe first,
432/// then splice to the final destination. In fact, the implementation of sendfile in kernel uses splice internally.
433///
434/// NOTE that even if fd_in or fd_out refers to a pipe, the splice operation can still fail with EINVAL if one of the
435/// fd doesn't explicitly support splice peration, e.g. reading from terminal is unsupported from kernel 5.7 to 5.11.
436/// See https://github.com/axboe/liburing/issues/291
437///
438/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
439pub fn splice(self: *IoUring, user_data: u64, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) !*linux.io_uring_sqe {
440 const sqe = try self.get_sqe();
441 sqe.prep_splice(fd_in, off_in, fd_out, off_out, len);
442 sqe.user_data = user_data;
443 return sqe;
444}
445
446/// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.
447/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
448/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
449///
450/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
451pub fn read_fixed(
452 self: *IoUring,
453 user_data: u64,
454 fd: os.fd_t,
455 buffer: *os.iovec,
456 offset: u64,
457 buffer_index: u16,
458) !*linux.io_uring_sqe {
459 const sqe = try self.get_sqe();
460 sqe.prep_read_fixed(fd, buffer, offset, buffer_index);
461 sqe.user_data = user_data;
462 return sqe;
463}
464
465/// Queues (but does not submit) an SQE to perform a `pwritev()`.
466/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
467/// For example, if you want to do a `pwritev2()` then set `rw_flags` on the returned SQE.
468/// See https://linux.die.net/man/2/pwritev.
469pub fn writev(
470 self: *IoUring,
471 user_data: u64,
472 fd: os.fd_t,
473 iovecs: []const os.iovec_const,
474 offset: u64,
475) !*linux.io_uring_sqe {
476 const sqe = try self.get_sqe();
477 sqe.prep_writev(fd, iovecs, offset);
478 sqe.user_data = user_data;
479 return sqe;
480}
481
482/// Queues (but does not submit) an SQE to perform a IORING_OP_WRITE_FIXED.
483/// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
484/// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
485///
486/// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
487pub fn write_fixed(
488 self: *IoUring,
489 user_data: u64,
490 fd: os.fd_t,
491 buffer: *os.iovec,
492 offset: u64,
493 buffer_index: u16,
494) !*linux.io_uring_sqe {
495 const sqe = try self.get_sqe();
496 sqe.prep_write_fixed(fd, buffer, offset, buffer_index);
497 sqe.user_data = user_data;
498 return sqe;
499}
500
501/// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
502/// Returns a pointer to the SQE.
503/// Available since 5.5
504pub fn accept(
505 self: *IoUring,
506 user_data: u64,
507 fd: os.fd_t,
508 addr: ?*os.sockaddr,
509 addrlen: ?*os.socklen_t,
510 flags: u32,
511) !*linux.io_uring_sqe {
512 const sqe = try self.get_sqe();
513 sqe.prep_accept(fd, addr, addrlen, flags);
514 sqe.user_data = user_data;
515 return sqe;
516}
517
518/// Queues an multishot accept on a socket.
519///
520/// Multishot variant allows an application to issue a single accept request,
521/// which will repeatedly trigger a CQE when a connection request comes in.
522/// While IORING_CQE_F_MORE flag is set in CQE flags accept will generate
523/// further CQEs.
524///
525/// Available since 5.19
526pub fn accept_multishot(
527 self: *IoUring,
528 user_data: u64,
529 fd: os.fd_t,
530 addr: ?*os.sockaddr,
531 addrlen: ?*os.socklen_t,
532 flags: u32,
533) !*linux.io_uring_sqe {
534 const sqe = try self.get_sqe();
535 sqe.prep_multishot_accept(fd, addr, addrlen, flags);
536 sqe.user_data = user_data;
537 return sqe;
538}
539
540/// Queues an accept using direct (registered) file descriptors.
541///
542/// To use an accept direct variant, the application must first have registered
543/// a file table (with register_files). An unused table index will be
544/// dynamically chosen and returned in the CQE res field.
545///
546/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
547/// flags member, and setting the SQE fd field to the direct descriptor value
548/// rather than the regular file descriptor.
549///
550/// Available since 5.19
551pub fn accept_direct(
552 self: *IoUring,
553 user_data: u64,
554 fd: os.fd_t,
555 addr: ?*os.sockaddr,
556 addrlen: ?*os.socklen_t,
557 flags: u32,
558) !*linux.io_uring_sqe {
559 const sqe = try self.get_sqe();
560 sqe.prep_accept_direct(fd, addr, addrlen, flags, linux.IORING_FILE_INDEX_ALLOC);
561 sqe.user_data = user_data;
562 return sqe;
563}
564
565/// Queues an multishot accept using direct (registered) file descriptors.
566/// Available since 5.19
567pub fn accept_multishot_direct(
568 self: *IoUring,
569 user_data: u64,
570 fd: os.fd_t,
571 addr: ?*os.sockaddr,
572 addrlen: ?*os.socklen_t,
573 flags: u32,
574) !*linux.io_uring_sqe {
575 const sqe = try self.get_sqe();
576 sqe.prep_multishot_accept_direct(fd, addr, addrlen, flags);
577 sqe.user_data = user_data;
578 return sqe;
579}
580
581/// Queue (but does not submit) an SQE to perform a `connect(2)` on a socket.
582/// Returns a pointer to the SQE.
583pub fn connect(
584 self: *IoUring,
585 user_data: u64,
586 fd: os.fd_t,
587 addr: *const os.sockaddr,
588 addrlen: os.socklen_t,
589) !*linux.io_uring_sqe {
590 const sqe = try self.get_sqe();
591 sqe.prep_connect(fd, addr, addrlen);
592 sqe.user_data = user_data;
593 return sqe;
594}
595
596/// Queues (but does not submit) an SQE to perform a `epoll_ctl(2)`.
597/// Returns a pointer to the SQE.
598pub fn epoll_ctl(
599 self: *IoUring,
600 user_data: u64,
601 epfd: os.fd_t,
602 fd: os.fd_t,
603 op: u32,
604 ev: ?*linux.epoll_event,
605) !*linux.io_uring_sqe {
606 const sqe = try self.get_sqe();
607 sqe.prep_epoll_ctl(epfd, fd, op, ev);
608 sqe.user_data = user_data;
609 return sqe;
610}
611
612/// Used to select how the recv call should be handled.
613pub const RecvBuffer = union(enum) {
614 /// io_uring will recv directly into this buffer
615 buffer: []u8,
616
617 /// io_uring will select a buffer that has previously been provided with `provide_buffers`.
618 /// The buffer group referenced by `group_id` must contain at least one buffer for the recv call to work.
619 /// `len` controls the number of bytes to read into the selected buffer.
620 buffer_selection: struct {
621 group_id: u16,
622 len: usize,
623 },
624};
625
626/// Queues (but does not submit) an SQE to perform a `recv(2)`.
627/// Returns a pointer to the SQE.
628/// Available since 5.6
629pub fn recv(
630 self: *IoUring,
631 user_data: u64,
632 fd: os.fd_t,
633 buffer: RecvBuffer,
634 flags: u32,
635) !*linux.io_uring_sqe {
636 const sqe = try self.get_sqe();
637 switch (buffer) {
638 .buffer => |slice| sqe.prep_recv(fd, slice, flags),
639 .buffer_selection => |selection| {
640 sqe.prep_rw(.RECV, fd, 0, selection.len, 0);
641 sqe.rw_flags = flags;
642 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
643 sqe.buf_index = selection.group_id;
644 },
645 }
646 sqe.user_data = user_data;
647 return sqe;
648}
649
650/// Queues (but does not submit) an SQE to perform a `send(2)`.
651/// Returns a pointer to the SQE.
652/// Available since 5.6
653pub fn send(
654 self: *IoUring,
655 user_data: u64,
656 fd: os.fd_t,
657 buffer: []const u8,
658 flags: u32,
659) !*linux.io_uring_sqe {
660 const sqe = try self.get_sqe();
661 sqe.prep_send(fd, buffer, flags);
662 sqe.user_data = user_data;
663 return sqe;
664}
665
666/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
667///
668/// This operation will most likely produce two CQEs. The flags field of the
669/// first cqe may likely contain IORING_CQE_F_MORE, which means that there will
670/// be a second cqe with the user_data field set to the same value. The user
671/// must not modify the data buffer until the notification is posted. The first
672/// cqe follows the usual rules and so its res field will contain the number of
673/// bytes sent or a negative error code. The notification's res field will be
674/// set to zero and the flags field will contain IORING_CQE_F_NOTIF. The two
675/// step model is needed because the kernel may hold on to buffers for a long
676/// time, e.g. waiting for a TCP ACK. Notifications responsible for controlling
677/// the lifetime of the buffers. Even errored requests may generate a
678/// notification.
679///
680/// Available since 6.0
681pub fn send_zc(
682 self: *IoUring,
683 user_data: u64,
684 fd: os.fd_t,
685 buffer: []const u8,
686 send_flags: u32,
687 zc_flags: u16,
688) !*linux.io_uring_sqe {
689 const sqe = try self.get_sqe();
690 sqe.prep_send_zc(fd, buffer, send_flags, zc_flags);
691 sqe.user_data = user_data;
692 return sqe;
693}
694
695/// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
696/// Returns a pointer to the SQE.
697/// Available since 6.0
698pub fn send_zc_fixed(
699 self: *IoUring,
700 user_data: u64,
701 fd: os.fd_t,
702 buffer: []const u8,
703 send_flags: u32,
704 zc_flags: u16,
705 buf_index: u16,
706) !*linux.io_uring_sqe {
707 const sqe = try self.get_sqe();
708 sqe.prep_send_zc_fixed(fd, buffer, send_flags, zc_flags, buf_index);
709 sqe.user_data = user_data;
710 return sqe;
711}
712
713/// Queues (but does not submit) an SQE to perform a `recvmsg(2)`.
714/// Returns a pointer to the SQE.
715/// Available since 5.3
716pub fn recvmsg(
717 self: *IoUring,
718 user_data: u64,
719 fd: os.fd_t,
720 msg: *os.msghdr,
721 flags: u32,
722) !*linux.io_uring_sqe {
723 const sqe = try self.get_sqe();
724 sqe.prep_recvmsg(fd, msg, flags);
725 sqe.user_data = user_data;
726 return sqe;
727}
728
729/// Queues (but does not submit) an SQE to perform a `sendmsg(2)`.
730/// Returns a pointer to the SQE.
731/// Available since 5.3
732pub fn sendmsg(
733 self: *IoUring,
734 user_data: u64,
735 fd: os.fd_t,
736 msg: *const os.msghdr_const,
737 flags: u32,
738) !*linux.io_uring_sqe {
739 const sqe = try self.get_sqe();
740 sqe.prep_sendmsg(fd, msg, flags);
741 sqe.user_data = user_data;
742 return sqe;
743}
744
745/// Queues (but does not submit) an SQE to perform an async zerocopy `sendmsg(2)`.
746/// Returns a pointer to the SQE.
747/// Available since 6.1
748pub fn sendmsg_zc(
749 self: *IoUring,
750 user_data: u64,
751 fd: os.fd_t,
752 msg: *const os.msghdr_const,
753 flags: u32,
754) !*linux.io_uring_sqe {
755 const sqe = try self.get_sqe();
756 sqe.prep_sendmsg_zc(fd, msg, flags);
757 sqe.user_data = user_data;
758 return sqe;
759}
760
761/// Queues (but does not submit) an SQE to perform an `openat(2)`.
762/// Returns a pointer to the SQE.
763/// Available since 5.6.
764pub fn openat(
765 self: *IoUring,
766 user_data: u64,
767 fd: os.fd_t,
768 path: [*:0]const u8,
769 flags: linux.O,
770 mode: os.mode_t,
771) !*linux.io_uring_sqe {
772 const sqe = try self.get_sqe();
773 sqe.prep_openat(fd, path, flags, mode);
774 sqe.user_data = user_data;
775 return sqe;
776}
777
778/// Queues an openat using direct (registered) file descriptors.
779///
780/// To use an accept direct variant, the application must first have registered
781/// a file table (with register_files). An unused table index will be
782/// dynamically chosen and returned in the CQE res field.
783///
784/// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
785/// flags member, and setting the SQE fd field to the direct descriptor value
786/// rather than the regular file descriptor.
787///
788/// Available since 5.15
789pub fn openat_direct(
790 self: *IoUring,
791 user_data: u64,
792 fd: os.fd_t,
793 path: [*:0]const u8,
794 flags: linux.O,
795 mode: os.mode_t,
796 file_index: u32,
797) !*linux.io_uring_sqe {
798 const sqe = try self.get_sqe();
799 sqe.prep_openat_direct(fd, path, flags, mode, file_index);
800 sqe.user_data = user_data;
801 return sqe;
802}
803
804/// Queues (but does not submit) an SQE to perform a `close(2)`.
805/// Returns a pointer to the SQE.
806/// Available since 5.6.
807pub fn close(self: *IoUring, user_data: u64, fd: os.fd_t) !*linux.io_uring_sqe {
808 const sqe = try self.get_sqe();
809 sqe.prep_close(fd);
810 sqe.user_data = user_data;
811 return sqe;
812}
813
814/// Queues close of registered file descriptor.
815/// Available since 5.15
816pub fn close_direct(self: *IoUring, user_data: u64, file_index: u32) !*linux.io_uring_sqe {
817 const sqe = try self.get_sqe();
818 sqe.prep_close_direct(file_index);
819 sqe.user_data = user_data;
820 return sqe;
821}
822
823/// Queues (but does not submit) an SQE to register a timeout operation.
824/// Returns a pointer to the SQE.
825///
826/// The timeout will complete when either the timeout expires, or after the specified number of
827/// events complete (if `count` is greater than `0`).
828///
829/// `flags` may be `0` for a relative timeout, or `IORING_TIMEOUT_ABS` for an absolute timeout.
830///
831/// The completion event result will be `-ETIME` if the timeout completed through expiration,
832/// `0` if the timeout completed after the specified number of events, or `-ECANCELED` if the
833/// timeout was removed before it expired.
834///
835/// io_uring timeouts use the `CLOCK.MONOTONIC` clock source.
836pub fn timeout(
837 self: *IoUring,
838 user_data: u64,
839 ts: *const os.linux.kernel_timespec,
840 count: u32,
841 flags: u32,
842) !*linux.io_uring_sqe {
843 const sqe = try self.get_sqe();
844 sqe.prep_timeout(ts, count, flags);
845 sqe.user_data = user_data;
846 return sqe;
847}
848
849/// Queues (but does not submit) an SQE to remove an existing timeout operation.
850/// Returns a pointer to the SQE.
851///
852/// The timeout is identified by its `user_data`.
853///
854/// The completion event result will be `0` if the timeout was found and cancelled successfully,
855/// `-EBUSY` if the timeout was found but expiration was already in progress, or
856/// `-ENOENT` if the timeout was not found.
857pub fn timeout_remove(
858 self: *IoUring,
859 user_data: u64,
860 timeout_user_data: u64,
861 flags: u32,
862) !*linux.io_uring_sqe {
863 const sqe = try self.get_sqe();
864 sqe.prep_timeout_remove(timeout_user_data, flags);
865 sqe.user_data = user_data;
866 return sqe;
867}
868
869/// Queues (but does not submit) an SQE to add a link timeout operation.
870/// Returns a pointer to the SQE.
871///
872/// You need to set linux.IOSQE_IO_LINK to flags of the target operation
873/// and then call this method right after the target operation.
874/// See https://lwn.net/Articles/803932/ for detail.
875///
876/// If the dependent request finishes before the linked timeout, the timeout
877/// is canceled. If the timeout finishes before the dependent request, the
878/// dependent request will be canceled.
879///
880/// The completion event result of the link_timeout will be
881/// `-ETIME` if the timeout finishes before the dependent request
882/// (in this case, the completion event result of the dependent request will
883/// be `-ECANCELED`), or
884/// `-EALREADY` if the dependent request finishes before the linked timeout.
885pub fn link_timeout(
886 self: *IoUring,
887 user_data: u64,
888 ts: *const os.linux.kernel_timespec,
889 flags: u32,
890) !*linux.io_uring_sqe {
891 const sqe = try self.get_sqe();
892 sqe.prep_link_timeout(ts, flags);
893 sqe.user_data = user_data;
894 return sqe;
895}
896
897/// Queues (but does not submit) an SQE to perform a `poll(2)`.
898/// Returns a pointer to the SQE.
899pub fn poll_add(
900 self: *IoUring,
901 user_data: u64,
902 fd: os.fd_t,
903 poll_mask: u32,
904) !*linux.io_uring_sqe {
905 const sqe = try self.get_sqe();
906 sqe.prep_poll_add(fd, poll_mask);
907 sqe.user_data = user_data;
908 return sqe;
909}
910
911/// Queues (but does not submit) an SQE to remove an existing poll operation.
912/// Returns a pointer to the SQE.
913pub fn poll_remove(
914 self: *IoUring,
915 user_data: u64,
916 target_user_data: u64,
917) !*linux.io_uring_sqe {
918 const sqe = try self.get_sqe();
919 sqe.prep_poll_remove(target_user_data);
920 sqe.user_data = user_data;
921 return sqe;
922}
923
924/// Queues (but does not submit) an SQE to update the user data of an existing poll
925/// operation. Returns a pointer to the SQE.
926pub fn poll_update(
927 self: *IoUring,
928 user_data: u64,
929 old_user_data: u64,
930 new_user_data: u64,
931 poll_mask: u32,
932 flags: u32,
933) !*linux.io_uring_sqe {
934 const sqe = try self.get_sqe();
935 sqe.prep_poll_update(old_user_data, new_user_data, poll_mask, flags);
936 sqe.user_data = user_data;
937 return sqe;
938}
939
940/// Queues (but does not submit) an SQE to perform an `fallocate(2)`.
941/// Returns a pointer to the SQE.
942pub fn fallocate(
943 self: *IoUring,
944 user_data: u64,
945 fd: os.fd_t,
946 mode: i32,
947 offset: u64,
948 len: u64,
949) !*linux.io_uring_sqe {
950 const sqe = try self.get_sqe();
951 sqe.prep_fallocate(fd, mode, offset, len);
952 sqe.user_data = user_data;
953 return sqe;
954}
955
956/// Queues (but does not submit) an SQE to perform an `statx(2)`.
957/// Returns a pointer to the SQE.
958pub fn statx(
959 self: *IoUring,
960 user_data: u64,
961 fd: os.fd_t,
962 path: [:0]const u8,
963 flags: u32,
964 mask: u32,
965 buf: *linux.Statx,
966) !*linux.io_uring_sqe {
967 const sqe = try self.get_sqe();
968 sqe.prep_statx(fd, path, flags, mask, buf);
969 sqe.user_data = user_data;
970 return sqe;
971}
972
973/// Queues (but does not submit) an SQE to remove an existing operation.
974/// Returns a pointer to the SQE.
975///
976/// The operation is identified by its `user_data`.
977///
978/// The completion event result will be `0` if the operation was found and cancelled successfully,
979/// `-EALREADY` if the operation was found but was already in progress, or
980/// `-ENOENT` if the operation was not found.
981pub fn cancel(
982 self: *IoUring,
983 user_data: u64,
984 cancel_user_data: u64,
985 flags: u32,
986) !*linux.io_uring_sqe {
987 const sqe = try self.get_sqe();
988 sqe.prep_cancel(cancel_user_data, flags);
989 sqe.user_data = user_data;
990 return sqe;
991}
992
993/// Queues (but does not submit) an SQE to perform a `shutdown(2)`.
994/// Returns a pointer to the SQE.
995///
996/// The operation is identified by its `user_data`.
997pub fn shutdown(
998 self: *IoUring,
999 user_data: u64,
1000 sockfd: os.socket_t,
1001 how: u32,
1002) !*linux.io_uring_sqe {
1003 const sqe = try self.get_sqe();
1004 sqe.prep_shutdown(sockfd, how);
1005 sqe.user_data = user_data;
1006 return sqe;
1007}
1008
1009/// Queues (but does not submit) an SQE to perform a `renameat2(2)`.
1010/// Returns a pointer to the SQE.
1011pub fn renameat(
1012 self: *IoUring,
1013 user_data: u64,
1014 old_dir_fd: os.fd_t,
1015 old_path: [*:0]const u8,
1016 new_dir_fd: os.fd_t,
1017 new_path: [*:0]const u8,
1018 flags: u32,
1019) !*linux.io_uring_sqe {
1020 const sqe = try self.get_sqe();
1021 sqe.prep_renameat(old_dir_fd, old_path, new_dir_fd, new_path, flags);
1022 sqe.user_data = user_data;
1023 return sqe;
1024}
1025
1026/// Queues (but does not submit) an SQE to perform a `unlinkat(2)`.
1027/// Returns a pointer to the SQE.
1028pub fn unlinkat(
1029 self: *IoUring,
1030 user_data: u64,
1031 dir_fd: os.fd_t,
1032 path: [*:0]const u8,
1033 flags: u32,
1034) !*linux.io_uring_sqe {
1035 const sqe = try self.get_sqe();
1036 sqe.prep_unlinkat(dir_fd, path, flags);
1037 sqe.user_data = user_data;
1038 return sqe;
1039}
1040
1041/// Queues (but does not submit) an SQE to perform a `mkdirat(2)`.
1042/// Returns a pointer to the SQE.
1043pub fn mkdirat(
1044 self: *IoUring,
1045 user_data: u64,
1046 dir_fd: os.fd_t,
1047 path: [*:0]const u8,
1048 mode: os.mode_t,
1049) !*linux.io_uring_sqe {
1050 const sqe = try self.get_sqe();
1051 sqe.prep_mkdirat(dir_fd, path, mode);
1052 sqe.user_data = user_data;
1053 return sqe;
1054}
1055
1056/// Queues (but does not submit) an SQE to perform a `symlinkat(2)`.
1057/// Returns a pointer to the SQE.
1058pub fn symlinkat(
1059 self: *IoUring,
1060 user_data: u64,
1061 target: [*:0]const u8,
1062 new_dir_fd: os.fd_t,
1063 link_path: [*:0]const u8,
1064) !*linux.io_uring_sqe {
1065 const sqe = try self.get_sqe();
1066 sqe.prep_symlinkat(target, new_dir_fd, link_path);
1067 sqe.user_data = user_data;
1068 return sqe;
1069}
1070
1071/// Queues (but does not submit) an SQE to perform a `linkat(2)`.
1072/// Returns a pointer to the SQE.
1073pub fn linkat(
1074 self: *IoUring,
1075 user_data: u64,
1076 old_dir_fd: os.fd_t,
1077 old_path: [*:0]const u8,
1078 new_dir_fd: os.fd_t,
1079 new_path: [*:0]const u8,
1080 flags: u32,
1081) !*linux.io_uring_sqe {
1082 const sqe = try self.get_sqe();
1083 sqe.prep_linkat(old_dir_fd, old_path, new_dir_fd, new_path, flags);
1084 sqe.user_data = user_data;
1085 return sqe;
1086}
1087
1088/// Queues (but does not submit) an SQE to provide a group of buffers used for commands that read/receive data.
1089/// Returns a pointer to the SQE.
1090///
1091/// Provided buffers can be used in `read`, `recv` or `recvmsg` commands via .buffer_selection.
1092///
1093/// The kernel expects a contiguous block of memory of size (buffers_count * buffer_size).
1094pub fn provide_buffers(
1095 self: *IoUring,
1096 user_data: u64,
1097 buffers: [*]u8,
1098 buffer_size: usize,
1099 buffers_count: usize,
1100 group_id: usize,
1101 buffer_id: usize,
1102) !*linux.io_uring_sqe {
1103 const sqe = try self.get_sqe();
1104 sqe.prep_provide_buffers(buffers, buffer_size, buffers_count, group_id, buffer_id);
1105 sqe.user_data = user_data;
1106 return sqe;
1107}
1108
1109/// Queues (but does not submit) an SQE to remove a group of provided buffers.
1110/// Returns a pointer to the SQE.
1111pub fn remove_buffers(
1112 self: *IoUring,
1113 user_data: u64,
1114 buffers_count: usize,
1115 group_id: usize,
1116) !*linux.io_uring_sqe {
1117 const sqe = try self.get_sqe();
1118 sqe.prep_remove_buffers(buffers_count, group_id);
1119 sqe.user_data = user_data;
1120 return sqe;
1121}
1122
1123/// Queues (but does not submit) an SQE to perform a `waitid(2)`.
1124/// Returns a pointer to the SQE.
1125pub fn waitid(
1126 self: *IoUring,
1127 user_data: u64,
1128 id_type: linux.P,
1129 id: i32,
1130 infop: *linux.siginfo_t,
1131 options: u32,
1132 flags: u32,
1133) !*linux.io_uring_sqe {
1134 const sqe = try self.get_sqe();
1135 sqe.prep_waitid(id_type, id, infop, options, flags);
1136 sqe.user_data = user_data;
1137 return sqe;
1138}
1139
1140/// Registers an array of file descriptors.
1141/// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
1142/// retrieve a reference to the file, and once I/O has completed the file reference must be
1143/// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
1144/// This slowdown can be avoided by pre-registering file descriptors.
1145/// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
1146/// and the SQE's fd must be set to the index of the file descriptor in the registered array.
1147/// Registering file descriptors will wait for the ring to idle.
1148/// Files are automatically unregistered by the kernel when the ring is torn down.
1149/// An application need unregister only if it wants to register a new array of file descriptors.
1150pub fn register_files(self: *IoUring, fds: []const os.fd_t) !void {
1151 assert(self.fd >= 0);
1152 const res = linux.io_uring_register(
1153 self.fd,
1154 .REGISTER_FILES,
1155 @as(*const anyopaque, @ptrCast(fds.ptr)),
1156 @as(u32, @intCast(fds.len)),
1157 );
1158 try handle_registration_result(res);
1159}
1160
1161/// Updates registered file descriptors.
1162///
1163/// Updates are applied starting at the provided offset in the original file descriptors slice.
1164/// There are three kind of updates:
1165/// * turning a sparse entry (where the fd is -1) into a real one
1166/// * removing an existing entry (set the fd to -1)
1167/// * replacing an existing entry with a new fd
1168/// Adding new file descriptors must be done with `register_files`.
1169pub fn register_files_update(self: *IoUring, offset: u32, fds: []const os.fd_t) !void {
1170 assert(self.fd >= 0);
1171
1172 const FilesUpdate = extern struct {
1173 offset: u32,
1174 resv: u32,
1175 fds: u64 align(8),
1176 };
1177 var update = FilesUpdate{
1178 .offset = offset,
1179 .resv = @as(u32, 0),
1180 .fds = @as(u64, @intFromPtr(fds.ptr)),
1181 };
1182
1183 const res = linux.io_uring_register(
1184 self.fd,
1185 .REGISTER_FILES_UPDATE,
1186 @as(*const anyopaque, @ptrCast(&update)),
1187 @as(u32, @intCast(fds.len)),
1188 );
1189 try handle_registration_result(res);
1190}
1191
1192/// Registers the file descriptor for an eventfd that will be notified of completion events on
1193/// an io_uring instance.
1194/// Only a single a eventfd can be registered at any given point in time.
1195pub fn register_eventfd(self: *IoUring, fd: os.fd_t) !void {
1196 assert(self.fd >= 0);
1197 const res = linux.io_uring_register(
1198 self.fd,
1199 .REGISTER_EVENTFD,
1200 @as(*const anyopaque, @ptrCast(&fd)),
1201 1,
1202 );
1203 try handle_registration_result(res);
1204}
1205
1206/// Registers the file descriptor for an eventfd that will be notified of completion events on
1207/// an io_uring instance. Notifications are only posted for events that complete in an async manner.
1208/// This means that events that complete inline while being submitted do not trigger a notification event.
1209/// Only a single eventfd can be registered at any given point in time.
1210pub fn register_eventfd_async(self: *IoUring, fd: os.fd_t) !void {
1211 assert(self.fd >= 0);
1212 const res = linux.io_uring_register(
1213 self.fd,
1214 .REGISTER_EVENTFD_ASYNC,
1215 @as(*const anyopaque, @ptrCast(&fd)),
1216 1,
1217 );
1218 try handle_registration_result(res);
1219}
1220
1221/// Unregister the registered eventfd file descriptor.
1222pub fn unregister_eventfd(self: *IoUring) !void {
1223 assert(self.fd >= 0);
1224 const res = linux.io_uring_register(
1225 self.fd,
1226 .UNREGISTER_EVENTFD,
1227 null,
1228 0,
1229 );
1230 try handle_registration_result(res);
1231}
1232
1233/// Registers an array of buffers for use with `read_fixed` and `write_fixed`.
1234pub fn register_buffers(self: *IoUring, buffers: []const os.iovec) !void {
1235 assert(self.fd >= 0);
1236 const res = linux.io_uring_register(
1237 self.fd,
1238 .REGISTER_BUFFERS,
1239 buffers.ptr,
1240 @as(u32, @intCast(buffers.len)),
1241 );
1242 try handle_registration_result(res);
1243}
1244
1245/// Unregister the registered buffers.
1246pub fn unregister_buffers(self: *IoUring) !void {
1247 assert(self.fd >= 0);
1248 const res = linux.io_uring_register(self.fd, .UNREGISTER_BUFFERS, null, 0);
1249 switch (linux.getErrno(res)) {
1250 .SUCCESS => {},
1251 .NXIO => return error.BuffersNotRegistered,
1252 else => |errno| return os.unexpectedErrno(errno),
1253 }
1254}
1255
1256fn handle_registration_result(res: usize) !void {
1257 switch (linux.getErrno(res)) {
1258 .SUCCESS => {},
1259 // One or more fds in the array are invalid, or the kernel does not support sparse sets:
1260 .BADF => return error.FileDescriptorInvalid,
1261 .BUSY => return error.FilesAlreadyRegistered,
1262 .INVAL => return error.FilesEmpty,
1263 // Adding `nr_args` file references would exceed the maximum allowed number of files the
1264 // user is allowed to have according to the per-user RLIMIT_NOFILE resource limit and
1265 // the CAP_SYS_RESOURCE capability is not set, or `nr_args` exceeds the maximum allowed
1266 // for a fixed file set (older kernels have a limit of 1024 files vs 64K files):
1267 .MFILE => return error.UserFdQuotaExceeded,
1268 // Insufficient kernel resources, or the caller had a non-zero RLIMIT_MEMLOCK soft
1269 // resource limit but tried to lock more memory than the limit permitted (not enforced
1270 // when the process is privileged with CAP_IPC_LOCK):
1271 .NOMEM => return error.SystemResources,
1272 // Attempt to register files on a ring already registering files or being torn down:
1273 .NXIO => return error.RingShuttingDownOrAlreadyRegisteringFiles,
1274 else => |errno| return os.unexpectedErrno(errno),
1275 }
1276}
1277
1278/// Unregisters all registered file descriptors previously associated with the ring.
1279pub fn unregister_files(self: *IoUring) !void {
1280 assert(self.fd >= 0);
1281 const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
1282 switch (linux.getErrno(res)) {
1283 .SUCCESS => {},
1284 .NXIO => return error.FilesNotRegistered,
1285 else => |errno| return os.unexpectedErrno(errno),
1286 }
1287}
1288
1289/// Prepares a socket creation request.
1290/// New socket fd will be returned in completion result.
1291/// Available since 5.19
1292pub fn socket(
1293 self: *IoUring,
1294 user_data: u64,
1295 domain: u32,
1296 socket_type: u32,
1297 protocol: u32,
1298 flags: u32,
1299) !*linux.io_uring_sqe {
1300 const sqe = try self.get_sqe();
1301 sqe.prep_socket(domain, socket_type, protocol, flags);
1302 sqe.user_data = user_data;
1303 return sqe;
1304}
1305
1306/// Prepares a socket creation request for registered file at index `file_index`.
1307/// Available since 5.19
1308pub fn socket_direct(
1309 self: *IoUring,
1310 user_data: u64,
1311 domain: u32,
1312 socket_type: u32,
1313 protocol: u32,
1314 flags: u32,
1315 file_index: u32,
1316) !*linux.io_uring_sqe {
1317 const sqe = try self.get_sqe();
1318 sqe.prep_socket_direct(domain, socket_type, protocol, flags, file_index);
1319 sqe.user_data = user_data;
1320 return sqe;
1321}
1322
1323/// Prepares a socket creation request for registered file, index chosen by kernel (file index alloc).
1324/// File index will be returned in CQE res field.
1325/// Available since 5.19
1326pub fn socket_direct_alloc(
1327 self: *IoUring,
1328 user_data: u64,
1329 domain: u32,
1330 socket_type: u32,
1331 protocol: u32,
1332 flags: u32,
1333) !*linux.io_uring_sqe {
1334 const sqe = try self.get_sqe();
1335 sqe.prep_socket_direct_alloc(domain, socket_type, protocol, flags);
1336 sqe.user_data = user_data;
1337 return sqe;
1338}
1339
1340pub const SubmissionQueue = struct {
1341 head: *u32,
1342 tail: *u32,
1343 mask: u32,
1344 flags: *u32,
1345 dropped: *u32,
1346 array: []u32,
1347 sqes: []linux.io_uring_sqe,
1348 mmap: []align(mem.page_size) u8,
1349 mmap_sqes: []align(mem.page_size) u8,
1350
1351 // We use `sqe_head` and `sqe_tail` in the same way as liburing:
1352 // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
1353 // We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
1354 // This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
1355 sqe_head: u32 = 0,
1356 sqe_tail: u32 = 0,
1357
1358 pub fn init(fd: os.fd_t, p: linux.io_uring_params) !SubmissionQueue {
1359 assert(fd >= 0);
1360 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1361 const size = @max(
1362 p.sq_off.array + p.sq_entries * @sizeOf(u32),
1363 p.cq_off.cqes + p.cq_entries * @sizeOf(linux.io_uring_cqe),
1364 );
1365 const mmap = try os.mmap(
1366 null,
1367 size,
1368 os.PROT.READ | os.PROT.WRITE,
1369 .{ .TYPE = .SHARED, .POPULATE = true },
1370 fd,
1371 linux.IORING_OFF_SQ_RING,
1372 );
1373 errdefer os.munmap(mmap);
1374 assert(mmap.len == size);
1375
1376 // The motivation for the `sqes` and `array` indirection is to make it possible for the
1377 // application to preallocate static linux.io_uring_sqe entries and then replay them when needed.
1378 const size_sqes = p.sq_entries * @sizeOf(linux.io_uring_sqe);
1379 const mmap_sqes = try os.mmap(
1380 null,
1381 size_sqes,
1382 os.PROT.READ | os.PROT.WRITE,
1383 .{ .TYPE = .SHARED, .POPULATE = true },
1384 fd,
1385 linux.IORING_OFF_SQES,
1386 );
1387 errdefer os.munmap(mmap_sqes);
1388 assert(mmap_sqes.len == size_sqes);
1389
1390 const array: [*]u32 = @ptrCast(@alignCast(&mmap[p.sq_off.array]));
1391 const sqes: [*]linux.io_uring_sqe = @ptrCast(@alignCast(&mmap_sqes[0]));
1392 // We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
1393 // see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
1394 assert(p.sq_entries == @as(*u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_entries]))).*);
1395 return SubmissionQueue{
1396 .head = @ptrCast(@alignCast(&mmap[p.sq_off.head])),
1397 .tail = @ptrCast(@alignCast(&mmap[p.sq_off.tail])),
1398 .mask = @as(*u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_mask]))).*,
1399 .flags = @ptrCast(@alignCast(&mmap[p.sq_off.flags])),
1400 .dropped = @ptrCast(@alignCast(&mmap[p.sq_off.dropped])),
1401 .array = array[0..p.sq_entries],
1402 .sqes = sqes[0..p.sq_entries],
1403 .mmap = mmap,
1404 .mmap_sqes = mmap_sqes,
1405 };
1406 }
1407
1408 pub fn deinit(self: *SubmissionQueue) void {
1409 os.munmap(self.mmap_sqes);
1410 os.munmap(self.mmap);
1411 }
1412};
1413
1414pub const CompletionQueue = struct {
1415 head: *u32,
1416 tail: *u32,
1417 mask: u32,
1418 overflow: *u32,
1419 cqes: []linux.io_uring_cqe,
1420
1421 pub fn init(fd: os.fd_t, p: linux.io_uring_params, sq: SubmissionQueue) !CompletionQueue {
1422 assert(fd >= 0);
1423 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1424 const mmap = sq.mmap;
1425 const cqes: [*]linux.io_uring_cqe = @ptrCast(@alignCast(&mmap[p.cq_off.cqes]));
1426 assert(p.cq_entries == @as(*u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_entries]))).*);
1427 return CompletionQueue{
1428 .head = @ptrCast(@alignCast(&mmap[p.cq_off.head])),
1429 .tail = @ptrCast(@alignCast(&mmap[p.cq_off.tail])),
1430 .mask = @as(*u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_mask]))).*,
1431 .overflow = @ptrCast(@alignCast(&mmap[p.cq_off.overflow])),
1432 .cqes = cqes[0..p.cq_entries],
1433 };
1434 }
1435
1436 pub fn deinit(self: *CompletionQueue) void {
1437 _ = self;
1438 // A no-op since we now share the mmap with the submission queue.
1439 // Here for symmetry with the submission queue, and for any future feature support.
1440 }
1441};
1442
1443test "structs/offsets/entries" {
1444 if (builtin.os.tag != .linux) return error.SkipZigTest;
1445
1446 try testing.expectEqual(@as(usize, 120), @sizeOf(linux.io_uring_params));
1447 try testing.expectEqual(@as(usize, 64), @sizeOf(linux.io_uring_sqe));
1448 try testing.expectEqual(@as(usize, 16), @sizeOf(linux.io_uring_cqe));
1449
1450 try testing.expectEqual(0, linux.IORING_OFF_SQ_RING);
1451 try testing.expectEqual(0x8000000, linux.IORING_OFF_CQ_RING);
1452 try testing.expectEqual(0x10000000, linux.IORING_OFF_SQES);
1453
1454 try testing.expectError(error.EntriesZero, IoUring.init(0, 0));
1455 try testing.expectError(error.EntriesNotPowerOfTwo, IoUring.init(3, 0));
1456}
1457
1458test "nop" {
1459 if (builtin.os.tag != .linux) return error.SkipZigTest;
1460
1461 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1462 error.SystemOutdated => return error.SkipZigTest,
1463 error.PermissionDenied => return error.SkipZigTest,
1464 else => return err,
1465 };
1466 defer {
1467 ring.deinit();
1468 testing.expectEqual(@as(os.fd_t, -1), ring.fd) catch @panic("test failed");
1469 }
1470
1471 const sqe = try ring.nop(0xaaaaaaaa);
1472 try testing.expectEqual(linux.io_uring_sqe{
1473 .opcode = .NOP,
1474 .flags = 0,
1475 .ioprio = 0,
1476 .fd = 0,
1477 .off = 0,
1478 .addr = 0,
1479 .len = 0,
1480 .rw_flags = 0,
1481 .user_data = 0xaaaaaaaa,
1482 .buf_index = 0,
1483 .personality = 0,
1484 .splice_fd_in = 0,
1485 .addr3 = 0,
1486 .resv = 0,
1487 }, sqe.*);
1488
1489 try testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
1490 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
1491 try testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
1492 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
1493 try testing.expectEqual(@as(u32, 1), ring.sq_ready());
1494 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
1495
1496 try testing.expectEqual(@as(u32, 1), try ring.submit());
1497 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
1498 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
1499 try testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
1500 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
1501 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
1502
1503 try testing.expectEqual(linux.io_uring_cqe{
1504 .user_data = 0xaaaaaaaa,
1505 .res = 0,
1506 .flags = 0,
1507 }, try ring.copy_cqe());
1508 try testing.expectEqual(@as(u32, 1), ring.cq.head.*);
1509 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
1510
1511 const sqe_barrier = try ring.nop(0xbbbbbbbb);
1512 sqe_barrier.flags |= linux.IOSQE_IO_DRAIN;
1513 try testing.expectEqual(@as(u32, 1), try ring.submit());
1514 try testing.expectEqual(linux.io_uring_cqe{
1515 .user_data = 0xbbbbbbbb,
1516 .res = 0,
1517 .flags = 0,
1518 }, try ring.copy_cqe());
1519 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
1520 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
1521 try testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
1522 try testing.expectEqual(@as(u32, 2), ring.cq.head.*);
1523}
1524
1525test "readv" {
1526 if (builtin.os.tag != .linux) return error.SkipZigTest;
1527
1528 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1529 error.SystemOutdated => return error.SkipZigTest,
1530 error.PermissionDenied => return error.SkipZigTest,
1531 else => return err,
1532 };
1533 defer ring.deinit();
1534
1535 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
1536 defer os.close(fd);
1537
1538 // Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
1539 // Linux Kernel 5.5 adds support for sparse fd sets.
1540 // Compare:
1541 // https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
1542 // https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
1543 // We therefore avoid stressing sparse fd sets here:
1544 var registered_fds = [_]os.fd_t{0} ** 1;
1545 const fd_index = 0;
1546 registered_fds[fd_index] = fd;
1547 try ring.register_files(registered_fds[0..]);
1548
1549 var buffer = [_]u8{42} ** 128;
1550 var iovecs = [_]os.iovec{os.iovec{ .iov_base = &buffer, .iov_len = buffer.len }};
1551 const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
1552 try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
1553 sqe.flags |= linux.IOSQE_FIXED_FILE;
1554
1555 try testing.expectError(error.SubmissionQueueFull, ring.nop(0));
1556 try testing.expectEqual(@as(u32, 1), try ring.submit());
1557 try testing.expectEqual(linux.io_uring_cqe{
1558 .user_data = 0xcccccccc,
1559 .res = buffer.len,
1560 .flags = 0,
1561 }, try ring.copy_cqe());
1562 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
1563
1564 try ring.unregister_files();
1565}
1566
1567test "writev/fsync/readv" {
1568 if (builtin.os.tag != .linux) return error.SkipZigTest;
1569
1570 var ring = IoUring.init(4, 0) catch |err| switch (err) {
1571 error.SystemOutdated => return error.SkipZigTest,
1572 error.PermissionDenied => return error.SkipZigTest,
1573 else => return err,
1574 };
1575 defer ring.deinit();
1576
1577 var tmp = std.testing.tmpDir(.{});
1578 defer tmp.cleanup();
1579
1580 const path = "test_io_uring_writev_fsync_readv";
1581 const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
1582 defer file.close();
1583 const fd = file.handle;
1584
1585 const buffer_write = [_]u8{42} ** 128;
1586 const iovecs_write = [_]os.iovec_const{
1587 os.iovec_const{ .iov_base = &buffer_write, .iov_len = buffer_write.len },
1588 };
1589 var buffer_read = [_]u8{0} ** 128;
1590 var iovecs_read = [_]os.iovec{
1591 os.iovec{ .iov_base = &buffer_read, .iov_len = buffer_read.len },
1592 };
1593
1594 const sqe_writev = try ring.writev(0xdddddddd, fd, iovecs_write[0..], 17);
1595 try testing.expectEqual(linux.IORING_OP.WRITEV, sqe_writev.opcode);
1596 try testing.expectEqual(@as(u64, 17), sqe_writev.off);
1597 sqe_writev.flags |= linux.IOSQE_IO_LINK;
1598
1599 const sqe_fsync = try ring.fsync(0xeeeeeeee, fd, 0);
1600 try testing.expectEqual(linux.IORING_OP.FSYNC, sqe_fsync.opcode);
1601 try testing.expectEqual(fd, sqe_fsync.fd);
1602 sqe_fsync.flags |= linux.IOSQE_IO_LINK;
1603
1604 const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
1605 try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
1606 try testing.expectEqual(@as(u64, 17), sqe_readv.off);
1607
1608 try testing.expectEqual(@as(u32, 3), ring.sq_ready());
1609 try testing.expectEqual(@as(u32, 3), try ring.submit_and_wait(3));
1610 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
1611 try testing.expectEqual(@as(u32, 3), ring.cq_ready());
1612
1613 try testing.expectEqual(linux.io_uring_cqe{
1614 .user_data = 0xdddddddd,
1615 .res = buffer_write.len,
1616 .flags = 0,
1617 }, try ring.copy_cqe());
1618 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
1619
1620 try testing.expectEqual(linux.io_uring_cqe{
1621 .user_data = 0xeeeeeeee,
1622 .res = 0,
1623 .flags = 0,
1624 }, try ring.copy_cqe());
1625 try testing.expectEqual(@as(u32, 1), ring.cq_ready());
1626
1627 try testing.expectEqual(linux.io_uring_cqe{
1628 .user_data = 0xffffffff,
1629 .res = buffer_read.len,
1630 .flags = 0,
1631 }, try ring.copy_cqe());
1632 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
1633
1634 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
1635}
1636
1637test "write/read" {
1638 if (builtin.os.tag != .linux) return error.SkipZigTest;
1639
1640 var ring = IoUring.init(2, 0) catch |err| switch (err) {
1641 error.SystemOutdated => return error.SkipZigTest,
1642 error.PermissionDenied => return error.SkipZigTest,
1643 else => return err,
1644 };
1645 defer ring.deinit();
1646
1647 var tmp = std.testing.tmpDir(.{});
1648 defer tmp.cleanup();
1649 const path = "test_io_uring_write_read";
1650 const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
1651 defer file.close();
1652 const fd = file.handle;
1653
1654 const buffer_write = [_]u8{97} ** 20;
1655 var buffer_read = [_]u8{98} ** 20;
1656 const sqe_write = try ring.write(0x11111111, fd, buffer_write[0..], 10);
1657 try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
1658 try testing.expectEqual(@as(u64, 10), sqe_write.off);
1659 sqe_write.flags |= linux.IOSQE_IO_LINK;
1660 const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
1661 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
1662 try testing.expectEqual(@as(u64, 10), sqe_read.off);
1663 try testing.expectEqual(@as(u32, 2), try ring.submit());
1664
1665 const cqe_write = try ring.copy_cqe();
1666 const cqe_read = try ring.copy_cqe();
1667 // Prior to Linux Kernel 5.6 this is the only way to test for read/write support:
1668 // https://lwn.net/Articles/809820/
1669 if (cqe_write.err() == .INVAL) return error.SkipZigTest;
1670 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
1671 try testing.expectEqual(linux.io_uring_cqe{
1672 .user_data = 0x11111111,
1673 .res = buffer_write.len,
1674 .flags = 0,
1675 }, cqe_write);
1676 try testing.expectEqual(linux.io_uring_cqe{
1677 .user_data = 0x22222222,
1678 .res = buffer_read.len,
1679 .flags = 0,
1680 }, cqe_read);
1681 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
1682}
1683
1684test "splice/read" {
1685 if (builtin.os.tag != .linux) return error.SkipZigTest;
1686
1687 var ring = IoUring.init(4, 0) catch |err| switch (err) {
1688 error.SystemOutdated => return error.SkipZigTest,
1689 error.PermissionDenied => return error.SkipZigTest,
1690 else => return err,
1691 };
1692 defer ring.deinit();
1693
1694 var tmp = std.testing.tmpDir(.{});
1695 const path_src = "test_io_uring_splice_src";
1696 const file_src = try tmp.dir.createFile(path_src, .{ .read = true, .truncate = true });
1697 defer file_src.close();
1698 const fd_src = file_src.handle;
1699
1700 const path_dst = "test_io_uring_splice_dst";
1701 const file_dst = try tmp.dir.createFile(path_dst, .{ .read = true, .truncate = true });
1702 defer file_dst.close();
1703 const fd_dst = file_dst.handle;
1704
1705 const buffer_write = [_]u8{97} ** 20;
1706 var buffer_read = [_]u8{98} ** 20;
1707 _ = try file_src.write(&buffer_write);
1708
1709 const fds = try os.pipe();
1710 const pipe_offset: u64 = std.math.maxInt(u64);
1711
1712 const sqe_splice_to_pipe = try ring.splice(0x11111111, fd_src, 0, fds[1], pipe_offset, buffer_write.len);
1713 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_to_pipe.opcode);
1714 try testing.expectEqual(@as(u64, 0), sqe_splice_to_pipe.addr);
1715 try testing.expectEqual(pipe_offset, sqe_splice_to_pipe.off);
1716 sqe_splice_to_pipe.flags |= linux.IOSQE_IO_LINK;
1717
1718 const sqe_splice_from_pipe = try ring.splice(0x22222222, fds[0], pipe_offset, fd_dst, 10, buffer_write.len);
1719 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_from_pipe.opcode);
1720 try testing.expectEqual(pipe_offset, sqe_splice_from_pipe.addr);
1721 try testing.expectEqual(@as(u64, 10), sqe_splice_from_pipe.off);
1722 sqe_splice_from_pipe.flags |= linux.IOSQE_IO_LINK;
1723
1724 const sqe_read = try ring.read(0x33333333, fd_dst, .{ .buffer = buffer_read[0..] }, 10);
1725 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
1726 try testing.expectEqual(@as(u64, 10), sqe_read.off);
1727 try testing.expectEqual(@as(u32, 3), try ring.submit());
1728
1729 const cqe_splice_to_pipe = try ring.copy_cqe();
1730 const cqe_splice_from_pipe = try ring.copy_cqe();
1731 const cqe_read = try ring.copy_cqe();
1732 // Prior to Linux Kernel 5.6 this is the only way to test for splice/read support:
1733 // https://lwn.net/Articles/809820/
1734 if (cqe_splice_to_pipe.err() == .INVAL) return error.SkipZigTest;
1735 if (cqe_splice_from_pipe.err() == .INVAL) return error.SkipZigTest;
1736 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
1737 try testing.expectEqual(linux.io_uring_cqe{
1738 .user_data = 0x11111111,
1739 .res = buffer_write.len,
1740 .flags = 0,
1741 }, cqe_splice_to_pipe);
1742 try testing.expectEqual(linux.io_uring_cqe{
1743 .user_data = 0x22222222,
1744 .res = buffer_write.len,
1745 .flags = 0,
1746 }, cqe_splice_from_pipe);
1747 try testing.expectEqual(linux.io_uring_cqe{
1748 .user_data = 0x33333333,
1749 .res = buffer_read.len,
1750 .flags = 0,
1751 }, cqe_read);
1752 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
1753}
1754
1755test "write_fixed/read_fixed" {
1756 if (builtin.os.tag != .linux) return error.SkipZigTest;
1757
1758 var ring = IoUring.init(2, 0) catch |err| switch (err) {
1759 error.SystemOutdated => return error.SkipZigTest,
1760 error.PermissionDenied => return error.SkipZigTest,
1761 else => return err,
1762 };
1763 defer ring.deinit();
1764
1765 var tmp = std.testing.tmpDir(.{});
1766 defer tmp.cleanup();
1767
1768 const path = "test_io_uring_write_read_fixed";
1769 const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
1770 defer file.close();
1771 const fd = file.handle;
1772
1773 var raw_buffers: [2][11]u8 = undefined;
1774 // First buffer will be written to the file.
1775 @memset(&raw_buffers[0], 'z');
1776 raw_buffers[0][0.."foobar".len].* = "foobar".*;
1777
1778 var buffers = [2]os.iovec{
1779 .{ .iov_base = &raw_buffers[0], .iov_len = raw_buffers[0].len },
1780 .{ .iov_base = &raw_buffers[1], .iov_len = raw_buffers[1].len },
1781 };
1782 ring.register_buffers(&buffers) catch |err| switch (err) {
1783 error.SystemResources => {
1784 // See https://github.com/ziglang/zig/issues/15362
1785 return error.SkipZigTest;
1786 },
1787 else => |e| return e,
1788 };
1789
1790 const sqe_write = try ring.write_fixed(0x45454545, fd, &buffers[0], 3, 0);
1791 try testing.expectEqual(linux.IORING_OP.WRITE_FIXED, sqe_write.opcode);
1792 try testing.expectEqual(@as(u64, 3), sqe_write.off);
1793 sqe_write.flags |= linux.IOSQE_IO_LINK;
1794
1795 const sqe_read = try ring.read_fixed(0x12121212, fd, &buffers[1], 0, 1);
1796 try testing.expectEqual(linux.IORING_OP.READ_FIXED, sqe_read.opcode);
1797 try testing.expectEqual(@as(u64, 0), sqe_read.off);
1798
1799 try testing.expectEqual(@as(u32, 2), try ring.submit());
1800
1801 const cqe_write = try ring.copy_cqe();
1802 const cqe_read = try ring.copy_cqe();
1803
1804 try testing.expectEqual(linux.io_uring_cqe{
1805 .user_data = 0x45454545,
1806 .res = @as(i32, @intCast(buffers[0].iov_len)),
1807 .flags = 0,
1808 }, cqe_write);
1809 try testing.expectEqual(linux.io_uring_cqe{
1810 .user_data = 0x12121212,
1811 .res = @as(i32, @intCast(buffers[1].iov_len)),
1812 .flags = 0,
1813 }, cqe_read);
1814
1815 try testing.expectEqualSlices(u8, "\x00\x00\x00", buffers[1].iov_base[0..3]);
1816 try testing.expectEqualSlices(u8, "foobar", buffers[1].iov_base[3..9]);
1817 try testing.expectEqualSlices(u8, "zz", buffers[1].iov_base[9..11]);
1818}
1819
1820test "openat" {
1821 if (builtin.os.tag != .linux) return error.SkipZigTest;
1822
1823 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1824 error.SystemOutdated => return error.SkipZigTest,
1825 error.PermissionDenied => return error.SkipZigTest,
1826 else => return err,
1827 };
1828 defer ring.deinit();
1829
1830 var tmp = std.testing.tmpDir(.{});
1831 defer tmp.cleanup();
1832
1833 const path = "test_io_uring_openat";
1834
1835 // Workaround for LLVM bug: https://github.com/ziglang/zig/issues/12014
1836 const path_addr = if (builtin.zig_backend == .stage2_llvm) p: {
1837 var workaround = path;
1838 _ = &workaround;
1839 break :p @intFromPtr(workaround);
1840 } else @intFromPtr(path);
1841
1842 const flags: linux.O = .{ .CLOEXEC = true, .ACCMODE = .RDWR, .CREAT = true };
1843 const mode: os.mode_t = 0o666;
1844 const sqe_openat = try ring.openat(0x33333333, tmp.dir.fd, path, flags, mode);
1845 try testing.expectEqual(linux.io_uring_sqe{
1846 .opcode = .OPENAT,
1847 .flags = 0,
1848 .ioprio = 0,
1849 .fd = tmp.dir.fd,
1850 .off = 0,
1851 .addr = path_addr,
1852 .len = mode,
1853 .rw_flags = @bitCast(flags),
1854 .user_data = 0x33333333,
1855 .buf_index = 0,
1856 .personality = 0,
1857 .splice_fd_in = 0,
1858 .addr3 = 0,
1859 .resv = 0,
1860 }, sqe_openat.*);
1861 try testing.expectEqual(@as(u32, 1), try ring.submit());
1862
1863 const cqe_openat = try ring.copy_cqe();
1864 try testing.expectEqual(@as(u64, 0x33333333), cqe_openat.user_data);
1865 if (cqe_openat.err() == .INVAL) return error.SkipZigTest;
1866 if (cqe_openat.err() == .BADF) return error.SkipZigTest;
1867 if (cqe_openat.res <= 0) std.debug.print("\ncqe_openat.res={}\n", .{cqe_openat.res});
1868 try testing.expect(cqe_openat.res > 0);
1869 try testing.expectEqual(@as(u32, 0), cqe_openat.flags);
1870
1871 os.close(cqe_openat.res);
1872}
1873
1874test "close" {
1875 if (builtin.os.tag != .linux) return error.SkipZigTest;
1876
1877 var ring = IoUring.init(1, 0) catch |err| switch (err) {
1878 error.SystemOutdated => return error.SkipZigTest,
1879 error.PermissionDenied => return error.SkipZigTest,
1880 else => return err,
1881 };
1882 defer ring.deinit();
1883
1884 var tmp = std.testing.tmpDir(.{});
1885 defer tmp.cleanup();
1886
1887 const path = "test_io_uring_close";
1888 const file = try tmp.dir.createFile(path, .{});
1889 errdefer file.close();
1890
1891 const sqe_close = try ring.close(0x44444444, file.handle);
1892 try testing.expectEqual(linux.IORING_OP.CLOSE, sqe_close.opcode);
1893 try testing.expectEqual(file.handle, sqe_close.fd);
1894 try testing.expectEqual(@as(u32, 1), try ring.submit());
1895
1896 const cqe_close = try ring.copy_cqe();
1897 if (cqe_close.err() == .INVAL) return error.SkipZigTest;
1898 try testing.expectEqual(linux.io_uring_cqe{
1899 .user_data = 0x44444444,
1900 .res = 0,
1901 .flags = 0,
1902 }, cqe_close);
1903}
1904
1905test "accept/connect/send/recv" {
1906 if (builtin.os.tag != .linux) return error.SkipZigTest;
1907
1908 var ring = IoUring.init(16, 0) catch |err| switch (err) {
1909 error.SystemOutdated => return error.SkipZigTest,
1910 error.PermissionDenied => return error.SkipZigTest,
1911 else => return err,
1912 };
1913 defer ring.deinit();
1914
1915 const socket_test_harness = try createSocketTestHarness(&ring);
1916 defer socket_test_harness.close();
1917
1918 const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
1919 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
1920
1921 const sqe_send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
1922 sqe_send.flags |= linux.IOSQE_IO_LINK;
1923 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
1924 try testing.expectEqual(@as(u32, 2), try ring.submit());
1925
1926 const cqe_send = try ring.copy_cqe();
1927 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
1928 try testing.expectEqual(linux.io_uring_cqe{
1929 .user_data = 0xeeeeeeee,
1930 .res = buffer_send.len,
1931 .flags = 0,
1932 }, cqe_send);
1933
1934 const cqe_recv = try ring.copy_cqe();
1935 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
1936 try testing.expectEqual(linux.io_uring_cqe{
1937 .user_data = 0xffffffff,
1938 .res = buffer_recv.len,
1939 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is only set on some systems
1940 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
1941 }, cqe_recv);
1942
1943 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
1944}
1945
1946test "sendmsg/recvmsg" {
1947 if (builtin.os.tag != .linux) return error.SkipZigTest;
1948
1949 var ring = IoUring.init(2, 0) catch |err| switch (err) {
1950 error.SystemOutdated => return error.SkipZigTest,
1951 error.PermissionDenied => return error.SkipZigTest,
1952 else => return err,
1953 };
1954 defer ring.deinit();
1955
1956 var address_server = try net.Address.parseIp4("127.0.0.1", 0);
1957
1958 const server = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
1959 defer os.close(server);
1960 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1)));
1961 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
1962 try os.bind(server, &address_server.any, address_server.getOsSockLen());
1963
1964 // set address_server to the OS-chosen IP/port.
1965 var slen: os.socklen_t = address_server.getOsSockLen();
1966 try os.getsockname(server, &address_server.any, &slen);
1967
1968 const client = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
1969 defer os.close(client);
1970
1971 const buffer_send = [_]u8{42} ** 128;
1972 const iovecs_send = [_]os.iovec_const{
1973 os.iovec_const{ .iov_base = &buffer_send, .iov_len = buffer_send.len },
1974 };
1975 const msg_send = os.msghdr_const{
1976 .name = &address_server.any,
1977 .namelen = address_server.getOsSockLen(),
1978 .iov = &iovecs_send,
1979 .iovlen = 1,
1980 .control = null,
1981 .controllen = 0,
1982 .flags = 0,
1983 };
1984 const sqe_sendmsg = try ring.sendmsg(0x11111111, client, &msg_send, 0);
1985 sqe_sendmsg.flags |= linux.IOSQE_IO_LINK;
1986 try testing.expectEqual(linux.IORING_OP.SENDMSG, sqe_sendmsg.opcode);
1987 try testing.expectEqual(client, sqe_sendmsg.fd);
1988
1989 var buffer_recv = [_]u8{0} ** 128;
1990 var iovecs_recv = [_]os.iovec{
1991 os.iovec{ .iov_base = &buffer_recv, .iov_len = buffer_recv.len },
1992 };
1993 const addr = [_]u8{0} ** 4;
1994 var address_recv = net.Address.initIp4(addr, 0);
1995 var msg_recv: os.msghdr = os.msghdr{
1996 .name = &address_recv.any,
1997 .namelen = address_recv.getOsSockLen(),
1998 .iov = &iovecs_recv,
1999 .iovlen = 1,
2000 .control = null,
2001 .controllen = 0,
2002 .flags = 0,
2003 };
2004 const sqe_recvmsg = try ring.recvmsg(0x22222222, server, &msg_recv, 0);
2005 try testing.expectEqual(linux.IORING_OP.RECVMSG, sqe_recvmsg.opcode);
2006 try testing.expectEqual(server, sqe_recvmsg.fd);
2007
2008 try testing.expectEqual(@as(u32, 2), ring.sq_ready());
2009 try testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
2010 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2011 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2012
2013 const cqe_sendmsg = try ring.copy_cqe();
2014 if (cqe_sendmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2015 try testing.expectEqual(linux.io_uring_cqe{
2016 .user_data = 0x11111111,
2017 .res = buffer_send.len,
2018 .flags = 0,
2019 }, cqe_sendmsg);
2020
2021 const cqe_recvmsg = try ring.copy_cqe();
2022 if (cqe_recvmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2023 try testing.expectEqual(linux.io_uring_cqe{
2024 .user_data = 0x22222222,
2025 .res = buffer_recv.len,
2026 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is set non-deterministically
2027 .flags = cqe_recvmsg.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2028 }, cqe_recvmsg);
2029
2030 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2031}
2032
2033test "timeout (after a relative time)" {
2034 if (builtin.os.tag != .linux) return error.SkipZigTest;
2035
2036 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2037 error.SystemOutdated => return error.SkipZigTest,
2038 error.PermissionDenied => return error.SkipZigTest,
2039 else => return err,
2040 };
2041 defer ring.deinit();
2042
2043 const ms = 10;
2044 const margin = 5;
2045 const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = ms * 1000000 };
2046
2047 const started = std.time.milliTimestamp();
2048 const sqe = try ring.timeout(0x55555555, &ts, 0, 0);
2049 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode);
2050 try testing.expectEqual(@as(u32, 1), try ring.submit());
2051 const cqe = try ring.copy_cqe();
2052 const stopped = std.time.milliTimestamp();
2053
2054 try testing.expectEqual(linux.io_uring_cqe{
2055 .user_data = 0x55555555,
2056 .res = -@as(i32, @intFromEnum(linux.E.TIME)),
2057 .flags = 0,
2058 }, cqe);
2059
2060 // Tests should not depend on timings: skip test if outside margin.
2061 if (!std.math.approxEqAbs(f64, ms, @as(f64, @floatFromInt(stopped - started)), margin)) return error.SkipZigTest;
2062}
2063
2064test "timeout (after a number of completions)" {
2065 if (builtin.os.tag != .linux) return error.SkipZigTest;
2066
2067 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2068 error.SystemOutdated => return error.SkipZigTest,
2069 error.PermissionDenied => return error.SkipZigTest,
2070 else => return err,
2071 };
2072 defer ring.deinit();
2073
2074 const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
2075 const count_completions: u64 = 1;
2076 const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0);
2077 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2078 try testing.expectEqual(count_completions, sqe_timeout.off);
2079 _ = try ring.nop(0x77777777);
2080 try testing.expectEqual(@as(u32, 2), try ring.submit());
2081
2082 const cqe_nop = try ring.copy_cqe();
2083 try testing.expectEqual(linux.io_uring_cqe{
2084 .user_data = 0x77777777,
2085 .res = 0,
2086 .flags = 0,
2087 }, cqe_nop);
2088
2089 const cqe_timeout = try ring.copy_cqe();
2090 try testing.expectEqual(linux.io_uring_cqe{
2091 .user_data = 0x66666666,
2092 .res = 0,
2093 .flags = 0,
2094 }, cqe_timeout);
2095}
2096
2097test "timeout_remove" {
2098 if (builtin.os.tag != .linux) return error.SkipZigTest;
2099
2100 var ring = IoUring.init(2, 0) catch |err| switch (err) {
2101 error.SystemOutdated => return error.SkipZigTest,
2102 error.PermissionDenied => return error.SkipZigTest,
2103 else => return err,
2104 };
2105 defer ring.deinit();
2106
2107 const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
2108 const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0);
2109 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2110 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data);
2111
2112 const sqe_timeout_remove = try ring.timeout_remove(0x99999999, 0x88888888, 0);
2113 try testing.expectEqual(linux.IORING_OP.TIMEOUT_REMOVE, sqe_timeout_remove.opcode);
2114 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout_remove.addr);
2115 try testing.expectEqual(@as(u64, 0x99999999), sqe_timeout_remove.user_data);
2116
2117 try testing.expectEqual(@as(u32, 2), try ring.submit());
2118
2119 // The order in which the CQE arrive is not clearly documented and it changed with kernel 5.18:
2120 // * kernel 5.10 gives user data 0x88888888 first, 0x99999999 second
2121 // * kernel 5.18 gives user data 0x99999999 first, 0x88888888 second
2122
2123 var cqes: [2]os.linux.io_uring_cqe = undefined;
2124 cqes[0] = try ring.copy_cqe();
2125 cqes[1] = try ring.copy_cqe();
2126
2127 for (cqes) |cqe| {
2128 // IORING_OP_TIMEOUT_REMOVE is not supported by this kernel version:
2129 // Timeout remove operations set the fd to -1, which results in EBADF before EINVAL.
2130 // We use IORING_FEAT_RW_CUR_POS as a safety check here to make sure we are at least pre-5.6.
2131 // We don't want to skip this test for newer kernels.
2132 if (cqe.user_data == 0x99999999 and
2133 cqe.err() == .BADF and
2134 (ring.features & linux.IORING_FEAT_RW_CUR_POS) == 0)
2135 {
2136 return error.SkipZigTest;
2137 }
2138
2139 try testing.expect(cqe.user_data == 0x88888888 or cqe.user_data == 0x99999999);
2140
2141 if (cqe.user_data == 0x88888888) {
2142 try testing.expectEqual(linux.io_uring_cqe{
2143 .user_data = 0x88888888,
2144 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2145 .flags = 0,
2146 }, cqe);
2147 } else if (cqe.user_data == 0x99999999) {
2148 try testing.expectEqual(linux.io_uring_cqe{
2149 .user_data = 0x99999999,
2150 .res = 0,
2151 .flags = 0,
2152 }, cqe);
2153 }
2154 }
2155}
2156
2157test "accept/connect/recv/link_timeout" {
2158 if (builtin.os.tag != .linux) return error.SkipZigTest;
2159
2160 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2161 error.SystemOutdated => return error.SkipZigTest,
2162 error.PermissionDenied => return error.SkipZigTest,
2163 else => return err,
2164 };
2165 defer ring.deinit();
2166
2167 const socket_test_harness = try createSocketTestHarness(&ring);
2168 defer socket_test_harness.close();
2169
2170 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2171
2172 const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2173 sqe_recv.flags |= linux.IOSQE_IO_LINK;
2174
2175 const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = 1000000 };
2176 _ = try ring.link_timeout(0x22222222, &ts, 0);
2177
2178 const nr_wait = try ring.submit();
2179 try testing.expectEqual(@as(u32, 2), nr_wait);
2180
2181 var i: usize = 0;
2182 while (i < nr_wait) : (i += 1) {
2183 const cqe = try ring.copy_cqe();
2184 switch (cqe.user_data) {
2185 0xffffffff => {
2186 if (cqe.res != -@as(i32, @intFromEnum(linux.E.INTR)) and
2187 cqe.res != -@as(i32, @intFromEnum(linux.E.CANCELED)))
2188 {
2189 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2190 try testing.expect(false);
2191 }
2192 },
2193 0x22222222 => {
2194 if (cqe.res != -@as(i32, @intFromEnum(linux.E.ALREADY)) and
2195 cqe.res != -@as(i32, @intFromEnum(linux.E.TIME)))
2196 {
2197 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2198 try testing.expect(false);
2199 }
2200 },
2201 else => @panic("should not happen"),
2202 }
2203 }
2204}
2205
2206test "fallocate" {
2207 if (builtin.os.tag != .linux) return error.SkipZigTest;
2208
2209 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2210 error.SystemOutdated => return error.SkipZigTest,
2211 error.PermissionDenied => return error.SkipZigTest,
2212 else => return err,
2213 };
2214 defer ring.deinit();
2215
2216 var tmp = std.testing.tmpDir(.{});
2217 defer tmp.cleanup();
2218
2219 const path = "test_io_uring_fallocate";
2220 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2221 defer file.close();
2222
2223 try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
2224
2225 const len: u64 = 65536;
2226 const sqe = try ring.fallocate(0xaaaaaaaa, file.handle, 0, 0, len);
2227 try testing.expectEqual(linux.IORING_OP.FALLOCATE, sqe.opcode);
2228 try testing.expectEqual(file.handle, sqe.fd);
2229 try testing.expectEqual(@as(u32, 1), try ring.submit());
2230
2231 const cqe = try ring.copy_cqe();
2232 switch (cqe.err()) {
2233 .SUCCESS => {},
2234 // This kernel's io_uring does not yet implement fallocate():
2235 .INVAL => return error.SkipZigTest,
2236 // This kernel does not implement fallocate():
2237 .NOSYS => return error.SkipZigTest,
2238 // The filesystem containing the file referred to by fd does not support this operation;
2239 // or the mode is not supported by the filesystem containing the file referred to by fd:
2240 .OPNOTSUPP => return error.SkipZigTest,
2241 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2242 }
2243 try testing.expectEqual(linux.io_uring_cqe{
2244 .user_data = 0xaaaaaaaa,
2245 .res = 0,
2246 .flags = 0,
2247 }, cqe);
2248
2249 try testing.expectEqual(len, (try file.stat()).size);
2250}
2251
2252test "statx" {
2253 if (builtin.os.tag != .linux) return error.SkipZigTest;
2254
2255 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2256 error.SystemOutdated => return error.SkipZigTest,
2257 error.PermissionDenied => return error.SkipZigTest,
2258 else => return err,
2259 };
2260 defer ring.deinit();
2261
2262 var tmp = std.testing.tmpDir(.{});
2263 defer tmp.cleanup();
2264 const path = "test_io_uring_statx";
2265 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2266 defer file.close();
2267
2268 try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
2269
2270 try file.writeAll("foobar");
2271
2272 var buf: linux.Statx = undefined;
2273 const sqe = try ring.statx(
2274 0xaaaaaaaa,
2275 tmp.dir.fd,
2276 path,
2277 0,
2278 linux.STATX_SIZE,
2279 &buf,
2280 );
2281 try testing.expectEqual(linux.IORING_OP.STATX, sqe.opcode);
2282 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2283 try testing.expectEqual(@as(u32, 1), try ring.submit());
2284
2285 const cqe = try ring.copy_cqe();
2286 switch (cqe.err()) {
2287 .SUCCESS => {},
2288 // This kernel's io_uring does not yet implement statx():
2289 .INVAL => return error.SkipZigTest,
2290 // This kernel does not implement statx():
2291 .NOSYS => return error.SkipZigTest,
2292 // The filesystem containing the file referred to by fd does not support this operation;
2293 // or the mode is not supported by the filesystem containing the file referred to by fd:
2294 .OPNOTSUPP => return error.SkipZigTest,
2295 // not supported on older kernels (5.4)
2296 .BADF => return error.SkipZigTest,
2297 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2298 }
2299 try testing.expectEqual(linux.io_uring_cqe{
2300 .user_data = 0xaaaaaaaa,
2301 .res = 0,
2302 .flags = 0,
2303 }, cqe);
2304
2305 try testing.expect(buf.mask & os.linux.STATX_SIZE == os.linux.STATX_SIZE);
2306 try testing.expectEqual(@as(u64, 6), buf.size);
2307}
2308
2309test "accept/connect/recv/cancel" {
2310 if (builtin.os.tag != .linux) return error.SkipZigTest;
2311
2312 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2313 error.SystemOutdated => return error.SkipZigTest,
2314 error.PermissionDenied => return error.SkipZigTest,
2315 else => return err,
2316 };
2317 defer ring.deinit();
2318
2319 const socket_test_harness = try createSocketTestHarness(&ring);
2320 defer socket_test_harness.close();
2321
2322 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2323
2324 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2325 try testing.expectEqual(@as(u32, 1), try ring.submit());
2326
2327 const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
2328 try testing.expectEqual(linux.IORING_OP.ASYNC_CANCEL, sqe_cancel.opcode);
2329 try testing.expectEqual(@as(u64, 0xffffffff), sqe_cancel.addr);
2330 try testing.expectEqual(@as(u64, 0x99999999), sqe_cancel.user_data);
2331 try testing.expectEqual(@as(u32, 1), try ring.submit());
2332
2333 var cqe_recv = try ring.copy_cqe();
2334 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2335 var cqe_cancel = try ring.copy_cqe();
2336 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
2337
2338 // The recv/cancel CQEs may arrive in any order, the recv CQE will sometimes come first:
2339 if (cqe_recv.user_data == 0x99999999 and cqe_cancel.user_data == 0xffffffff) {
2340 const a = cqe_recv;
2341 const b = cqe_cancel;
2342 cqe_recv = b;
2343 cqe_cancel = a;
2344 }
2345
2346 try testing.expectEqual(linux.io_uring_cqe{
2347 .user_data = 0xffffffff,
2348 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2349 .flags = 0,
2350 }, cqe_recv);
2351
2352 try testing.expectEqual(linux.io_uring_cqe{
2353 .user_data = 0x99999999,
2354 .res = 0,
2355 .flags = 0,
2356 }, cqe_cancel);
2357}
2358
2359test "register_files_update" {
2360 if (builtin.os.tag != .linux) return error.SkipZigTest;
2361
2362 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2363 error.SystemOutdated => return error.SkipZigTest,
2364 error.PermissionDenied => return error.SkipZigTest,
2365 else => return err,
2366 };
2367 defer ring.deinit();
2368
2369 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2370 defer os.close(fd);
2371
2372 var registered_fds = [_]os.fd_t{0} ** 2;
2373 const fd_index = 0;
2374 const fd_index2 = 1;
2375 registered_fds[fd_index] = fd;
2376 registered_fds[fd_index2] = -1;
2377
2378 ring.register_files(registered_fds[0..]) catch |err| switch (err) {
2379 // Happens when the kernel doesn't support sparse entry (-1) in the file descriptors array.
2380 error.FileDescriptorInvalid => return error.SkipZigTest,
2381 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2382 };
2383
2384 // Test IORING_REGISTER_FILES_UPDATE
2385 // Only available since Linux 5.5
2386
2387 const fd2 = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2388 defer os.close(fd2);
2389
2390 registered_fds[fd_index] = fd2;
2391 registered_fds[fd_index2] = -1;
2392 try ring.register_files_update(0, registered_fds[0..]);
2393
2394 var buffer = [_]u8{42} ** 128;
2395 {
2396 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2397 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2398 sqe.flags |= linux.IOSQE_FIXED_FILE;
2399
2400 try testing.expectEqual(@as(u32, 1), try ring.submit());
2401 try testing.expectEqual(linux.io_uring_cqe{
2402 .user_data = 0xcccccccc,
2403 .res = buffer.len,
2404 .flags = 0,
2405 }, try ring.copy_cqe());
2406 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2407 }
2408
2409 // Test with a non-zero offset
2410
2411 registered_fds[fd_index] = -1;
2412 registered_fds[fd_index2] = -1;
2413 try ring.register_files_update(1, registered_fds[1..]);
2414
2415 {
2416 // Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
2417 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2418 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2419 sqe.flags |= linux.IOSQE_FIXED_FILE;
2420
2421 try testing.expectEqual(@as(u32, 1), try ring.submit());
2422 try testing.expectEqual(linux.io_uring_cqe{
2423 .user_data = 0xcccccccc,
2424 .res = buffer.len,
2425 .flags = 0,
2426 }, try ring.copy_cqe());
2427 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2428 }
2429
2430 try ring.register_files_update(0, registered_fds[0..]);
2431
2432 {
2433 // Now this should fail since both fds are sparse (-1)
2434 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2435 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2436 sqe.flags |= linux.IOSQE_FIXED_FILE;
2437
2438 try testing.expectEqual(@as(u32, 1), try ring.submit());
2439 const cqe = try ring.copy_cqe();
2440 try testing.expectEqual(os.linux.E.BADF, cqe.err());
2441 }
2442
2443 try ring.unregister_files();
2444}
2445
2446test "shutdown" {
2447 if (builtin.os.tag != .linux) return error.SkipZigTest;
2448
2449 var ring = IoUring.init(16, 0) catch |err| switch (err) {
2450 error.SystemOutdated => return error.SkipZigTest,
2451 error.PermissionDenied => return error.SkipZigTest,
2452 else => return err,
2453 };
2454 defer ring.deinit();
2455
2456 var address = try net.Address.parseIp4("127.0.0.1", 0);
2457
2458 // Socket bound, expect shutdown to work
2459 {
2460 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
2461 defer os.close(server);
2462 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2463 try os.bind(server, &address.any, address.getOsSockLen());
2464 try os.listen(server, 1);
2465
2466 // set address to the OS-chosen IP/port.
2467 var slen: os.socklen_t = address.getOsSockLen();
2468 try os.getsockname(server, &address.any, &slen);
2469
2470 const shutdown_sqe = try ring.shutdown(0x445445445, server, os.linux.SHUT.RD);
2471 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
2472 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
2473
2474 try testing.expectEqual(@as(u32, 1), try ring.submit());
2475
2476 const cqe = try ring.copy_cqe();
2477 switch (cqe.err()) {
2478 .SUCCESS => {},
2479 // This kernel's io_uring does not yet implement shutdown (kernel version < 5.11)
2480 .INVAL => return error.SkipZigTest,
2481 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2482 }
2483
2484 try testing.expectEqual(linux.io_uring_cqe{
2485 .user_data = 0x445445445,
2486 .res = 0,
2487 .flags = 0,
2488 }, cqe);
2489 }
2490
2491 // Socket not bound, expect to fail with ENOTCONN
2492 {
2493 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
2494 defer os.close(server);
2495
2496 const shutdown_sqe = ring.shutdown(0x445445445, server, os.linux.SHUT.RD) catch |err| switch (err) {
2497 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2498 };
2499 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
2500 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
2501
2502 try testing.expectEqual(@as(u32, 1), try ring.submit());
2503
2504 const cqe = try ring.copy_cqe();
2505 try testing.expectEqual(@as(u64, 0x445445445), cqe.user_data);
2506 try testing.expectEqual(os.linux.E.NOTCONN, cqe.err());
2507 }
2508}
2509
2510test "renameat" {
2511 if (builtin.os.tag != .linux) return error.SkipZigTest;
2512
2513 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2514 error.SystemOutdated => return error.SkipZigTest,
2515 error.PermissionDenied => return error.SkipZigTest,
2516 else => return err,
2517 };
2518 defer ring.deinit();
2519
2520 const old_path = "test_io_uring_renameat_old";
2521 const new_path = "test_io_uring_renameat_new";
2522
2523 var tmp = std.testing.tmpDir(.{});
2524 defer tmp.cleanup();
2525
2526 // Write old file with data
2527
2528 const old_file = try tmp.dir.createFile(old_path, .{ .truncate = true, .mode = 0o666 });
2529 defer old_file.close();
2530 try old_file.writeAll("hello");
2531
2532 // Submit renameat
2533
2534 const sqe = try ring.renameat(
2535 0x12121212,
2536 tmp.dir.fd,
2537 old_path,
2538 tmp.dir.fd,
2539 new_path,
2540 0,
2541 );
2542 try testing.expectEqual(linux.IORING_OP.RENAMEAT, sqe.opcode);
2543 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2544 try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
2545 try testing.expectEqual(@as(u32, 1), try ring.submit());
2546
2547 const cqe = try ring.copy_cqe();
2548 switch (cqe.err()) {
2549 .SUCCESS => {},
2550 // This kernel's io_uring does not yet implement renameat (kernel version < 5.11)
2551 .BADF, .INVAL => return error.SkipZigTest,
2552 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2553 }
2554 try testing.expectEqual(linux.io_uring_cqe{
2555 .user_data = 0x12121212,
2556 .res = 0,
2557 .flags = 0,
2558 }, cqe);
2559
2560 // Validate that the old file doesn't exist anymore
2561 {
2562 _ = tmp.dir.openFile(old_path, .{}) catch |err| switch (err) {
2563 error.FileNotFound => {},
2564 else => std.debug.panic("unexpected error: {}", .{err}),
2565 };
2566 }
2567
2568 // Validate that the new file exists with the proper content
2569 {
2570 const new_file = try tmp.dir.openFile(new_path, .{});
2571 defer new_file.close();
2572
2573 var new_file_data: [16]u8 = undefined;
2574 const bytes_read = try new_file.readAll(&new_file_data);
2575 try testing.expectEqualStrings("hello", new_file_data[0..bytes_read]);
2576 }
2577}
2578
2579test "unlinkat" {
2580 if (builtin.os.tag != .linux) return error.SkipZigTest;
2581
2582 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2583 error.SystemOutdated => return error.SkipZigTest,
2584 error.PermissionDenied => return error.SkipZigTest,
2585 else => return err,
2586 };
2587 defer ring.deinit();
2588
2589 const path = "test_io_uring_unlinkat";
2590
2591 var tmp = std.testing.tmpDir(.{});
2592 defer tmp.cleanup();
2593
2594 // Write old file with data
2595
2596 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2597 defer file.close();
2598
2599 // Submit unlinkat
2600
2601 const sqe = try ring.unlinkat(
2602 0x12121212,
2603 tmp.dir.fd,
2604 path,
2605 0,
2606 );
2607 try testing.expectEqual(linux.IORING_OP.UNLINKAT, sqe.opcode);
2608 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2609 try testing.expectEqual(@as(u32, 1), try ring.submit());
2610
2611 const cqe = try ring.copy_cqe();
2612 switch (cqe.err()) {
2613 .SUCCESS => {},
2614 // This kernel's io_uring does not yet implement unlinkat (kernel version < 5.11)
2615 .BADF, .INVAL => return error.SkipZigTest,
2616 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2617 }
2618 try testing.expectEqual(linux.io_uring_cqe{
2619 .user_data = 0x12121212,
2620 .res = 0,
2621 .flags = 0,
2622 }, cqe);
2623
2624 // Validate that the file doesn't exist anymore
2625 _ = tmp.dir.openFile(path, .{}) catch |err| switch (err) {
2626 error.FileNotFound => {},
2627 else => std.debug.panic("unexpected error: {}", .{err}),
2628 };
2629}
2630
2631test "mkdirat" {
2632 if (builtin.os.tag != .linux) return error.SkipZigTest;
2633
2634 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2635 error.SystemOutdated => return error.SkipZigTest,
2636 error.PermissionDenied => return error.SkipZigTest,
2637 else => return err,
2638 };
2639 defer ring.deinit();
2640
2641 var tmp = std.testing.tmpDir(.{});
2642 defer tmp.cleanup();
2643
2644 const path = "test_io_uring_mkdirat";
2645
2646 // Submit mkdirat
2647
2648 const sqe = try ring.mkdirat(
2649 0x12121212,
2650 tmp.dir.fd,
2651 path,
2652 0o0755,
2653 );
2654 try testing.expectEqual(linux.IORING_OP.MKDIRAT, sqe.opcode);
2655 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2656 try testing.expectEqual(@as(u32, 1), try ring.submit());
2657
2658 const cqe = try ring.copy_cqe();
2659 switch (cqe.err()) {
2660 .SUCCESS => {},
2661 // This kernel's io_uring does not yet implement mkdirat (kernel version < 5.15)
2662 .BADF, .INVAL => return error.SkipZigTest,
2663 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2664 }
2665 try testing.expectEqual(linux.io_uring_cqe{
2666 .user_data = 0x12121212,
2667 .res = 0,
2668 .flags = 0,
2669 }, cqe);
2670
2671 // Validate that the directory exist
2672 _ = try tmp.dir.openDir(path, .{});
2673}
2674
2675test "symlinkat" {
2676 if (builtin.os.tag != .linux) return error.SkipZigTest;
2677
2678 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2679 error.SystemOutdated => return error.SkipZigTest,
2680 error.PermissionDenied => return error.SkipZigTest,
2681 else => return err,
2682 };
2683 defer ring.deinit();
2684
2685 var tmp = std.testing.tmpDir(.{});
2686 defer tmp.cleanup();
2687
2688 const path = "test_io_uring_symlinkat";
2689 const link_path = "test_io_uring_symlinkat_link";
2690
2691 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2692 defer file.close();
2693
2694 // Submit symlinkat
2695
2696 const sqe = try ring.symlinkat(
2697 0x12121212,
2698 path,
2699 tmp.dir.fd,
2700 link_path,
2701 );
2702 try testing.expectEqual(linux.IORING_OP.SYMLINKAT, sqe.opcode);
2703 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2704 try testing.expectEqual(@as(u32, 1), try ring.submit());
2705
2706 const cqe = try ring.copy_cqe();
2707 switch (cqe.err()) {
2708 .SUCCESS => {},
2709 // This kernel's io_uring does not yet implement symlinkat (kernel version < 5.15)
2710 .BADF, .INVAL => return error.SkipZigTest,
2711 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2712 }
2713 try testing.expectEqual(linux.io_uring_cqe{
2714 .user_data = 0x12121212,
2715 .res = 0,
2716 .flags = 0,
2717 }, cqe);
2718
2719 // Validate that the symlink exist
2720 _ = try tmp.dir.openFile(link_path, .{});
2721}
2722
2723test "linkat" {
2724 if (builtin.os.tag != .linux) return error.SkipZigTest;
2725
2726 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2727 error.SystemOutdated => return error.SkipZigTest,
2728 error.PermissionDenied => return error.SkipZigTest,
2729 else => return err,
2730 };
2731 defer ring.deinit();
2732
2733 var tmp = std.testing.tmpDir(.{});
2734 defer tmp.cleanup();
2735
2736 const first_path = "test_io_uring_linkat_first";
2737 const second_path = "test_io_uring_linkat_second";
2738
2739 // Write file with data
2740
2741 const first_file = try tmp.dir.createFile(first_path, .{ .truncate = true, .mode = 0o666 });
2742 defer first_file.close();
2743 try first_file.writeAll("hello");
2744
2745 // Submit linkat
2746
2747 const sqe = try ring.linkat(
2748 0x12121212,
2749 tmp.dir.fd,
2750 first_path,
2751 tmp.dir.fd,
2752 second_path,
2753 0,
2754 );
2755 try testing.expectEqual(linux.IORING_OP.LINKAT, sqe.opcode);
2756 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2757 try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
2758 try testing.expectEqual(@as(u32, 1), try ring.submit());
2759
2760 const cqe = try ring.copy_cqe();
2761 switch (cqe.err()) {
2762 .SUCCESS => {},
2763 // This kernel's io_uring does not yet implement linkat (kernel version < 5.15)
2764 .BADF, .INVAL => return error.SkipZigTest,
2765 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2766 }
2767 try testing.expectEqual(linux.io_uring_cqe{
2768 .user_data = 0x12121212,
2769 .res = 0,
2770 .flags = 0,
2771 }, cqe);
2772
2773 // Validate the second file
2774 const second_file = try tmp.dir.openFile(second_path, .{});
2775 defer second_file.close();
2776
2777 var second_file_data: [16]u8 = undefined;
2778 const bytes_read = try second_file.readAll(&second_file_data);
2779 try testing.expectEqualStrings("hello", second_file_data[0..bytes_read]);
2780}
2781
2782test "provide_buffers: read" {
2783 if (builtin.os.tag != .linux) return error.SkipZigTest;
2784
2785 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2786 error.SystemOutdated => return error.SkipZigTest,
2787 error.PermissionDenied => return error.SkipZigTest,
2788 else => return err,
2789 };
2790 defer ring.deinit();
2791
2792 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2793 defer os.close(fd);
2794
2795 const group_id = 1337;
2796 const buffer_id = 0;
2797
2798 const buffer_len = 128;
2799
2800 var buffers: [4][buffer_len]u8 = undefined;
2801
2802 // Provide 4 buffers
2803
2804 {
2805 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
2806 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
2807 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
2808 try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
2809 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2810 try testing.expectEqual(@as(u32, 1), try ring.submit());
2811
2812 const cqe = try ring.copy_cqe();
2813 switch (cqe.err()) {
2814 // Happens when the kernel is < 5.7
2815 .INVAL => return error.SkipZigTest,
2816 .SUCCESS => {},
2817 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2818 }
2819 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
2820 }
2821
2822 // Do 4 reads which should consume all buffers
2823
2824 var i: usize = 0;
2825 while (i < buffers.len) : (i += 1) {
2826 const sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2827 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2828 try testing.expectEqual(@as(i32, fd), sqe.fd);
2829 try testing.expectEqual(@as(u64, 0), sqe.addr);
2830 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
2831 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2832 try testing.expectEqual(@as(u32, 1), try ring.submit());
2833
2834 const cqe = try ring.copy_cqe();
2835 switch (cqe.err()) {
2836 .SUCCESS => {},
2837 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2838 }
2839
2840 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
2841 const used_buffer_id = cqe.flags >> 16;
2842 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
2843 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
2844
2845 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
2846 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
2847 }
2848
2849 // This read should fail
2850
2851 {
2852 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2853 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2854 try testing.expectEqual(@as(i32, fd), sqe.fd);
2855 try testing.expectEqual(@as(u64, 0), sqe.addr);
2856 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
2857 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2858 try testing.expectEqual(@as(u32, 1), try ring.submit());
2859
2860 const cqe = try ring.copy_cqe();
2861 switch (cqe.err()) {
2862 // Expected
2863 .NOBUFS => {},
2864 .SUCCESS => std.debug.panic("unexpected success", .{}),
2865 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2866 }
2867 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
2868 }
2869
2870 // Provide 1 buffer again
2871
2872 // Deliberately put something we don't expect in the buffers
2873 @memset(mem.sliceAsBytes(&buffers), 42);
2874
2875 const reprovided_buffer_id = 2;
2876
2877 {
2878 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
2879 try testing.expectEqual(@as(u32, 1), try ring.submit());
2880
2881 const cqe = try ring.copy_cqe();
2882 switch (cqe.err()) {
2883 .SUCCESS => {},
2884 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2885 }
2886 }
2887
2888 // Final read which should work
2889
2890 {
2891 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2892 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2893 try testing.expectEqual(@as(i32, fd), sqe.fd);
2894 try testing.expectEqual(@as(u64, 0), sqe.addr);
2895 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
2896 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2897 try testing.expectEqual(@as(u32, 1), try ring.submit());
2898
2899 const cqe = try ring.copy_cqe();
2900 switch (cqe.err()) {
2901 .SUCCESS => {},
2902 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2903 }
2904
2905 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
2906 const used_buffer_id = cqe.flags >> 16;
2907 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
2908 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
2909 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
2910 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
2911 }
2912}
2913
2914test "remove_buffers" {
2915 if (builtin.os.tag != .linux) return error.SkipZigTest;
2916
2917 var ring = IoUring.init(1, 0) catch |err| switch (err) {
2918 error.SystemOutdated => return error.SkipZigTest,
2919 error.PermissionDenied => return error.SkipZigTest,
2920 else => return err,
2921 };
2922 defer ring.deinit();
2923
2924 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2925 defer os.close(fd);
2926
2927 const group_id = 1337;
2928 const buffer_id = 0;
2929
2930 const buffer_len = 128;
2931
2932 var buffers: [4][buffer_len]u8 = undefined;
2933
2934 // Provide 4 buffers
2935
2936 {
2937 _ = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
2938 try testing.expectEqual(@as(u32, 1), try ring.submit());
2939
2940 const cqe = try ring.copy_cqe();
2941 switch (cqe.err()) {
2942 .INVAL => return error.SkipZigTest,
2943 .SUCCESS => {},
2944 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2945 }
2946 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
2947 }
2948
2949 // Remove 3 buffers
2950
2951 {
2952 const sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
2953 try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
2954 try testing.expectEqual(@as(i32, 3), sqe.fd);
2955 try testing.expectEqual(@as(u64, 0), sqe.addr);
2956 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
2957 try testing.expectEqual(@as(u32, 1), try ring.submit());
2958
2959 const cqe = try ring.copy_cqe();
2960 switch (cqe.err()) {
2961 .SUCCESS => {},
2962 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2963 }
2964 try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
2965 }
2966
2967 // This read should work
2968
2969 {
2970 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2971 try testing.expectEqual(@as(u32, 1), try ring.submit());
2972
2973 const cqe = try ring.copy_cqe();
2974 switch (cqe.err()) {
2975 .SUCCESS => {},
2976 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2977 }
2978
2979 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
2980 const used_buffer_id = cqe.flags >> 16;
2981 try testing.expect(used_buffer_id >= 0 and used_buffer_id < 4);
2982 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
2983 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
2984 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
2985 }
2986
2987 // Final read should _not_ work
2988
2989 {
2990 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
2991 try testing.expectEqual(@as(u32, 1), try ring.submit());
2992
2993 const cqe = try ring.copy_cqe();
2994 switch (cqe.err()) {
2995 // Expected
2996 .NOBUFS => {},
2997 .SUCCESS => std.debug.panic("unexpected success", .{}),
2998 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2999 }
3000 }
3001}
3002
3003test "provide_buffers: accept/connect/send/recv" {
3004 if (builtin.os.tag != .linux) return error.SkipZigTest;
3005
3006 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3007 error.SystemOutdated => return error.SkipZigTest,
3008 error.PermissionDenied => return error.SkipZigTest,
3009 else => return err,
3010 };
3011 defer ring.deinit();
3012
3013 const group_id = 1337;
3014 const buffer_id = 0;
3015
3016 const buffer_len = 128;
3017 var buffers: [4][buffer_len]u8 = undefined;
3018
3019 // Provide 4 buffers
3020
3021 {
3022 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3023 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3024 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3025 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3026 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3027 try testing.expectEqual(@as(u32, 1), try ring.submit());
3028
3029 const cqe = try ring.copy_cqe();
3030 switch (cqe.err()) {
3031 // Happens when the kernel is < 5.7
3032 .INVAL => return error.SkipZigTest,
3033 // Happens on the kernel 5.4
3034 .BADF => return error.SkipZigTest,
3035 .SUCCESS => {},
3036 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3037 }
3038 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3039 }
3040
3041 const socket_test_harness = try createSocketTestHarness(&ring);
3042 defer socket_test_harness.close();
3043
3044 // Do 4 send on the socket
3045
3046 {
3047 var i: usize = 0;
3048 while (i < buffers.len) : (i += 1) {
3049 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
3050 try testing.expectEqual(@as(u32, 1), try ring.submit());
3051 }
3052
3053 var cqes: [4]linux.io_uring_cqe = undefined;
3054 try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
3055 }
3056
3057 // Do 4 recv which should consume all buffers
3058
3059 // Deliberately put something we don't expect in the buffers
3060 @memset(mem.sliceAsBytes(&buffers), 1);
3061
3062 var i: usize = 0;
3063 while (i < buffers.len) : (i += 1) {
3064 const sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3065 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3066 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3067 try testing.expectEqual(@as(u64, 0), sqe.addr);
3068 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3069 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3070 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3071 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3072 try testing.expectEqual(@as(u32, 1), try ring.submit());
3073
3074 const cqe = try ring.copy_cqe();
3075 switch (cqe.err()) {
3076 .SUCCESS => {},
3077 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3078 }
3079
3080 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3081 const used_buffer_id = cqe.flags >> 16;
3082 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3083 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3084
3085 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3086 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3087 try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
3088 }
3089
3090 // This recv should fail
3091
3092 {
3093 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3094 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3095 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3096 try testing.expectEqual(@as(u64, 0), sqe.addr);
3097 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3098 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3099 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3100 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3101 try testing.expectEqual(@as(u32, 1), try ring.submit());
3102
3103 const cqe = try ring.copy_cqe();
3104 switch (cqe.err()) {
3105 // Expected
3106 .NOBUFS => {},
3107 .SUCCESS => std.debug.panic("unexpected success", .{}),
3108 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3109 }
3110 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3111 }
3112
3113 // Provide 1 buffer again
3114
3115 const reprovided_buffer_id = 2;
3116
3117 {
3118 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3119 try testing.expectEqual(@as(u32, 1), try ring.submit());
3120
3121 const cqe = try ring.copy_cqe();
3122 switch (cqe.err()) {
3123 .SUCCESS => {},
3124 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3125 }
3126 }
3127
3128 // Redo 1 send on the server socket
3129
3130 {
3131 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
3132 try testing.expectEqual(@as(u32, 1), try ring.submit());
3133
3134 _ = try ring.copy_cqe();
3135 }
3136
3137 // Final recv which should work
3138
3139 // Deliberately put something we don't expect in the buffers
3140 @memset(mem.sliceAsBytes(&buffers), 1);
3141
3142 {
3143 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3144 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3145 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3146 try testing.expectEqual(@as(u64, 0), sqe.addr);
3147 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3148 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3149 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3150 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3151 try testing.expectEqual(@as(u32, 1), try ring.submit());
3152
3153 const cqe = try ring.copy_cqe();
3154 switch (cqe.err()) {
3155 .SUCCESS => {},
3156 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3157 }
3158
3159 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3160 const used_buffer_id = cqe.flags >> 16;
3161 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3162 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3163 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3164 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3165 try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
3166 }
3167}
3168
3169/// Used for testing server/client interactions.
3170const SocketTestHarness = struct {
3171 listener: os.socket_t,
3172 server: os.socket_t,
3173 client: os.socket_t,
3174
3175 fn close(self: SocketTestHarness) void {
3176 posix.close(self.client);
3177 posix.close(self.listener);
3178 }
3179};
3180
3181fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness {
3182 // Create a TCP server socket
3183 var address = try net.Address.parseIp4("127.0.0.1", 0);
3184 const listener_socket = try createListenerSocket(&address);
3185 errdefer posix.close(listener_socket);
3186
3187 // Submit 1 accept
3188 var accept_addr: os.sockaddr = undefined;
3189 var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
3190 _ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
3191
3192 // Create a TCP client socket
3193 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3194 errdefer posix.close(client);
3195 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
3196
3197 try testing.expectEqual(@as(u32, 2), try ring.submit());
3198
3199 var cqe_accept = try ring.copy_cqe();
3200 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
3201 var cqe_connect = try ring.copy_cqe();
3202 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
3203
3204 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
3205 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
3206 const a = cqe_accept;
3207 const b = cqe_connect;
3208 cqe_accept = b;
3209 cqe_connect = a;
3210 }
3211
3212 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
3213 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
3214 try testing.expect(cqe_accept.res > 0);
3215 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
3216 try testing.expectEqual(linux.io_uring_cqe{
3217 .user_data = 0xcccccccc,
3218 .res = 0,
3219 .flags = 0,
3220 }, cqe_connect);
3221
3222 // All good
3223
3224 return SocketTestHarness{
3225 .listener = listener_socket,
3226 .server = cqe_accept.res,
3227 .client = client,
3228 };
3229}
3230
3231fn createListenerSocket(address: *net.Address) !os.socket_t {
3232 const kernel_backlog = 1;
3233 const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3234 errdefer posix.close(listener_socket);
3235
3236 try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3237 try os.bind(listener_socket, &address.any, address.getOsSockLen());
3238 try os.listen(listener_socket, kernel_backlog);
3239
3240 // set address to the OS-chosen IP/port.
3241 var slen: os.socklen_t = address.getOsSockLen();
3242 try os.getsockname(listener_socket, &address.any, &slen);
3243
3244 return listener_socket;
3245}
3246
3247test "accept multishot" {
3248 if (builtin.os.tag != .linux) return error.SkipZigTest;
3249
3250 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3251 error.SystemOutdated => return error.SkipZigTest,
3252 error.PermissionDenied => return error.SkipZigTest,
3253 else => return err,
3254 };
3255 defer ring.deinit();
3256
3257 var address = try net.Address.parseIp4("127.0.0.1", 0);
3258 const listener_socket = try createListenerSocket(&address);
3259 defer posix.close(listener_socket);
3260
3261 // submit multishot accept operation
3262 var addr: os.sockaddr = undefined;
3263 var addr_len: os.socklen_t = @sizeOf(@TypeOf(addr));
3264 const userdata: u64 = 0xaaaaaaaa;
3265 _ = try ring.accept_multishot(userdata, listener_socket, &addr, &addr_len, 0);
3266 try testing.expectEqual(@as(u32, 1), try ring.submit());
3267
3268 var nr: usize = 4; // number of clients to connect
3269 while (nr > 0) : (nr -= 1) {
3270 // connect client
3271 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3272 errdefer posix.close(client);
3273 try os.connect(client, &address.any, address.getOsSockLen());
3274
3275 // test accept completion
3276 var cqe = try ring.copy_cqe();
3277 if (cqe.err() == .INVAL) return error.SkipZigTest;
3278 try testing.expect(cqe.res > 0);
3279 try testing.expect(cqe.user_data == userdata);
3280 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
3281
3282 posix.close(client);
3283 }
3284}
3285
3286test "accept/connect/send_zc/recv" {
3287 try skipKernelLessThan(.{ .major = 6, .minor = 0, .patch = 0 });
3288
3289 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3290 error.SystemOutdated => return error.SkipZigTest,
3291 error.PermissionDenied => return error.SkipZigTest,
3292 else => return err,
3293 };
3294 defer ring.deinit();
3295
3296 const socket_test_harness = try createSocketTestHarness(&ring);
3297 defer socket_test_harness.close();
3298
3299 const buffer_send = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3300 var buffer_recv = [_]u8{0} ** 10;
3301
3302 // zero-copy send
3303 const sqe_send = try ring.send_zc(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0, 0);
3304 sqe_send.flags |= linux.IOSQE_IO_LINK;
3305 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
3306 try testing.expectEqual(@as(u32, 2), try ring.submit());
3307
3308 // First completion of zero-copy send.
3309 // IORING_CQE_F_MORE, means that there
3310 // will be a second completion event / notification for the
3311 // request, with the user_data field set to the same value.
3312 // buffer_send must be keep alive until second cqe.
3313 var cqe_send = try ring.copy_cqe();
3314 try testing.expectEqual(linux.io_uring_cqe{
3315 .user_data = 0xeeeeeeee,
3316 .res = buffer_send.len,
3317 .flags = linux.IORING_CQE_F_MORE,
3318 }, cqe_send);
3319
3320 const cqe_recv = try ring.copy_cqe();
3321 try testing.expectEqual(linux.io_uring_cqe{
3322 .user_data = 0xffffffff,
3323 .res = buffer_recv.len,
3324 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
3325 }, cqe_recv);
3326
3327 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
3328
3329 // Second completion of zero-copy send.
3330 // IORING_CQE_F_NOTIF in flags signals that kernel is done with send_buffer
3331 cqe_send = try ring.copy_cqe();
3332 try testing.expectEqual(linux.io_uring_cqe{
3333 .user_data = 0xeeeeeeee,
3334 .res = 0,
3335 .flags = linux.IORING_CQE_F_NOTIF,
3336 }, cqe_send);
3337}
3338
3339test "accept_direct" {
3340 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3341
3342 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3343 error.SystemOutdated => return error.SkipZigTest,
3344 error.PermissionDenied => return error.SkipZigTest,
3345 else => return err,
3346 };
3347 defer ring.deinit();
3348 var address = try net.Address.parseIp4("127.0.0.1", 0);
3349
3350 // register direct file descriptors
3351 var registered_fds = [_]os.fd_t{-1} ** 2;
3352 try ring.register_files(registered_fds[0..]);
3353
3354 const listener_socket = try createListenerSocket(&address);
3355 defer posix.close(listener_socket);
3356
3357 const accept_userdata: u64 = 0xaaaaaaaa;
3358 const read_userdata: u64 = 0xbbbbbbbb;
3359 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3360
3361 for (0..2) |_| {
3362 for (registered_fds, 0..) |_, i| {
3363 var buffer_recv = [_]u8{0} ** 16;
3364 const buffer_send: []const u8 = data[0 .. data.len - i]; // make it different at each loop
3365
3366 // submit accept, will chose registered fd and return index in cqe
3367 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3368 try testing.expectEqual(@as(u32, 1), try ring.submit());
3369
3370 // connect
3371 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3372 try os.connect(client, &address.any, address.getOsSockLen());
3373 defer posix.close(client);
3374
3375 // accept completion
3376 const cqe_accept = try ring.copy_cqe();
3377 try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
3378 const fd_index = cqe_accept.res;
3379 try testing.expect(fd_index < registered_fds.len);
3380 try testing.expect(cqe_accept.user_data == accept_userdata);
3381
3382 // send data
3383 _ = try os.send(client, buffer_send, 0);
3384
3385 // Example of how to use registered fd:
3386 // Submit receive to fixed file returned by accept (fd_index).
3387 // Fd field is set to registered file index, returned by accept.
3388 // Flag linux.IOSQE_FIXED_FILE must be set.
3389 const recv_sqe = try ring.recv(read_userdata, fd_index, .{ .buffer = &buffer_recv }, 0);
3390 recv_sqe.flags |= linux.IOSQE_FIXED_FILE;
3391 try testing.expectEqual(@as(u32, 1), try ring.submit());
3392
3393 // accept receive
3394 const recv_cqe = try ring.copy_cqe();
3395 try testing.expect(recv_cqe.user_data == read_userdata);
3396 try testing.expect(recv_cqe.res == buffer_send.len);
3397 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
3398 }
3399 // no more available fds, accept will get NFILE error
3400 {
3401 // submit accept
3402 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3403 try testing.expectEqual(@as(u32, 1), try ring.submit());
3404 // connect
3405 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3406 try os.connect(client, &address.any, address.getOsSockLen());
3407 defer posix.close(client);
3408 // completion with error
3409 const cqe_accept = try ring.copy_cqe();
3410 try testing.expect(cqe_accept.user_data == accept_userdata);
3411 try testing.expectEqual(os.E.NFILE, cqe_accept.err());
3412 }
3413 // return file descriptors to kernel
3414 try ring.register_files_update(0, registered_fds[0..]);
3415 }
3416 try ring.unregister_files();
3417}
3418
3419test "accept_multishot_direct" {
3420 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3421
3422 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3423 error.SystemOutdated => return error.SkipZigTest,
3424 error.PermissionDenied => return error.SkipZigTest,
3425 else => return err,
3426 };
3427 defer ring.deinit();
3428
3429 var address = try net.Address.parseIp4("127.0.0.1", 0);
3430
3431 var registered_fds = [_]os.fd_t{-1} ** 2;
3432 try ring.register_files(registered_fds[0..]);
3433
3434 const listener_socket = try createListenerSocket(&address);
3435 defer posix.close(listener_socket);
3436
3437 const accept_userdata: u64 = 0xaaaaaaaa;
3438
3439 for (0..2) |_| {
3440 // submit multishot accept
3441 // Will chose registered fd and return index of the selected registered file in cqe.
3442 _ = try ring.accept_multishot_direct(accept_userdata, listener_socket, null, null, 0);
3443 try testing.expectEqual(@as(u32, 1), try ring.submit());
3444
3445 for (registered_fds) |_| {
3446 // connect
3447 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3448 try os.connect(client, &address.any, address.getOsSockLen());
3449 defer posix.close(client);
3450
3451 // accept completion
3452 const cqe_accept = try ring.copy_cqe();
3453 const fd_index = cqe_accept.res;
3454 try testing.expect(fd_index < registered_fds.len);
3455 try testing.expect(cqe_accept.user_data == accept_userdata);
3456 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE > 0); // has more is set
3457 }
3458 // No more available fds, accept will get NFILE error.
3459 // Multishot is terminated (more flag is not set).
3460 {
3461 // connect
3462 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3463 try os.connect(client, &address.any, address.getOsSockLen());
3464 defer posix.close(client);
3465 // completion with error
3466 const cqe_accept = try ring.copy_cqe();
3467 try testing.expect(cqe_accept.user_data == accept_userdata);
3468 try testing.expectEqual(os.E.NFILE, cqe_accept.err());
3469 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE == 0); // has more is not set
3470 }
3471 // return file descriptors to kernel
3472 try ring.register_files_update(0, registered_fds[0..]);
3473 }
3474 try ring.unregister_files();
3475}
3476
3477test "socket" {
3478 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3479
3480 var ring = IoUring.init(1, 0) catch |err| switch (err) {
3481 error.SystemOutdated => return error.SkipZigTest,
3482 error.PermissionDenied => return error.SkipZigTest,
3483 else => return err,
3484 };
3485 defer ring.deinit();
3486
3487 // prepare, submit socket operation
3488 _ = try ring.socket(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
3489 try testing.expectEqual(@as(u32, 1), try ring.submit());
3490
3491 // test completion
3492 var cqe = try ring.copy_cqe();
3493 try testing.expectEqual(os.E.SUCCESS, cqe.err());
3494 const fd: os.fd_t = @intCast(cqe.res);
3495 try testing.expect(fd > 2);
3496
3497 os.close(fd);
3498}
3499
3500test "socket_direct/socket_direct_alloc/close_direct" {
3501 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3502
3503 var ring = IoUring.init(2, 0) catch |err| switch (err) {
3504 error.SystemOutdated => return error.SkipZigTest,
3505 error.PermissionDenied => return error.SkipZigTest,
3506 else => return err,
3507 };
3508 defer ring.deinit();
3509
3510 var registered_fds = [_]os.fd_t{-1} ** 3;
3511 try ring.register_files(registered_fds[0..]);
3512
3513 // create socket in registered file descriptor at index 0 (last param)
3514 _ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 0);
3515 try testing.expectEqual(@as(u32, 1), try ring.submit());
3516 var cqe_socket = try ring.copy_cqe();
3517 try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
3518 try testing.expect(cqe_socket.res == 0);
3519
3520 // create socket in registered file descriptor at index 1 (last param)
3521 _ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 1);
3522 try testing.expectEqual(@as(u32, 1), try ring.submit());
3523 cqe_socket = try ring.copy_cqe();
3524 try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
3525 try testing.expect(cqe_socket.res == 0); // res is 0 when index is specified
3526
3527 // create socket in kernel chosen file descriptor index (_alloc version)
3528 // completion res has index from registered files
3529 _ = try ring.socket_direct_alloc(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
3530 try testing.expectEqual(@as(u32, 1), try ring.submit());
3531 cqe_socket = try ring.copy_cqe();
3532 try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
3533 try testing.expect(cqe_socket.res == 2); // returns registered file index
3534
3535 // use sockets from registered_fds in connect operation
3536 var address = try net.Address.parseIp4("127.0.0.1", 0);
3537 const listener_socket = try createListenerSocket(&address);
3538 defer posix.close(listener_socket);
3539 const accept_userdata: u64 = 0xaaaaaaaa;
3540 const connect_userdata: u64 = 0xbbbbbbbb;
3541 const close_userdata: u64 = 0xcccccccc;
3542 for (registered_fds, 0..) |_, fd_index| {
3543 // prepare accept
3544 _ = try ring.accept(accept_userdata, listener_socket, null, null, 0);
3545 // prepare connect with fixed socket
3546 const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), &address.any, address.getOsSockLen());
3547 connect_sqe.flags |= linux.IOSQE_FIXED_FILE; // fd is fixed file index
3548 // submit both
3549 try testing.expectEqual(@as(u32, 2), try ring.submit());
3550 // get completions
3551 var cqe_connect = try ring.copy_cqe();
3552 var cqe_accept = try ring.copy_cqe();
3553 // ignore order
3554 if (cqe_connect.user_data == accept_userdata and cqe_accept.user_data == connect_userdata) {
3555 const a = cqe_accept;
3556 const b = cqe_connect;
3557 cqe_accept = b;
3558 cqe_connect = a;
3559 }
3560 // test connect completion
3561 try testing.expect(cqe_connect.user_data == connect_userdata);
3562 try testing.expectEqual(os.E.SUCCESS, cqe_connect.err());
3563 // test accept completion
3564 try testing.expect(cqe_accept.user_data == accept_userdata);
3565 try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
3566
3567 // submit and test close_direct
3568 _ = try ring.close_direct(close_userdata, @intCast(fd_index));
3569 try testing.expectEqual(@as(u32, 1), try ring.submit());
3570 var cqe_close = try ring.copy_cqe();
3571 try testing.expect(cqe_close.user_data == close_userdata);
3572 try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
3573 }
3574
3575 try ring.unregister_files();
3576}
3577
3578test "openat_direct/close_direct" {
3579 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3580
3581 var ring = IoUring.init(2, 0) catch |err| switch (err) {
3582 error.SystemOutdated => return error.SkipZigTest,
3583 error.PermissionDenied => return error.SkipZigTest,
3584 else => return err,
3585 };
3586 defer ring.deinit();
3587
3588 var registered_fds = [_]os.fd_t{-1} ** 3;
3589 try ring.register_files(registered_fds[0..]);
3590
3591 var tmp = std.testing.tmpDir(.{});
3592 defer tmp.cleanup();
3593 const path = "test_io_uring_close_direct";
3594 const flags: linux.O = .{ .ACCMODE = .RDWR, .CREAT = true };
3595 const mode: os.mode_t = 0o666;
3596 const user_data: u64 = 0;
3597
3598 // use registered file at index 0 (last param)
3599 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 0);
3600 try testing.expectEqual(@as(u32, 1), try ring.submit());
3601 var cqe = try ring.copy_cqe();
3602 try testing.expectEqual(os.E.SUCCESS, cqe.err());
3603 try testing.expect(cqe.res == 0);
3604
3605 // use registered file at index 1
3606 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 1);
3607 try testing.expectEqual(@as(u32, 1), try ring.submit());
3608 cqe = try ring.copy_cqe();
3609 try testing.expectEqual(os.E.SUCCESS, cqe.err());
3610 try testing.expect(cqe.res == 0); // res is 0 when we specify index
3611
3612 // let kernel choose registered file index
3613 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, linux.IORING_FILE_INDEX_ALLOC);
3614 try testing.expectEqual(@as(u32, 1), try ring.submit());
3615 cqe = try ring.copy_cqe();
3616 try testing.expectEqual(os.E.SUCCESS, cqe.err());
3617 try testing.expect(cqe.res == 2); // chosen index is in res
3618
3619 // close all open file descriptors
3620 for (registered_fds, 0..) |_, fd_index| {
3621 _ = try ring.close_direct(user_data, @intCast(fd_index));
3622 try testing.expectEqual(@as(u32, 1), try ring.submit());
3623 var cqe_close = try ring.copy_cqe();
3624 try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
3625 }
3626 try ring.unregister_files();
3627}
3628
3629test "waitid" {
3630 try skipKernelLessThan(.{ .major = 6, .minor = 7, .patch = 0 });
3631
3632 var ring = IoUring.init(16, 0) catch |err| switch (err) {
3633 error.SystemOutdated => return error.SkipZigTest,
3634 error.PermissionDenied => return error.SkipZigTest,
3635 else => return err,
3636 };
3637 defer ring.deinit();
3638
3639 const pid = try os.fork();
3640 if (pid == 0) {
3641 os.exit(7);
3642 }
3643
3644 var siginfo: os.siginfo_t = undefined;
3645 _ = try ring.waitid(0, .PID, pid, &siginfo, os.W.EXITED, 0);
3646
3647 try testing.expectEqual(1, try ring.submit());
3648
3649 const cqe_waitid = try ring.copy_cqe();
3650 try testing.expectEqual(0, cqe_waitid.res);
3651 try testing.expectEqual(pid, siginfo.fields.common.first.piduid.pid);
3652 try testing.expectEqual(7, siginfo.fields.common.second.sigchld.status);
3653}
3654
3655/// For use in tests. Returns SkipZigTest is kernel version is less than required.
3656inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
3657 if (builtin.os.tag != .linux) return error.SkipZigTest;
3658
3659 var uts: linux.utsname = undefined;
3660 const res = linux.uname(&uts);
3661 switch (linux.getErrno(res)) {
3662 .SUCCESS => {},
3663 else => |errno| return os.unexpectedErrno(errno),
3664 }
3665
3666 const release = mem.sliceTo(&uts.release, 0);
3667 var current = try std.SemanticVersion.parse(release);
3668 current.pre = null; // don't check pre field
3669 if (required.order(current) == .gt) return error.SkipZigTest;
3670}
lib/std/os/linux/io_uring.zig deleted-4228
...@@ -1,4228 +0,0 @@
1const std = @import("../../std.zig");
2const builtin = @import("builtin");
3const assert = std.debug.assert;
4const mem = std.mem;
5const net = std.net;
6const os = std.os;
7const posix = std.posix;
8const linux = os.linux;
9const testing = std.testing;
10
11pub const IO_Uring = struct {
12 fd: os.fd_t = -1,
13 sq: SubmissionQueue,
14 cq: CompletionQueue,
15 flags: u32,
16 features: u32,
17
18 /// A friendly way to setup an io_uring, with default linux.io_uring_params.
19 /// `entries` must be a power of two between 1 and 32768, although the kernel will make the final
20 /// call on how many entries the submission and completion queues will ultimately have,
21 /// see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L8027-L8050.
22 /// Matches the interface of io_uring_queue_init() in liburing.
23 pub fn init(entries: u16, flags: u32) !IO_Uring {
24 var params = mem.zeroInit(linux.io_uring_params, .{
25 .flags = flags,
26 .sq_thread_idle = 1000,
27 });
28 return try IO_Uring.init_params(entries, &params);
29 }
30
31 /// A powerful way to setup an io_uring, if you want to tweak linux.io_uring_params such as submission
32 /// queue thread cpu affinity or thread idle timeout (the kernel and our default is 1 second).
33 /// `params` is passed by reference because the kernel needs to modify the parameters.
34 /// Matches the interface of io_uring_queue_init_params() in liburing.
35 pub fn init_params(entries: u16, p: *linux.io_uring_params) !IO_Uring {
36 if (entries == 0) return error.EntriesZero;
37 if (!std.math.isPowerOfTwo(entries)) return error.EntriesNotPowerOfTwo;
38
39 assert(p.sq_entries == 0);
40 assert(p.cq_entries == 0 or p.flags & linux.IORING_SETUP_CQSIZE != 0);
41 assert(p.features == 0);
42 assert(p.wq_fd == 0 or p.flags & linux.IORING_SETUP_ATTACH_WQ != 0);
43 assert(p.resv[0] == 0);
44 assert(p.resv[1] == 0);
45 assert(p.resv[2] == 0);
46
47 const res = linux.io_uring_setup(entries, p);
48 switch (linux.getErrno(res)) {
49 .SUCCESS => {},
50 .FAULT => return error.ParamsOutsideAccessibleAddressSpace,
51 // The resv array contains non-zero data, p.flags contains an unsupported flag,
52 // entries out of bounds, IORING_SETUP_SQ_AFF was specified without IORING_SETUP_SQPOLL,
53 // or IORING_SETUP_CQSIZE was specified but linux.io_uring_params.cq_entries was invalid:
54 .INVAL => return error.ArgumentsInvalid,
55 .MFILE => return error.ProcessFdQuotaExceeded,
56 .NFILE => return error.SystemFdQuotaExceeded,
57 .NOMEM => return error.SystemResources,
58 // IORING_SETUP_SQPOLL was specified but effective user ID lacks sufficient privileges,
59 // or a container seccomp policy prohibits io_uring syscalls:
60 .PERM => return error.PermissionDenied,
61 .NOSYS => return error.SystemOutdated,
62 else => |errno| return os.unexpectedErrno(errno),
63 }
64 const fd = @as(os.fd_t, @intCast(res));
65 assert(fd >= 0);
66 errdefer os.close(fd);
67
68 // Kernel versions 5.4 and up use only one mmap() for the submission and completion queues.
69 // This is not an optional feature for us... if the kernel does it, we have to do it.
70 // The thinking on this by the kernel developers was that both the submission and the
71 // completion queue rings have sizes just over a power of two, but the submission queue ring
72 // is significantly smaller with u32 slots. By bundling both in a single mmap, the kernel
73 // gets the submission queue ring for free.
74 // See https://patchwork.kernel.org/patch/11115257 for the kernel patch.
75 // We do not support the double mmap() done before 5.4, because we want to keep the
76 // init/deinit mmap paths simple and because io_uring has had many bug fixes even since 5.4.
77 if ((p.features & linux.IORING_FEAT_SINGLE_MMAP) == 0) {
78 return error.SystemOutdated;
79 }
80
81 // Check that the kernel has actually set params and that "impossible is nothing".
82 assert(p.sq_entries != 0);
83 assert(p.cq_entries != 0);
84 assert(p.cq_entries >= p.sq_entries);
85
86 // From here on, we only need to read from params, so pass `p` by value as immutable.
87 // The completion queue shares the mmap with the submission queue, so pass `sq` there too.
88 var sq = try SubmissionQueue.init(fd, p.*);
89 errdefer sq.deinit();
90 var cq = try CompletionQueue.init(fd, p.*, sq);
91 errdefer cq.deinit();
92
93 // Check that our starting state is as we expect.
94 assert(sq.head.* == 0);
95 assert(sq.tail.* == 0);
96 assert(sq.mask == p.sq_entries - 1);
97 // Allow flags.* to be non-zero, since the kernel may set IORING_SQ_NEED_WAKEUP at any time.
98 assert(sq.dropped.* == 0);
99 assert(sq.array.len == p.sq_entries);
100 assert(sq.sqes.len == p.sq_entries);
101 assert(sq.sqe_head == 0);
102 assert(sq.sqe_tail == 0);
103
104 assert(cq.head.* == 0);
105 assert(cq.tail.* == 0);
106 assert(cq.mask == p.cq_entries - 1);
107 assert(cq.overflow.* == 0);
108 assert(cq.cqes.len == p.cq_entries);
109
110 return IO_Uring{
111 .fd = fd,
112 .sq = sq,
113 .cq = cq,
114 .flags = p.flags,
115 .features = p.features,
116 };
117 }
118
119 pub fn deinit(self: *IO_Uring) void {
120 assert(self.fd >= 0);
121 // The mmaps depend on the fd, so the order of these calls is important:
122 self.cq.deinit();
123 self.sq.deinit();
124 os.close(self.fd);
125 self.fd = -1;
126 }
127
128 /// Returns a pointer to a vacant SQE, or an error if the submission queue is full.
129 /// We follow the implementation (and atomics) of liburing's `io_uring_get_sqe()` exactly.
130 /// However, instead of a null we return an error to force safe handling.
131 /// Any situation where the submission queue is full tends more towards a control flow error,
132 /// and the null return in liburing is more a C idiom than anything else, for lack of a better
133 /// alternative. In Zig, we have first-class error handling... so let's use it.
134 /// Matches the implementation of io_uring_get_sqe() in liburing.
135 pub fn get_sqe(self: *IO_Uring) !*linux.io_uring_sqe {
136 const head = @atomicLoad(u32, self.sq.head, .Acquire);
137 // Remember that these head and tail offsets wrap around every four billion operations.
138 // We must therefore use wrapping addition and subtraction to avoid a runtime crash.
139 const next = self.sq.sqe_tail +% 1;
140 if (next -% head > self.sq.sqes.len) return error.SubmissionQueueFull;
141 const sqe = &self.sq.sqes[self.sq.sqe_tail & self.sq.mask];
142 self.sq.sqe_tail = next;
143 return sqe;
144 }
145
146 /// Submits the SQEs acquired via get_sqe() to the kernel. You can call this once after you have
147 /// called get_sqe() multiple times to setup multiple I/O requests.
148 /// Returns the number of SQEs submitted, if not used alongside IORING_SETUP_SQPOLL.
149 /// If the io_uring instance is uses IORING_SETUP_SQPOLL, the value returned on success is not
150 /// guaranteed to match the amount of actually submitted sqes during this call. A value higher
151 /// or lower, including 0, may be returned.
152 /// Matches the implementation of io_uring_submit() in liburing.
153 pub fn submit(self: *IO_Uring) !u32 {
154 return self.submit_and_wait(0);
155 }
156
157 /// Like submit(), but allows waiting for events as well.
158 /// Returns the number of SQEs submitted.
159 /// Matches the implementation of io_uring_submit_and_wait() in liburing.
160 pub fn submit_and_wait(self: *IO_Uring, wait_nr: u32) !u32 {
161 const submitted = self.flush_sq();
162 var flags: u32 = 0;
163 if (self.sq_ring_needs_enter(&flags) or wait_nr > 0) {
164 if (wait_nr > 0 or (self.flags & linux.IORING_SETUP_IOPOLL) != 0) {
165 flags |= linux.IORING_ENTER_GETEVENTS;
166 }
167 return try self.enter(submitted, wait_nr, flags);
168 }
169 return submitted;
170 }
171
172 /// Tell the kernel we have submitted SQEs and/or want to wait for CQEs.
173 /// Returns the number of SQEs submitted.
174 pub fn enter(self: *IO_Uring, to_submit: u32, min_complete: u32, flags: u32) !u32 {
175 assert(self.fd >= 0);
176 const res = linux.io_uring_enter(self.fd, to_submit, min_complete, flags, null);
177 switch (linux.getErrno(res)) {
178 .SUCCESS => {},
179 // The kernel was unable to allocate memory or ran out of resources for the request.
180 // The application should wait for some completions and try again:
181 .AGAIN => return error.SystemResources,
182 // The SQE `fd` is invalid, or IOSQE_FIXED_FILE was set but no files were registered:
183 .BADF => return error.FileDescriptorInvalid,
184 // The file descriptor is valid, but the ring is not in the right state.
185 // See io_uring_register(2) for how to enable the ring.
186 .BADFD => return error.FileDescriptorInBadState,
187 // The application attempted to overcommit the number of requests it can have pending.
188 // The application should wait for some completions and try again:
189 .BUSY => return error.CompletionQueueOvercommitted,
190 // The SQE is invalid, or valid but the ring was setup with IORING_SETUP_IOPOLL:
191 .INVAL => return error.SubmissionQueueEntryInvalid,
192 // The buffer is outside the process' accessible address space, or IORING_OP_READ_FIXED
193 // or IORING_OP_WRITE_FIXED was specified but no buffers were registered, or the range
194 // described by `addr` and `len` is not within the buffer registered at `buf_index`:
195 .FAULT => return error.BufferInvalid,
196 .NXIO => return error.RingShuttingDown,
197 // The kernel believes our `self.fd` does not refer to an io_uring instance,
198 // or the opcode is valid but not supported by this kernel (more likely):
199 .OPNOTSUPP => return error.OpcodeNotSupported,
200 // The operation was interrupted by a delivery of a signal before it could complete.
201 // This can happen while waiting for events with IORING_ENTER_GETEVENTS:
202 .INTR => return error.SignalInterrupt,
203 else => |errno| return os.unexpectedErrno(errno),
204 }
205 return @as(u32, @intCast(res));
206 }
207
208 /// Sync internal state with kernel ring state on the SQ side.
209 /// Returns the number of all pending events in the SQ ring, for the shared ring.
210 /// This return value includes previously flushed SQEs, as per liburing.
211 /// The rationale is to suggest that an io_uring_enter() call is needed rather than not.
212 /// Matches the implementation of __io_uring_flush_sq() in liburing.
213 pub fn flush_sq(self: *IO_Uring) u32 {
214 if (self.sq.sqe_head != self.sq.sqe_tail) {
215 // Fill in SQEs that we have queued up, adding them to the kernel ring.
216 const to_submit = self.sq.sqe_tail -% self.sq.sqe_head;
217 var tail = self.sq.tail.*;
218 var i: usize = 0;
219 while (i < to_submit) : (i += 1) {
220 self.sq.array[tail & self.sq.mask] = self.sq.sqe_head & self.sq.mask;
221 tail +%= 1;
222 self.sq.sqe_head +%= 1;
223 }
224 // Ensure that the kernel can actually see the SQE updates when it sees the tail update.
225 @atomicStore(u32, self.sq.tail, tail, .Release);
226 }
227 return self.sq_ready();
228 }
229
230 /// Returns true if we are not using an SQ thread (thus nobody submits but us),
231 /// or if IORING_SQ_NEED_WAKEUP is set and the SQ thread must be explicitly awakened.
232 /// For the latter case, we set the SQ thread wakeup flag.
233 /// Matches the implementation of sq_ring_needs_enter() in liburing.
234 pub fn sq_ring_needs_enter(self: *IO_Uring, flags: *u32) bool {
235 assert(flags.* == 0);
236 if ((self.flags & linux.IORING_SETUP_SQPOLL) == 0) return true;
237 if ((@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_NEED_WAKEUP) != 0) {
238 flags.* |= linux.IORING_ENTER_SQ_WAKEUP;
239 return true;
240 }
241 return false;
242 }
243
244 /// Returns the number of flushed and unflushed SQEs pending in the submission queue.
245 /// In other words, this is the number of SQEs in the submission queue, i.e. its length.
246 /// These are SQEs that the kernel is yet to consume.
247 /// Matches the implementation of io_uring_sq_ready in liburing.
248 pub fn sq_ready(self: *IO_Uring) u32 {
249 // Always use the shared ring state (i.e. head and not sqe_head) to avoid going out of sync,
250 // see https://github.com/axboe/liburing/issues/92.
251 return self.sq.sqe_tail -% @atomicLoad(u32, self.sq.head, .Acquire);
252 }
253
254 /// Returns the number of CQEs in the completion queue, i.e. its length.
255 /// These are CQEs that the application is yet to consume.
256 /// Matches the implementation of io_uring_cq_ready in liburing.
257 pub fn cq_ready(self: *IO_Uring) u32 {
258 return @atomicLoad(u32, self.cq.tail, .Acquire) -% self.cq.head.*;
259 }
260
261 /// Copies as many CQEs as are ready, and that can fit into the destination `cqes` slice.
262 /// If none are available, enters into the kernel to wait for at most `wait_nr` CQEs.
263 /// Returns the number of CQEs copied, advancing the CQ ring.
264 /// Provides all the wait/peek methods found in liburing, but with batching and a single method.
265 /// The rationale for copying CQEs rather than copying pointers is that pointers are 8 bytes
266 /// whereas CQEs are not much more at only 16 bytes, and this provides a safer faster interface.
267 /// Safer, because you no longer need to call cqe_seen(), avoiding idempotency bugs.
268 /// Faster, because we can now amortize the atomic store release to `cq.head` across the batch.
269 /// See https://github.com/axboe/liburing/issues/103#issuecomment-686665007.
270 /// Matches the implementation of io_uring_peek_batch_cqe() in liburing, but supports waiting.
271 pub fn copy_cqes(self: *IO_Uring, cqes: []linux.io_uring_cqe, wait_nr: u32) !u32 {
272 const count = self.copy_cqes_ready(cqes);
273 if (count > 0) return count;
274 if (self.cq_ring_needs_flush() or wait_nr > 0) {
275 _ = try self.enter(0, wait_nr, linux.IORING_ENTER_GETEVENTS);
276 return self.copy_cqes_ready(cqes);
277 }
278 return 0;
279 }
280
281 fn copy_cqes_ready(self: *IO_Uring, cqes: []linux.io_uring_cqe) u32 {
282 const ready = self.cq_ready();
283 const count = @min(cqes.len, ready);
284 const head = self.cq.head.* & self.cq.mask;
285 const tail = (self.cq.head.* +% count) & self.cq.mask;
286
287 if (head <= tail) {
288 // head behind tail -> no wrapping
289 @memcpy(cqes[0..count], self.cq.cqes[head..tail]);
290 } else {
291 // head in front of tail -> buffer wraps
292 const two_copies_required: bool = self.cq.cqes.len - head < count;
293 const amount_to_copy_in_first = if (two_copies_required) self.cq.cqes.len - head else count;
294 @memcpy(cqes[0..amount_to_copy_in_first], self.cq.cqes[head .. head + amount_to_copy_in_first]);
295 if (two_copies_required) {
296 @memcpy(cqes[amount_to_copy_in_first..count], self.cq.cqes[0..tail]);
297 }
298 }
299
300 self.cq_advance(count);
301 return count;
302 }
303
304 /// Returns a copy of an I/O completion, waiting for it if necessary, and advancing the CQ ring.
305 /// A convenience method for `copy_cqes()` for when you don't need to batch or peek.
306 pub fn copy_cqe(ring: *IO_Uring) !linux.io_uring_cqe {
307 var cqes: [1]linux.io_uring_cqe = undefined;
308 while (true) {
309 const count = try ring.copy_cqes(&cqes, 1);
310 if (count > 0) return cqes[0];
311 }
312 }
313
314 /// Matches the implementation of cq_ring_needs_flush() in liburing.
315 pub fn cq_ring_needs_flush(self: *IO_Uring) bool {
316 return (@atomicLoad(u32, self.sq.flags, .Unordered) & linux.IORING_SQ_CQ_OVERFLOW) != 0;
317 }
318
319 /// For advanced use cases only that implement custom completion queue methods.
320 /// If you use copy_cqes() or copy_cqe() you must not call cqe_seen() or cq_advance().
321 /// Must be called exactly once after a zero-copy CQE has been processed by your application.
322 /// Not idempotent, calling more than once will result in other CQEs being lost.
323 /// Matches the implementation of cqe_seen() in liburing.
324 pub fn cqe_seen(self: *IO_Uring, cqe: *linux.io_uring_cqe) void {
325 _ = cqe;
326 self.cq_advance(1);
327 }
328
329 /// For advanced use cases only that implement custom completion queue methods.
330 /// Matches the implementation of cq_advance() in liburing.
331 pub fn cq_advance(self: *IO_Uring, count: u32) void {
332 if (count > 0) {
333 // Ensure the kernel only sees the new head value after the CQEs have been read.
334 @atomicStore(u32, self.cq.head, self.cq.head.* +% count, .Release);
335 }
336 }
337
338 /// Queues (but does not submit) an SQE to perform an `fsync(2)`.
339 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
340 /// For example, for `fdatasync()` you can set `IORING_FSYNC_DATASYNC` in the SQE's `rw_flags`.
341 /// N.B. While SQEs are initiated in the order in which they appear in the submission queue,
342 /// operations execute in parallel and completions are unordered. Therefore, an application that
343 /// submits a write followed by an fsync in the submission queue cannot expect the fsync to
344 /// apply to the write, since the fsync may complete before the write is issued to the disk.
345 /// You should preferably use `link_with_next_sqe()` on a write's SQE to link it with an fsync,
346 /// or else insert a full write barrier using `drain_previous_sqes()` when queueing an fsync.
347 pub fn fsync(self: *IO_Uring, user_data: u64, fd: os.fd_t, flags: u32) !*linux.io_uring_sqe {
348 const sqe = try self.get_sqe();
349 io_uring_prep_fsync(sqe, fd, flags);
350 sqe.user_data = user_data;
351 return sqe;
352 }
353
354 /// Queues (but does not submit) an SQE to perform a no-op.
355 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
356 /// A no-op is more useful than may appear at first glance.
357 /// For example, you could call `drain_previous_sqes()` on the returned SQE, to use the no-op to
358 /// know when the ring is idle before acting on a kill signal.
359 pub fn nop(self: *IO_Uring, user_data: u64) !*linux.io_uring_sqe {
360 const sqe = try self.get_sqe();
361 io_uring_prep_nop(sqe);
362 sqe.user_data = user_data;
363 return sqe;
364 }
365
366 /// Used to select how the read should be handled.
367 pub const ReadBuffer = union(enum) {
368 /// io_uring will read directly into this buffer
369 buffer: []u8,
370
371 /// io_uring will read directly into these buffers using readv.
372 iovecs: []const os.iovec,
373
374 /// io_uring will select a buffer that has previously been provided with `provide_buffers`.
375 /// The buffer group reference by `group_id` must contain at least one buffer for the read to work.
376 /// `len` controls the number of bytes to read into the selected buffer.
377 buffer_selection: struct {
378 group_id: u16,
379 len: usize,
380 },
381 };
382
383 /// Queues (but does not submit) an SQE to perform a `read(2)` or `preadv(2)` depending on the buffer type.
384 /// * Reading into a `ReadBuffer.buffer` uses `read(2)`
385 /// * Reading into a `ReadBuffer.iovecs` uses `preadv(2)`
386 /// If you want to do a `preadv2(2)` then set `rw_flags` on the returned SQE. See https://man7.org/linux/man-pages/man2/preadv2.2.html
387 ///
388 /// Returns a pointer to the SQE.
389 pub fn read(
390 self: *IO_Uring,
391 user_data: u64,
392 fd: os.fd_t,
393 buffer: ReadBuffer,
394 offset: u64,
395 ) !*linux.io_uring_sqe {
396 const sqe = try self.get_sqe();
397 switch (buffer) {
398 .buffer => |slice| io_uring_prep_read(sqe, fd, slice, offset),
399 .iovecs => |vecs| io_uring_prep_readv(sqe, fd, vecs, offset),
400 .buffer_selection => |selection| {
401 io_uring_prep_rw(.READ, sqe, fd, 0, selection.len, offset);
402 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
403 sqe.buf_index = selection.group_id;
404 },
405 }
406 sqe.user_data = user_data;
407 return sqe;
408 }
409
410 /// Queues (but does not submit) an SQE to perform a `write(2)`.
411 /// Returns a pointer to the SQE.
412 pub fn write(
413 self: *IO_Uring,
414 user_data: u64,
415 fd: os.fd_t,
416 buffer: []const u8,
417 offset: u64,
418 ) !*linux.io_uring_sqe {
419 const sqe = try self.get_sqe();
420 io_uring_prep_write(sqe, fd, buffer, offset);
421 sqe.user_data = user_data;
422 return sqe;
423 }
424
425 /// Queues (but does not submit) an SQE to perform a `splice(2)`
426 /// Either `fd_in` or `fd_out` must be a pipe.
427 /// If `fd_in` refers to a pipe, `off_in` is ignored and must be set to std.math.maxInt(u64).
428 /// If `fd_in` does not refer to a pipe and `off_in` is maxInt(u64), then `len` are read
429 /// from `fd_in` starting from the file offset, which is incremented by the number of bytes read.
430 /// If `fd_in` does not refer to a pipe and `off_in` is not maxInt(u64), then the starting offset of `fd_in` will be `off_in`.
431 /// This splice operation can be used to implement sendfile by splicing to an intermediate pipe first,
432 /// then splice to the final destination. In fact, the implementation of sendfile in kernel uses splice internally.
433 ///
434 /// NOTE that even if fd_in or fd_out refers to a pipe, the splice operation can still fail with EINVAL if one of the
435 /// fd doesn't explicitly support splice peration, e.g. reading from terminal is unsupported from kernel 5.7 to 5.11.
436 /// See https://github.com/axboe/liburing/issues/291
437 ///
438 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
439 pub fn splice(self: *IO_Uring, user_data: u64, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) !*linux.io_uring_sqe {
440 const sqe = try self.get_sqe();
441 io_uring_prep_splice(sqe, fd_in, off_in, fd_out, off_out, len);
442 sqe.user_data = user_data;
443 return sqe;
444 }
445
446 /// Queues (but does not submit) an SQE to perform a IORING_OP_READ_FIXED.
447 /// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
448 /// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
449 ///
450 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
451 pub fn read_fixed(
452 self: *IO_Uring,
453 user_data: u64,
454 fd: os.fd_t,
455 buffer: *os.iovec,
456 offset: u64,
457 buffer_index: u16,
458 ) !*linux.io_uring_sqe {
459 const sqe = try self.get_sqe();
460 io_uring_prep_read_fixed(sqe, fd, buffer, offset, buffer_index);
461 sqe.user_data = user_data;
462 return sqe;
463 }
464
465 /// Queues (but does not submit) an SQE to perform a `pwritev()`.
466 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
467 /// For example, if you want to do a `pwritev2()` then set `rw_flags` on the returned SQE.
468 /// See https://linux.die.net/man/2/pwritev.
469 pub fn writev(
470 self: *IO_Uring,
471 user_data: u64,
472 fd: os.fd_t,
473 iovecs: []const os.iovec_const,
474 offset: u64,
475 ) !*linux.io_uring_sqe {
476 const sqe = try self.get_sqe();
477 io_uring_prep_writev(sqe, fd, iovecs, offset);
478 sqe.user_data = user_data;
479 return sqe;
480 }
481
482 /// Queues (but does not submit) an SQE to perform a IORING_OP_WRITE_FIXED.
483 /// The `buffer` provided must be registered with the kernel by calling `register_buffers` first.
484 /// The `buffer_index` must be the same as its index in the array provided to `register_buffers`.
485 ///
486 /// Returns a pointer to the SQE so that you can further modify the SQE for advanced use cases.
487 pub fn write_fixed(
488 self: *IO_Uring,
489 user_data: u64,
490 fd: os.fd_t,
491 buffer: *os.iovec,
492 offset: u64,
493 buffer_index: u16,
494 ) !*linux.io_uring_sqe {
495 const sqe = try self.get_sqe();
496 io_uring_prep_write_fixed(sqe, fd, buffer, offset, buffer_index);
497 sqe.user_data = user_data;
498 return sqe;
499 }
500
501 /// Queues (but does not submit) an SQE to perform an `accept4(2)` on a socket.
502 /// Returns a pointer to the SQE.
503 /// Available since 5.5
504 pub fn accept(
505 self: *IO_Uring,
506 user_data: u64,
507 fd: os.fd_t,
508 addr: ?*os.sockaddr,
509 addrlen: ?*os.socklen_t,
510 flags: u32,
511 ) !*linux.io_uring_sqe {
512 const sqe = try self.get_sqe();
513 io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
514 sqe.user_data = user_data;
515 return sqe;
516 }
517
518 /// Queues an multishot accept on a socket.
519 ///
520 /// Multishot variant allows an application to issue a single accept request,
521 /// which will repeatedly trigger a CQE when a connection request comes in.
522 /// While IORING_CQE_F_MORE flag is set in CQE flags accept will generate
523 /// further CQEs.
524 ///
525 /// Available since 5.19
526 pub fn accept_multishot(
527 self: *IO_Uring,
528 user_data: u64,
529 fd: os.fd_t,
530 addr: ?*os.sockaddr,
531 addrlen: ?*os.socklen_t,
532 flags: u32,
533 ) !*linux.io_uring_sqe {
534 const sqe = try self.get_sqe();
535 io_uring_prep_multishot_accept(sqe, fd, addr, addrlen, flags);
536 sqe.user_data = user_data;
537 return sqe;
538 }
539
540 /// Queues an accept using direct (registered) file descriptors.
541 ///
542 /// To use an accept direct variant, the application must first have registered
543 /// a file table (with register_files). An unused table index will be
544 /// dynamically chosen and returned in the CQE res field.
545 ///
546 /// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
547 /// flags member, and setting the SQE fd field to the direct descriptor value
548 /// rather than the regular file descriptor.
549 ///
550 /// Available since 5.19
551 pub fn accept_direct(
552 self: *IO_Uring,
553 user_data: u64,
554 fd: os.fd_t,
555 addr: ?*os.sockaddr,
556 addrlen: ?*os.socklen_t,
557 flags: u32,
558 ) !*linux.io_uring_sqe {
559 const sqe = try self.get_sqe();
560 io_uring_prep_accept_direct(sqe, fd, addr, addrlen, flags, linux.IORING_FILE_INDEX_ALLOC);
561 sqe.user_data = user_data;
562 return sqe;
563 }
564
565 /// Queues an multishot accept using direct (registered) file descriptors.
566 /// Available since 5.19
567 pub fn accept_multishot_direct(
568 self: *IO_Uring,
569 user_data: u64,
570 fd: os.fd_t,
571 addr: ?*os.sockaddr,
572 addrlen: ?*os.socklen_t,
573 flags: u32,
574 ) !*linux.io_uring_sqe {
575 const sqe = try self.get_sqe();
576 io_uring_prep_multishot_accept_direct(sqe, fd, addr, addrlen, flags);
577 sqe.user_data = user_data;
578 return sqe;
579 }
580
581 /// Queue (but does not submit) an SQE to perform a `connect(2)` on a socket.
582 /// Returns a pointer to the SQE.
583 pub fn connect(
584 self: *IO_Uring,
585 user_data: u64,
586 fd: os.fd_t,
587 addr: *const os.sockaddr,
588 addrlen: os.socklen_t,
589 ) !*linux.io_uring_sqe {
590 const sqe = try self.get_sqe();
591 io_uring_prep_connect(sqe, fd, addr, addrlen);
592 sqe.user_data = user_data;
593 return sqe;
594 }
595
596 /// Queues (but does not submit) an SQE to perform a `epoll_ctl(2)`.
597 /// Returns a pointer to the SQE.
598 pub fn epoll_ctl(
599 self: *IO_Uring,
600 user_data: u64,
601 epfd: os.fd_t,
602 fd: os.fd_t,
603 op: u32,
604 ev: ?*linux.epoll_event,
605 ) !*linux.io_uring_sqe {
606 const sqe = try self.get_sqe();
607 io_uring_prep_epoll_ctl(sqe, epfd, fd, op, ev);
608 sqe.user_data = user_data;
609 return sqe;
610 }
611
612 /// Used to select how the recv call should be handled.
613 pub const RecvBuffer = union(enum) {
614 /// io_uring will recv directly into this buffer
615 buffer: []u8,
616
617 /// io_uring will select a buffer that has previously been provided with `provide_buffers`.
618 /// The buffer group referenced by `group_id` must contain at least one buffer for the recv call to work.
619 /// `len` controls the number of bytes to read into the selected buffer.
620 buffer_selection: struct {
621 group_id: u16,
622 len: usize,
623 },
624 };
625
626 /// Queues (but does not submit) an SQE to perform a `recv(2)`.
627 /// Returns a pointer to the SQE.
628 /// Available since 5.6
629 pub fn recv(
630 self: *IO_Uring,
631 user_data: u64,
632 fd: os.fd_t,
633 buffer: RecvBuffer,
634 flags: u32,
635 ) !*linux.io_uring_sqe {
636 const sqe = try self.get_sqe();
637 switch (buffer) {
638 .buffer => |slice| io_uring_prep_recv(sqe, fd, slice, flags),
639 .buffer_selection => |selection| {
640 io_uring_prep_rw(.RECV, sqe, fd, 0, selection.len, 0);
641 sqe.rw_flags = flags;
642 sqe.flags |= linux.IOSQE_BUFFER_SELECT;
643 sqe.buf_index = selection.group_id;
644 },
645 }
646 sqe.user_data = user_data;
647 return sqe;
648 }
649
650 /// Queues (but does not submit) an SQE to perform a `send(2)`.
651 /// Returns a pointer to the SQE.
652 /// Available since 5.6
653 pub fn send(
654 self: *IO_Uring,
655 user_data: u64,
656 fd: os.fd_t,
657 buffer: []const u8,
658 flags: u32,
659 ) !*linux.io_uring_sqe {
660 const sqe = try self.get_sqe();
661 io_uring_prep_send(sqe, fd, buffer, flags);
662 sqe.user_data = user_data;
663 return sqe;
664 }
665
666 /// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
667 ///
668 /// This operation will most likely produce two CQEs. The flags field of the
669 /// first cqe may likely contain IORING_CQE_F_MORE, which means that there will
670 /// be a second cqe with the user_data field set to the same value. The user
671 /// must not modify the data buffer until the notification is posted. The first
672 /// cqe follows the usual rules and so its res field will contain the number of
673 /// bytes sent or a negative error code. The notification's res field will be
674 /// set to zero and the flags field will contain IORING_CQE_F_NOTIF. The two
675 /// step model is needed because the kernel may hold on to buffers for a long
676 /// time, e.g. waiting for a TCP ACK. Notifications responsible for controlling
677 /// the lifetime of the buffers. Even errored requests may generate a
678 /// notification.
679 ///
680 /// Available since 6.0
681 pub fn send_zc(
682 self: *IO_Uring,
683 user_data: u64,
684 fd: os.fd_t,
685 buffer: []const u8,
686 send_flags: u32,
687 zc_flags: u16,
688 ) !*linux.io_uring_sqe {
689 const sqe = try self.get_sqe();
690 io_uring_prep_send_zc(sqe, fd, buffer, send_flags, zc_flags);
691 sqe.user_data = user_data;
692 return sqe;
693 }
694
695 /// Queues (but does not submit) an SQE to perform an async zerocopy `send(2)`.
696 /// Returns a pointer to the SQE.
697 /// Available since 6.0
698 pub fn send_zc_fixed(
699 self: *IO_Uring,
700 user_data: u64,
701 fd: os.fd_t,
702 buffer: []const u8,
703 send_flags: u32,
704 zc_flags: u16,
705 buf_index: u16,
706 ) !*linux.io_uring_sqe {
707 const sqe = try self.get_sqe();
708 io_uring_prep_send_zc_fixed(sqe, fd, buffer, send_flags, zc_flags, buf_index);
709 sqe.user_data = user_data;
710 return sqe;
711 }
712
713 /// Queues (but does not submit) an SQE to perform a `recvmsg(2)`.
714 /// Returns a pointer to the SQE.
715 /// Available since 5.3
716 pub fn recvmsg(
717 self: *IO_Uring,
718 user_data: u64,
719 fd: os.fd_t,
720 msg: *os.msghdr,
721 flags: u32,
722 ) !*linux.io_uring_sqe {
723 const sqe = try self.get_sqe();
724 io_uring_prep_recvmsg(sqe, fd, msg, flags);
725 sqe.user_data = user_data;
726 return sqe;
727 }
728
729 /// Queues (but does not submit) an SQE to perform a `sendmsg(2)`.
730 /// Returns a pointer to the SQE.
731 /// Available since 5.3
732 pub fn sendmsg(
733 self: *IO_Uring,
734 user_data: u64,
735 fd: os.fd_t,
736 msg: *const os.msghdr_const,
737 flags: u32,
738 ) !*linux.io_uring_sqe {
739 const sqe = try self.get_sqe();
740 io_uring_prep_sendmsg(sqe, fd, msg, flags);
741 sqe.user_data = user_data;
742 return sqe;
743 }
744
745 /// Queues (but does not submit) an SQE to perform an async zerocopy `sendmsg(2)`.
746 /// Returns a pointer to the SQE.
747 /// Available since 6.1
748 pub fn sendmsg_zc(
749 self: *IO_Uring,
750 user_data: u64,
751 fd: os.fd_t,
752 msg: *const os.msghdr_const,
753 flags: u32,
754 ) !*linux.io_uring_sqe {
755 const sqe = try self.get_sqe();
756 io_uring_prep_sendmsg_zc(sqe, fd, msg, flags);
757 sqe.user_data = user_data;
758 return sqe;
759 }
760
761 /// Queues (but does not submit) an SQE to perform an `openat(2)`.
762 /// Returns a pointer to the SQE.
763 /// Available since 5.6.
764 pub fn openat(
765 self: *IO_Uring,
766 user_data: u64,
767 fd: os.fd_t,
768 path: [*:0]const u8,
769 flags: linux.O,
770 mode: os.mode_t,
771 ) !*linux.io_uring_sqe {
772 const sqe = try self.get_sqe();
773 io_uring_prep_openat(sqe, fd, path, flags, mode);
774 sqe.user_data = user_data;
775 return sqe;
776 }
777
778 /// Queues an openat using direct (registered) file descriptors.
779 ///
780 /// To use an accept direct variant, the application must first have registered
781 /// a file table (with register_files). An unused table index will be
782 /// dynamically chosen and returned in the CQE res field.
783 ///
784 /// After creation, they can be used by setting IOSQE_FIXED_FILE in the SQE
785 /// flags member, and setting the SQE fd field to the direct descriptor value
786 /// rather than the regular file descriptor.
787 ///
788 /// Available since 5.15
789 pub fn openat_direct(
790 self: *IO_Uring,
791 user_data: u64,
792 fd: os.fd_t,
793 path: [*:0]const u8,
794 flags: linux.O,
795 mode: os.mode_t,
796 file_index: u32,
797 ) !*linux.io_uring_sqe {
798 const sqe = try self.get_sqe();
799 io_uring_prep_openat_direct(sqe, fd, path, flags, mode, file_index);
800 sqe.user_data = user_data;
801 return sqe;
802 }
803
804 /// Queues (but does not submit) an SQE to perform a `close(2)`.
805 /// Returns a pointer to the SQE.
806 /// Available since 5.6.
807 pub fn close(self: *IO_Uring, user_data: u64, fd: os.fd_t) !*linux.io_uring_sqe {
808 const sqe = try self.get_sqe();
809 io_uring_prep_close(sqe, fd);
810 sqe.user_data = user_data;
811 return sqe;
812 }
813
814 /// Queues close of registered file descriptor.
815 /// Available since 5.15
816 pub fn close_direct(self: *IO_Uring, user_data: u64, file_index: u32) !*linux.io_uring_sqe {
817 const sqe = try self.get_sqe();
818 io_uring_prep_close_direct(sqe, file_index);
819 sqe.user_data = user_data;
820 return sqe;
821 }
822
823 /// Queues (but does not submit) an SQE to register a timeout operation.
824 /// Returns a pointer to the SQE.
825 ///
826 /// The timeout will complete when either the timeout expires, or after the specified number of
827 /// events complete (if `count` is greater than `0`).
828 ///
829 /// `flags` may be `0` for a relative timeout, or `IORING_TIMEOUT_ABS` for an absolute timeout.
830 ///
831 /// The completion event result will be `-ETIME` if the timeout completed through expiration,
832 /// `0` if the timeout completed after the specified number of events, or `-ECANCELED` if the
833 /// timeout was removed before it expired.
834 ///
835 /// io_uring timeouts use the `CLOCK.MONOTONIC` clock source.
836 pub fn timeout(
837 self: *IO_Uring,
838 user_data: u64,
839 ts: *const os.linux.kernel_timespec,
840 count: u32,
841 flags: u32,
842 ) !*linux.io_uring_sqe {
843 const sqe = try self.get_sqe();
844 io_uring_prep_timeout(sqe, ts, count, flags);
845 sqe.user_data = user_data;
846 return sqe;
847 }
848
849 /// Queues (but does not submit) an SQE to remove an existing timeout operation.
850 /// Returns a pointer to the SQE.
851 ///
852 /// The timeout is identified by its `user_data`.
853 ///
854 /// The completion event result will be `0` if the timeout was found and cancelled successfully,
855 /// `-EBUSY` if the timeout was found but expiration was already in progress, or
856 /// `-ENOENT` if the timeout was not found.
857 pub fn timeout_remove(
858 self: *IO_Uring,
859 user_data: u64,
860 timeout_user_data: u64,
861 flags: u32,
862 ) !*linux.io_uring_sqe {
863 const sqe = try self.get_sqe();
864 io_uring_prep_timeout_remove(sqe, timeout_user_data, flags);
865 sqe.user_data = user_data;
866 return sqe;
867 }
868
869 /// Queues (but does not submit) an SQE to add a link timeout operation.
870 /// Returns a pointer to the SQE.
871 ///
872 /// You need to set linux.IOSQE_IO_LINK to flags of the target operation
873 /// and then call this method right after the target operation.
874 /// See https://lwn.net/Articles/803932/ for detail.
875 ///
876 /// If the dependent request finishes before the linked timeout, the timeout
877 /// is canceled. If the timeout finishes before the dependent request, the
878 /// dependent request will be canceled.
879 ///
880 /// The completion event result of the link_timeout will be
881 /// `-ETIME` if the timeout finishes before the dependent request
882 /// (in this case, the completion event result of the dependent request will
883 /// be `-ECANCELED`), or
884 /// `-EALREADY` if the dependent request finishes before the linked timeout.
885 pub fn link_timeout(
886 self: *IO_Uring,
887 user_data: u64,
888 ts: *const os.linux.kernel_timespec,
889 flags: u32,
890 ) !*linux.io_uring_sqe {
891 const sqe = try self.get_sqe();
892 io_uring_prep_link_timeout(sqe, ts, flags);
893 sqe.user_data = user_data;
894 return sqe;
895 }
896
897 /// Queues (but does not submit) an SQE to perform a `poll(2)`.
898 /// Returns a pointer to the SQE.
899 pub fn poll_add(
900 self: *IO_Uring,
901 user_data: u64,
902 fd: os.fd_t,
903 poll_mask: u32,
904 ) !*linux.io_uring_sqe {
905 const sqe = try self.get_sqe();
906 io_uring_prep_poll_add(sqe, fd, poll_mask);
907 sqe.user_data = user_data;
908 return sqe;
909 }
910
911 /// Queues (but does not submit) an SQE to remove an existing poll operation.
912 /// Returns a pointer to the SQE.
913 pub fn poll_remove(
914 self: *IO_Uring,
915 user_data: u64,
916 target_user_data: u64,
917 ) !*linux.io_uring_sqe {
918 const sqe = try self.get_sqe();
919 io_uring_prep_poll_remove(sqe, target_user_data);
920 sqe.user_data = user_data;
921 return sqe;
922 }
923
924 /// Queues (but does not submit) an SQE to update the user data of an existing poll
925 /// operation. Returns a pointer to the SQE.
926 pub fn poll_update(
927 self: *IO_Uring,
928 user_data: u64,
929 old_user_data: u64,
930 new_user_data: u64,
931 poll_mask: u32,
932 flags: u32,
933 ) !*linux.io_uring_sqe {
934 const sqe = try self.get_sqe();
935 io_uring_prep_poll_update(sqe, old_user_data, new_user_data, poll_mask, flags);
936 sqe.user_data = user_data;
937 return sqe;
938 }
939
940 /// Queues (but does not submit) an SQE to perform an `fallocate(2)`.
941 /// Returns a pointer to the SQE.
942 pub fn fallocate(
943 self: *IO_Uring,
944 user_data: u64,
945 fd: os.fd_t,
946 mode: i32,
947 offset: u64,
948 len: u64,
949 ) !*linux.io_uring_sqe {
950 const sqe = try self.get_sqe();
951 io_uring_prep_fallocate(sqe, fd, mode, offset, len);
952 sqe.user_data = user_data;
953 return sqe;
954 }
955
956 /// Queues (but does not submit) an SQE to perform an `statx(2)`.
957 /// Returns a pointer to the SQE.
958 pub fn statx(
959 self: *IO_Uring,
960 user_data: u64,
961 fd: os.fd_t,
962 path: [:0]const u8,
963 flags: u32,
964 mask: u32,
965 buf: *linux.Statx,
966 ) !*linux.io_uring_sqe {
967 const sqe = try self.get_sqe();
968 io_uring_prep_statx(sqe, fd, path, flags, mask, buf);
969 sqe.user_data = user_data;
970 return sqe;
971 }
972
973 /// Queues (but does not submit) an SQE to remove an existing operation.
974 /// Returns a pointer to the SQE.
975 ///
976 /// The operation is identified by its `user_data`.
977 ///
978 /// The completion event result will be `0` if the operation was found and cancelled successfully,
979 /// `-EALREADY` if the operation was found but was already in progress, or
980 /// `-ENOENT` if the operation was not found.
981 pub fn cancel(
982 self: *IO_Uring,
983 user_data: u64,
984 cancel_user_data: u64,
985 flags: u32,
986 ) !*linux.io_uring_sqe {
987 const sqe = try self.get_sqe();
988 io_uring_prep_cancel(sqe, cancel_user_data, flags);
989 sqe.user_data = user_data;
990 return sqe;
991 }
992
993 /// Queues (but does not submit) an SQE to perform a `shutdown(2)`.
994 /// Returns a pointer to the SQE.
995 ///
996 /// The operation is identified by its `user_data`.
997 pub fn shutdown(
998 self: *IO_Uring,
999 user_data: u64,
1000 sockfd: os.socket_t,
1001 how: u32,
1002 ) !*linux.io_uring_sqe {
1003 const sqe = try self.get_sqe();
1004 io_uring_prep_shutdown(sqe, sockfd, how);
1005 sqe.user_data = user_data;
1006 return sqe;
1007 }
1008
1009 /// Queues (but does not submit) an SQE to perform a `renameat2(2)`.
1010 /// Returns a pointer to the SQE.
1011 pub fn renameat(
1012 self: *IO_Uring,
1013 user_data: u64,
1014 old_dir_fd: os.fd_t,
1015 old_path: [*:0]const u8,
1016 new_dir_fd: os.fd_t,
1017 new_path: [*:0]const u8,
1018 flags: u32,
1019 ) !*linux.io_uring_sqe {
1020 const sqe = try self.get_sqe();
1021 io_uring_prep_renameat(sqe, old_dir_fd, old_path, new_dir_fd, new_path, flags);
1022 sqe.user_data = user_data;
1023 return sqe;
1024 }
1025
1026 /// Queues (but does not submit) an SQE to perform a `unlinkat(2)`.
1027 /// Returns a pointer to the SQE.
1028 pub fn unlinkat(
1029 self: *IO_Uring,
1030 user_data: u64,
1031 dir_fd: os.fd_t,
1032 path: [*:0]const u8,
1033 flags: u32,
1034 ) !*linux.io_uring_sqe {
1035 const sqe = try self.get_sqe();
1036 io_uring_prep_unlinkat(sqe, dir_fd, path, flags);
1037 sqe.user_data = user_data;
1038 return sqe;
1039 }
1040
1041 /// Queues (but does not submit) an SQE to perform a `mkdirat(2)`.
1042 /// Returns a pointer to the SQE.
1043 pub fn mkdirat(
1044 self: *IO_Uring,
1045 user_data: u64,
1046 dir_fd: os.fd_t,
1047 path: [*:0]const u8,
1048 mode: os.mode_t,
1049 ) !*linux.io_uring_sqe {
1050 const sqe = try self.get_sqe();
1051 io_uring_prep_mkdirat(sqe, dir_fd, path, mode);
1052 sqe.user_data = user_data;
1053 return sqe;
1054 }
1055
1056 /// Queues (but does not submit) an SQE to perform a `symlinkat(2)`.
1057 /// Returns a pointer to the SQE.
1058 pub fn symlinkat(
1059 self: *IO_Uring,
1060 user_data: u64,
1061 target: [*:0]const u8,
1062 new_dir_fd: os.fd_t,
1063 link_path: [*:0]const u8,
1064 ) !*linux.io_uring_sqe {
1065 const sqe = try self.get_sqe();
1066 io_uring_prep_symlinkat(sqe, target, new_dir_fd, link_path);
1067 sqe.user_data = user_data;
1068 return sqe;
1069 }
1070
1071 /// Queues (but does not submit) an SQE to perform a `linkat(2)`.
1072 /// Returns a pointer to the SQE.
1073 pub fn linkat(
1074 self: *IO_Uring,
1075 user_data: u64,
1076 old_dir_fd: os.fd_t,
1077 old_path: [*:0]const u8,
1078 new_dir_fd: os.fd_t,
1079 new_path: [*:0]const u8,
1080 flags: u32,
1081 ) !*linux.io_uring_sqe {
1082 const sqe = try self.get_sqe();
1083 io_uring_prep_linkat(sqe, old_dir_fd, old_path, new_dir_fd, new_path, flags);
1084 sqe.user_data = user_data;
1085 return sqe;
1086 }
1087
1088 /// Queues (but does not submit) an SQE to provide a group of buffers used for commands that read/receive data.
1089 /// Returns a pointer to the SQE.
1090 ///
1091 /// Provided buffers can be used in `read`, `recv` or `recvmsg` commands via .buffer_selection.
1092 ///
1093 /// The kernel expects a contiguous block of memory of size (buffers_count * buffer_size).
1094 pub fn provide_buffers(
1095 self: *IO_Uring,
1096 user_data: u64,
1097 buffers: [*]u8,
1098 buffer_size: usize,
1099 buffers_count: usize,
1100 group_id: usize,
1101 buffer_id: usize,
1102 ) !*linux.io_uring_sqe {
1103 const sqe = try self.get_sqe();
1104 io_uring_prep_provide_buffers(sqe, buffers, buffer_size, buffers_count, group_id, buffer_id);
1105 sqe.user_data = user_data;
1106 return sqe;
1107 }
1108
1109 /// Queues (but does not submit) an SQE to remove a group of provided buffers.
1110 /// Returns a pointer to the SQE.
1111 pub fn remove_buffers(
1112 self: *IO_Uring,
1113 user_data: u64,
1114 buffers_count: usize,
1115 group_id: usize,
1116 ) !*linux.io_uring_sqe {
1117 const sqe = try self.get_sqe();
1118 io_uring_prep_remove_buffers(sqe, buffers_count, group_id);
1119 sqe.user_data = user_data;
1120 return sqe;
1121 }
1122
1123 /// Queues (but does not submit) an SQE to perform a `waitid(2)`.
1124 /// Returns a pointer to the SQE.
1125 pub fn waitid(
1126 self: *IO_Uring,
1127 user_data: u64,
1128 id_type: linux.P,
1129 id: i32,
1130 infop: *linux.siginfo_t,
1131 options: u32,
1132 flags: u32,
1133 ) !*linux.io_uring_sqe {
1134 const sqe = try self.get_sqe();
1135 io_uring_prep_waitid(sqe, id_type, id, infop, options, flags);
1136 sqe.user_data = user_data;
1137 return sqe;
1138 }
1139
1140 /// Registers an array of file descriptors.
1141 /// Every time a file descriptor is put in an SQE and submitted to the kernel, the kernel must
1142 /// retrieve a reference to the file, and once I/O has completed the file reference must be
1143 /// dropped. The atomic nature of this file reference can be a slowdown for high IOPS workloads.
1144 /// This slowdown can be avoided by pre-registering file descriptors.
1145 /// To refer to a registered file descriptor, IOSQE_FIXED_FILE must be set in the SQE's flags,
1146 /// and the SQE's fd must be set to the index of the file descriptor in the registered array.
1147 /// Registering file descriptors will wait for the ring to idle.
1148 /// Files are automatically unregistered by the kernel when the ring is torn down.
1149 /// An application need unregister only if it wants to register a new array of file descriptors.
1150 pub fn register_files(self: *IO_Uring, fds: []const os.fd_t) !void {
1151 assert(self.fd >= 0);
1152 const res = linux.io_uring_register(
1153 self.fd,
1154 .REGISTER_FILES,
1155 @as(*const anyopaque, @ptrCast(fds.ptr)),
1156 @as(u32, @intCast(fds.len)),
1157 );
1158 try handle_registration_result(res);
1159 }
1160
1161 /// Updates registered file descriptors.
1162 ///
1163 /// Updates are applied starting at the provided offset in the original file descriptors slice.
1164 /// There are three kind of updates:
1165 /// * turning a sparse entry (where the fd is -1) into a real one
1166 /// * removing an existing entry (set the fd to -1)
1167 /// * replacing an existing entry with a new fd
1168 /// Adding new file descriptors must be done with `register_files`.
1169 pub fn register_files_update(self: *IO_Uring, offset: u32, fds: []const os.fd_t) !void {
1170 assert(self.fd >= 0);
1171
1172 const FilesUpdate = extern struct {
1173 offset: u32,
1174 resv: u32,
1175 fds: u64 align(8),
1176 };
1177 var update = FilesUpdate{
1178 .offset = offset,
1179 .resv = @as(u32, 0),
1180 .fds = @as(u64, @intFromPtr(fds.ptr)),
1181 };
1182
1183 const res = linux.io_uring_register(
1184 self.fd,
1185 .REGISTER_FILES_UPDATE,
1186 @as(*const anyopaque, @ptrCast(&update)),
1187 @as(u32, @intCast(fds.len)),
1188 );
1189 try handle_registration_result(res);
1190 }
1191
1192 /// Registers the file descriptor for an eventfd that will be notified of completion events on
1193 /// an io_uring instance.
1194 /// Only a single a eventfd can be registered at any given point in time.
1195 pub fn register_eventfd(self: *IO_Uring, fd: os.fd_t) !void {
1196 assert(self.fd >= 0);
1197 const res = linux.io_uring_register(
1198 self.fd,
1199 .REGISTER_EVENTFD,
1200 @as(*const anyopaque, @ptrCast(&fd)),
1201 1,
1202 );
1203 try handle_registration_result(res);
1204 }
1205
1206 /// Registers the file descriptor for an eventfd that will be notified of completion events on
1207 /// an io_uring instance. Notifications are only posted for events that complete in an async manner.
1208 /// This means that events that complete inline while being submitted do not trigger a notification event.
1209 /// Only a single eventfd can be registered at any given point in time.
1210 pub fn register_eventfd_async(self: *IO_Uring, fd: os.fd_t) !void {
1211 assert(self.fd >= 0);
1212 const res = linux.io_uring_register(
1213 self.fd,
1214 .REGISTER_EVENTFD_ASYNC,
1215 @as(*const anyopaque, @ptrCast(&fd)),
1216 1,
1217 );
1218 try handle_registration_result(res);
1219 }
1220
1221 /// Unregister the registered eventfd file descriptor.
1222 pub fn unregister_eventfd(self: *IO_Uring) !void {
1223 assert(self.fd >= 0);
1224 const res = linux.io_uring_register(
1225 self.fd,
1226 .UNREGISTER_EVENTFD,
1227 null,
1228 0,
1229 );
1230 try handle_registration_result(res);
1231 }
1232
1233 /// Registers an array of buffers for use with `read_fixed` and `write_fixed`.
1234 pub fn register_buffers(self: *IO_Uring, buffers: []const os.iovec) !void {
1235 assert(self.fd >= 0);
1236 const res = linux.io_uring_register(
1237 self.fd,
1238 .REGISTER_BUFFERS,
1239 buffers.ptr,
1240 @as(u32, @intCast(buffers.len)),
1241 );
1242 try handle_registration_result(res);
1243 }
1244
1245 /// Unregister the registered buffers.
1246 pub fn unregister_buffers(self: *IO_Uring) !void {
1247 assert(self.fd >= 0);
1248 const res = linux.io_uring_register(self.fd, .UNREGISTER_BUFFERS, null, 0);
1249 switch (linux.getErrno(res)) {
1250 .SUCCESS => {},
1251 .NXIO => return error.BuffersNotRegistered,
1252 else => |errno| return os.unexpectedErrno(errno),
1253 }
1254 }
1255
1256 fn handle_registration_result(res: usize) !void {
1257 switch (linux.getErrno(res)) {
1258 .SUCCESS => {},
1259 // One or more fds in the array are invalid, or the kernel does not support sparse sets:
1260 .BADF => return error.FileDescriptorInvalid,
1261 .BUSY => return error.FilesAlreadyRegistered,
1262 .INVAL => return error.FilesEmpty,
1263 // Adding `nr_args` file references would exceed the maximum allowed number of files the
1264 // user is allowed to have according to the per-user RLIMIT_NOFILE resource limit and
1265 // the CAP_SYS_RESOURCE capability is not set, or `nr_args` exceeds the maximum allowed
1266 // for a fixed file set (older kernels have a limit of 1024 files vs 64K files):
1267 .MFILE => return error.UserFdQuotaExceeded,
1268 // Insufficient kernel resources, or the caller had a non-zero RLIMIT_MEMLOCK soft
1269 // resource limit but tried to lock more memory than the limit permitted (not enforced
1270 // when the process is privileged with CAP_IPC_LOCK):
1271 .NOMEM => return error.SystemResources,
1272 // Attempt to register files on a ring already registering files or being torn down:
1273 .NXIO => return error.RingShuttingDownOrAlreadyRegisteringFiles,
1274 else => |errno| return os.unexpectedErrno(errno),
1275 }
1276 }
1277
1278 /// Unregisters all registered file descriptors previously associated with the ring.
1279 pub fn unregister_files(self: *IO_Uring) !void {
1280 assert(self.fd >= 0);
1281 const res = linux.io_uring_register(self.fd, .UNREGISTER_FILES, null, 0);
1282 switch (linux.getErrno(res)) {
1283 .SUCCESS => {},
1284 .NXIO => return error.FilesNotRegistered,
1285 else => |errno| return os.unexpectedErrno(errno),
1286 }
1287 }
1288
1289 /// Prepares a socket creation request.
1290 /// New socket fd will be returned in completion result.
1291 /// Available since 5.19
1292 pub fn socket(
1293 self: *IO_Uring,
1294 user_data: u64,
1295 domain: u32,
1296 socket_type: u32,
1297 protocol: u32,
1298 flags: u32,
1299 ) !*linux.io_uring_sqe {
1300 const sqe = try self.get_sqe();
1301 io_uring_prep_socket(sqe, domain, socket_type, protocol, flags);
1302 sqe.user_data = user_data;
1303 return sqe;
1304 }
1305
1306 /// Prepares a socket creation request for registered file at index `file_index`.
1307 /// Available since 5.19
1308 pub fn socket_direct(
1309 self: *IO_Uring,
1310 user_data: u64,
1311 domain: u32,
1312 socket_type: u32,
1313 protocol: u32,
1314 flags: u32,
1315 file_index: u32,
1316 ) !*linux.io_uring_sqe {
1317 const sqe = try self.get_sqe();
1318 io_uring_prep_socket_direct(sqe, domain, socket_type, protocol, flags, file_index);
1319 sqe.user_data = user_data;
1320 return sqe;
1321 }
1322
1323 /// Prepares a socket creation request for registered file, index chosen by kernel (file index alloc).
1324 /// File index will be returned in CQE res field.
1325 /// Available since 5.19
1326 pub fn socket_direct_alloc(
1327 self: *IO_Uring,
1328 user_data: u64,
1329 domain: u32,
1330 socket_type: u32,
1331 protocol: u32,
1332 flags: u32,
1333 ) !*linux.io_uring_sqe {
1334 const sqe = try self.get_sqe();
1335 io_uring_prep_socket_direct_alloc(sqe, domain, socket_type, protocol, flags);
1336 sqe.user_data = user_data;
1337 return sqe;
1338 }
1339};
1340
1341pub const SubmissionQueue = struct {
1342 head: *u32,
1343 tail: *u32,
1344 mask: u32,
1345 flags: *u32,
1346 dropped: *u32,
1347 array: []u32,
1348 sqes: []linux.io_uring_sqe,
1349 mmap: []align(mem.page_size) u8,
1350 mmap_sqes: []align(mem.page_size) u8,
1351
1352 // We use `sqe_head` and `sqe_tail` in the same way as liburing:
1353 // We increment `sqe_tail` (but not `tail`) for each call to `get_sqe()`.
1354 // We then set `tail` to `sqe_tail` once, only when these events are actually submitted.
1355 // This allows us to amortize the cost of the @atomicStore to `tail` across multiple SQEs.
1356 sqe_head: u32 = 0,
1357 sqe_tail: u32 = 0,
1358
1359 pub fn init(fd: os.fd_t, p: linux.io_uring_params) !SubmissionQueue {
1360 assert(fd >= 0);
1361 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1362 const size = @max(
1363 p.sq_off.array + p.sq_entries * @sizeOf(u32),
1364 p.cq_off.cqes + p.cq_entries * @sizeOf(linux.io_uring_cqe),
1365 );
1366 const mmap = try os.mmap(
1367 null,
1368 size,
1369 os.PROT.READ | os.PROT.WRITE,
1370 .{ .TYPE = .SHARED, .POPULATE = true },
1371 fd,
1372 linux.IORING_OFF_SQ_RING,
1373 );
1374 errdefer os.munmap(mmap);
1375 assert(mmap.len == size);
1376
1377 // The motivation for the `sqes` and `array` indirection is to make it possible for the
1378 // application to preallocate static linux.io_uring_sqe entries and then replay them when needed.
1379 const size_sqes = p.sq_entries * @sizeOf(linux.io_uring_sqe);
1380 const mmap_sqes = try os.mmap(
1381 null,
1382 size_sqes,
1383 os.PROT.READ | os.PROT.WRITE,
1384 .{ .TYPE = .SHARED, .POPULATE = true },
1385 fd,
1386 linux.IORING_OFF_SQES,
1387 );
1388 errdefer os.munmap(mmap_sqes);
1389 assert(mmap_sqes.len == size_sqes);
1390
1391 const array: [*]u32 = @ptrCast(@alignCast(&mmap[p.sq_off.array]));
1392 const sqes: [*]linux.io_uring_sqe = @ptrCast(@alignCast(&mmap_sqes[0]));
1393 // We expect the kernel copies p.sq_entries to the u32 pointed to by p.sq_off.ring_entries,
1394 // see https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L7843-L7844.
1395 assert(p.sq_entries == @as(*u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_entries]))).*);
1396 return SubmissionQueue{
1397 .head = @ptrCast(@alignCast(&mmap[p.sq_off.head])),
1398 .tail = @ptrCast(@alignCast(&mmap[p.sq_off.tail])),
1399 .mask = @as(*u32, @ptrCast(@alignCast(&mmap[p.sq_off.ring_mask]))).*,
1400 .flags = @ptrCast(@alignCast(&mmap[p.sq_off.flags])),
1401 .dropped = @ptrCast(@alignCast(&mmap[p.sq_off.dropped])),
1402 .array = array[0..p.sq_entries],
1403 .sqes = sqes[0..p.sq_entries],
1404 .mmap = mmap,
1405 .mmap_sqes = mmap_sqes,
1406 };
1407 }
1408
1409 pub fn deinit(self: *SubmissionQueue) void {
1410 os.munmap(self.mmap_sqes);
1411 os.munmap(self.mmap);
1412 }
1413};
1414
1415pub const CompletionQueue = struct {
1416 head: *u32,
1417 tail: *u32,
1418 mask: u32,
1419 overflow: *u32,
1420 cqes: []linux.io_uring_cqe,
1421
1422 pub fn init(fd: os.fd_t, p: linux.io_uring_params, sq: SubmissionQueue) !CompletionQueue {
1423 assert(fd >= 0);
1424 assert((p.features & linux.IORING_FEAT_SINGLE_MMAP) != 0);
1425 const mmap = sq.mmap;
1426 const cqes: [*]linux.io_uring_cqe = @ptrCast(@alignCast(&mmap[p.cq_off.cqes]));
1427 assert(p.cq_entries == @as(*u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_entries]))).*);
1428 return CompletionQueue{
1429 .head = @ptrCast(@alignCast(&mmap[p.cq_off.head])),
1430 .tail = @ptrCast(@alignCast(&mmap[p.cq_off.tail])),
1431 .mask = @as(*u32, @ptrCast(@alignCast(&mmap[p.cq_off.ring_mask]))).*,
1432 .overflow = @ptrCast(@alignCast(&mmap[p.cq_off.overflow])),
1433 .cqes = cqes[0..p.cq_entries],
1434 };
1435 }
1436
1437 pub fn deinit(self: *CompletionQueue) void {
1438 _ = self;
1439 // A no-op since we now share the mmap with the submission queue.
1440 // Here for symmetry with the submission queue, and for any future feature support.
1441 }
1442};
1443
1444pub fn io_uring_prep_nop(sqe: *linux.io_uring_sqe) void {
1445 sqe.* = .{
1446 .opcode = .NOP,
1447 .flags = 0,
1448 .ioprio = 0,
1449 .fd = 0,
1450 .off = 0,
1451 .addr = 0,
1452 .len = 0,
1453 .rw_flags = 0,
1454 .user_data = 0,
1455 .buf_index = 0,
1456 .personality = 0,
1457 .splice_fd_in = 0,
1458 .addr3 = 0,
1459 .resv = 0,
1460 };
1461}
1462
1463pub fn io_uring_prep_fsync(sqe: *linux.io_uring_sqe, fd: os.fd_t, flags: u32) void {
1464 sqe.* = .{
1465 .opcode = .FSYNC,
1466 .flags = 0,
1467 .ioprio = 0,
1468 .fd = fd,
1469 .off = 0,
1470 .addr = 0,
1471 .len = 0,
1472 .rw_flags = flags,
1473 .user_data = 0,
1474 .buf_index = 0,
1475 .personality = 0,
1476 .splice_fd_in = 0,
1477 .addr3 = 0,
1478 .resv = 0,
1479 };
1480}
1481
1482pub fn io_uring_prep_rw(
1483 op: linux.IORING_OP,
1484 sqe: *linux.io_uring_sqe,
1485 fd: os.fd_t,
1486 addr: u64,
1487 len: usize,
1488 offset: u64,
1489) void {
1490 sqe.* = .{
1491 .opcode = op,
1492 .flags = 0,
1493 .ioprio = 0,
1494 .fd = fd,
1495 .off = offset,
1496 .addr = addr,
1497 .len = @as(u32, @intCast(len)),
1498 .rw_flags = 0,
1499 .user_data = 0,
1500 .buf_index = 0,
1501 .personality = 0,
1502 .splice_fd_in = 0,
1503 .addr3 = 0,
1504 .resv = 0,
1505 };
1506}
1507
1508pub fn io_uring_prep_read(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, offset: u64) void {
1509 io_uring_prep_rw(.READ, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
1510}
1511
1512pub fn io_uring_prep_write(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, offset: u64) void {
1513 io_uring_prep_rw(.WRITE, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
1514}
1515
1516pub fn io_uring_prep_splice(sqe: *linux.io_uring_sqe, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) void {
1517 io_uring_prep_rw(.SPLICE, sqe, fd_out, undefined, len, off_out);
1518 sqe.addr = off_in;
1519 sqe.splice_fd_in = fd_in;
1520}
1521
1522pub fn io_uring_prep_readv(
1523 sqe: *linux.io_uring_sqe,
1524 fd: os.fd_t,
1525 iovecs: []const os.iovec,
1526 offset: u64,
1527) void {
1528 io_uring_prep_rw(.READV, sqe, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
1529}
1530
1531pub fn io_uring_prep_writev(
1532 sqe: *linux.io_uring_sqe,
1533 fd: os.fd_t,
1534 iovecs: []const os.iovec_const,
1535 offset: u64,
1536) void {
1537 io_uring_prep_rw(.WRITEV, sqe, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
1538}
1539
1540pub fn io_uring_prep_read_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: *os.iovec, offset: u64, buffer_index: u16) void {
1541 io_uring_prep_rw(.READ_FIXED, sqe, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
1542 sqe.buf_index = buffer_index;
1543}
1544
1545pub fn io_uring_prep_write_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: *os.iovec, offset: u64, buffer_index: u16) void {
1546 io_uring_prep_rw(.WRITE_FIXED, sqe, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
1547 sqe.buf_index = buffer_index;
1548}
1549
1550/// Poll masks previously used to comprise of 16 bits in the flags union of
1551/// a SQE, but were then extended to comprise of 32 bits in order to make
1552/// room for additional option flags. To ensure that the correct bits of
1553/// poll masks are consistently and properly read across multiple kernel
1554/// versions, poll masks are enforced to be little-endian.
1555/// https://www.spinics.net/lists/io-uring/msg02848.html
1556pub inline fn __io_uring_prep_poll_mask(poll_mask: u32) u32 {
1557 return std.mem.nativeToLittle(u32, poll_mask);
1558}
1559
1560pub fn io_uring_prep_accept(
1561 sqe: *linux.io_uring_sqe,
1562 fd: os.fd_t,
1563 addr: ?*os.sockaddr,
1564 addrlen: ?*os.socklen_t,
1565 flags: u32,
1566) void {
1567 // `addr` holds a pointer to `sockaddr`, and `addr2` holds a pointer to socklen_t`.
1568 // `addr2` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
1569 io_uring_prep_rw(.ACCEPT, sqe, fd, @intFromPtr(addr), 0, @intFromPtr(addrlen));
1570 sqe.rw_flags = flags;
1571}
1572
1573pub fn io_uring_prep_accept_direct(
1574 sqe: *linux.io_uring_sqe,
1575 fd: os.fd_t,
1576 addr: ?*os.sockaddr,
1577 addrlen: ?*os.socklen_t,
1578 flags: u32,
1579 file_index: u32,
1580) void {
1581 io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
1582 __io_uring_set_target_fixed_file(sqe, file_index);
1583}
1584
1585pub fn io_uring_prep_multishot_accept_direct(
1586 sqe: *linux.io_uring_sqe,
1587 fd: os.fd_t,
1588 addr: ?*os.sockaddr,
1589 addrlen: ?*os.socklen_t,
1590 flags: u32,
1591) void {
1592 io_uring_prep_multishot_accept(sqe, fd, addr, addrlen, flags);
1593 __io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
1594}
1595
1596fn __io_uring_set_target_fixed_file(sqe: *linux.io_uring_sqe, file_index: u32) void {
1597 const sqe_file_index: u32 = if (file_index == linux.IORING_FILE_INDEX_ALLOC)
1598 linux.IORING_FILE_INDEX_ALLOC
1599 else
1600 // 0 means no fixed files, indexes should be encoded as "index + 1"
1601 file_index + 1;
1602 // This filed is overloaded in liburing:
1603 // splice_fd_in: i32
1604 // sqe_file_index: u32
1605 sqe.splice_fd_in = @bitCast(sqe_file_index);
1606}
1607
1608pub fn io_uring_prep_connect(
1609 sqe: *linux.io_uring_sqe,
1610 fd: os.fd_t,
1611 addr: *const os.sockaddr,
1612 addrlen: os.socklen_t,
1613) void {
1614 // `addrlen` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
1615 io_uring_prep_rw(.CONNECT, sqe, fd, @intFromPtr(addr), 0, addrlen);
1616}
1617
1618pub fn io_uring_prep_epoll_ctl(
1619 sqe: *linux.io_uring_sqe,
1620 epfd: os.fd_t,
1621 fd: os.fd_t,
1622 op: u32,
1623 ev: ?*linux.epoll_event,
1624) void {
1625 io_uring_prep_rw(.EPOLL_CTL, sqe, epfd, @intFromPtr(ev), op, @as(u64, @intCast(fd)));
1626}
1627
1628pub fn io_uring_prep_recv(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, flags: u32) void {
1629 io_uring_prep_rw(.RECV, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
1630 sqe.rw_flags = flags;
1631}
1632
1633pub fn io_uring_prep_send(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32) void {
1634 io_uring_prep_rw(.SEND, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
1635 sqe.rw_flags = flags;
1636}
1637
1638pub fn io_uring_prep_send_zc(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16) void {
1639 io_uring_prep_rw(.SEND_ZC, sqe, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
1640 sqe.rw_flags = flags;
1641 sqe.ioprio = zc_flags;
1642}
1643
1644pub fn io_uring_prep_send_zc_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16, buf_index: u16) void {
1645 io_uring_prep_send_zc(sqe, fd, buffer, flags, zc_flags);
1646 sqe.ioprio |= linux.IORING_RECVSEND_FIXED_BUF;
1647 sqe.buf_index = buf_index;
1648}
1649
1650pub fn io_uring_prep_sendmsg_zc(
1651 sqe: *linux.io_uring_sqe,
1652 fd: os.fd_t,
1653 msg: *const os.msghdr_const,
1654 flags: u32,
1655) void {
1656 io_uring_prep_sendmsg(sqe, fd, msg, flags);
1657 sqe.opcode = .SENDMSG_ZC;
1658}
1659
1660pub fn io_uring_prep_recvmsg(
1661 sqe: *linux.io_uring_sqe,
1662 fd: os.fd_t,
1663 msg: *os.msghdr,
1664 flags: u32,
1665) void {
1666 linux.io_uring_prep_rw(.RECVMSG, sqe, fd, @intFromPtr(msg), 1, 0);
1667 sqe.rw_flags = flags;
1668}
1669
1670pub fn io_uring_prep_sendmsg(
1671 sqe: *linux.io_uring_sqe,
1672 fd: os.fd_t,
1673 msg: *const os.msghdr_const,
1674 flags: u32,
1675) void {
1676 linux.io_uring_prep_rw(.SENDMSG, sqe, fd, @intFromPtr(msg), 1, 0);
1677 sqe.rw_flags = flags;
1678}
1679
1680pub fn io_uring_prep_openat(
1681 sqe: *linux.io_uring_sqe,
1682 fd: os.fd_t,
1683 path: [*:0]const u8,
1684 flags: linux.O,
1685 mode: os.mode_t,
1686) void {
1687 io_uring_prep_rw(.OPENAT, sqe, fd, @intFromPtr(path), mode, 0);
1688 sqe.rw_flags = @bitCast(flags);
1689}
1690
1691pub fn io_uring_prep_openat_direct(
1692 sqe: *linux.io_uring_sqe,
1693 fd: os.fd_t,
1694 path: [*:0]const u8,
1695 flags: linux.O,
1696 mode: os.mode_t,
1697 file_index: u32,
1698) void {
1699 io_uring_prep_openat(sqe, fd, path, flags, mode);
1700 __io_uring_set_target_fixed_file(sqe, file_index);
1701}
1702
1703pub fn io_uring_prep_close(sqe: *linux.io_uring_sqe, fd: os.fd_t) void {
1704 sqe.* = .{
1705 .opcode = .CLOSE,
1706 .flags = 0,
1707 .ioprio = 0,
1708 .fd = fd,
1709 .off = 0,
1710 .addr = 0,
1711 .len = 0,
1712 .rw_flags = 0,
1713 .user_data = 0,
1714 .buf_index = 0,
1715 .personality = 0,
1716 .splice_fd_in = 0,
1717 .addr3 = 0,
1718 .resv = 0,
1719 };
1720}
1721
1722pub fn io_uring_prep_close_direct(sqe: *linux.io_uring_sqe, file_index: u32) void {
1723 io_uring_prep_close(sqe, 0);
1724 __io_uring_set_target_fixed_file(sqe, file_index);
1725}
1726
1727pub fn io_uring_prep_timeout(
1728 sqe: *linux.io_uring_sqe,
1729 ts: *const os.linux.kernel_timespec,
1730 count: u32,
1731 flags: u32,
1732) void {
1733 io_uring_prep_rw(.TIMEOUT, sqe, -1, @intFromPtr(ts), 1, count);
1734 sqe.rw_flags = flags;
1735}
1736
1737pub fn io_uring_prep_timeout_remove(sqe: *linux.io_uring_sqe, timeout_user_data: u64, flags: u32) void {
1738 sqe.* = .{
1739 .opcode = .TIMEOUT_REMOVE,
1740 .flags = 0,
1741 .ioprio = 0,
1742 .fd = -1,
1743 .off = 0,
1744 .addr = timeout_user_data,
1745 .len = 0,
1746 .rw_flags = flags,
1747 .user_data = 0,
1748 .buf_index = 0,
1749 .personality = 0,
1750 .splice_fd_in = 0,
1751 .addr3 = 0,
1752 .resv = 0,
1753 };
1754}
1755
1756pub fn io_uring_prep_link_timeout(
1757 sqe: *linux.io_uring_sqe,
1758 ts: *const os.linux.kernel_timespec,
1759 flags: u32,
1760) void {
1761 linux.io_uring_prep_rw(.LINK_TIMEOUT, sqe, -1, @intFromPtr(ts), 1, 0);
1762 sqe.rw_flags = flags;
1763}
1764
1765pub fn io_uring_prep_poll_add(
1766 sqe: *linux.io_uring_sqe,
1767 fd: os.fd_t,
1768 poll_mask: u32,
1769) void {
1770 io_uring_prep_rw(.POLL_ADD, sqe, fd, @intFromPtr(@as(?*anyopaque, null)), 0, 0);
1771 sqe.rw_flags = __io_uring_prep_poll_mask(poll_mask);
1772}
1773
1774pub fn io_uring_prep_poll_remove(
1775 sqe: *linux.io_uring_sqe,
1776 target_user_data: u64,
1777) void {
1778 io_uring_prep_rw(.POLL_REMOVE, sqe, -1, target_user_data, 0, 0);
1779}
1780
1781pub fn io_uring_prep_poll_update(
1782 sqe: *linux.io_uring_sqe,
1783 old_user_data: u64,
1784 new_user_data: u64,
1785 poll_mask: u32,
1786 flags: u32,
1787) void {
1788 io_uring_prep_rw(.POLL_REMOVE, sqe, -1, old_user_data, flags, new_user_data);
1789 sqe.rw_flags = __io_uring_prep_poll_mask(poll_mask);
1790}
1791
1792pub fn io_uring_prep_fallocate(
1793 sqe: *linux.io_uring_sqe,
1794 fd: os.fd_t,
1795 mode: i32,
1796 offset: u64,
1797 len: u64,
1798) void {
1799 sqe.* = .{
1800 .opcode = .FALLOCATE,
1801 .flags = 0,
1802 .ioprio = 0,
1803 .fd = fd,
1804 .off = offset,
1805 .addr = len,
1806 .len = @as(u32, @intCast(mode)),
1807 .rw_flags = 0,
1808 .user_data = 0,
1809 .buf_index = 0,
1810 .personality = 0,
1811 .splice_fd_in = 0,
1812 .addr3 = 0,
1813 .resv = 0,
1814 };
1815}
1816
1817pub fn io_uring_prep_statx(
1818 sqe: *linux.io_uring_sqe,
1819 fd: os.fd_t,
1820 path: [*:0]const u8,
1821 flags: u32,
1822 mask: u32,
1823 buf: *linux.Statx,
1824) void {
1825 io_uring_prep_rw(.STATX, sqe, fd, @intFromPtr(path), mask, @intFromPtr(buf));
1826 sqe.rw_flags = flags;
1827}
1828
1829pub fn io_uring_prep_cancel(
1830 sqe: *linux.io_uring_sqe,
1831 cancel_user_data: u64,
1832 flags: u32,
1833) void {
1834 io_uring_prep_rw(.ASYNC_CANCEL, sqe, -1, cancel_user_data, 0, 0);
1835 sqe.rw_flags = flags;
1836}
1837
1838pub fn io_uring_prep_shutdown(
1839 sqe: *linux.io_uring_sqe,
1840 sockfd: os.socket_t,
1841 how: u32,
1842) void {
1843 io_uring_prep_rw(.SHUTDOWN, sqe, sockfd, 0, how, 0);
1844}
1845
1846pub fn io_uring_prep_renameat(
1847 sqe: *linux.io_uring_sqe,
1848 old_dir_fd: os.fd_t,
1849 old_path: [*:0]const u8,
1850 new_dir_fd: os.fd_t,
1851 new_path: [*:0]const u8,
1852 flags: u32,
1853) void {
1854 io_uring_prep_rw(
1855 .RENAMEAT,
1856 sqe,
1857 old_dir_fd,
1858 @intFromPtr(old_path),
1859 0,
1860 @intFromPtr(new_path),
1861 );
1862 sqe.len = @bitCast(new_dir_fd);
1863 sqe.rw_flags = flags;
1864}
1865
1866pub fn io_uring_prep_unlinkat(
1867 sqe: *linux.io_uring_sqe,
1868 dir_fd: os.fd_t,
1869 path: [*:0]const u8,
1870 flags: u32,
1871) void {
1872 io_uring_prep_rw(.UNLINKAT, sqe, dir_fd, @intFromPtr(path), 0, 0);
1873 sqe.rw_flags = flags;
1874}
1875
1876pub fn io_uring_prep_mkdirat(
1877 sqe: *linux.io_uring_sqe,
1878 dir_fd: os.fd_t,
1879 path: [*:0]const u8,
1880 mode: os.mode_t,
1881) void {
1882 io_uring_prep_rw(.MKDIRAT, sqe, dir_fd, @intFromPtr(path), mode, 0);
1883}
1884
1885pub fn io_uring_prep_symlinkat(
1886 sqe: *linux.io_uring_sqe,
1887 target: [*:0]const u8,
1888 new_dir_fd: os.fd_t,
1889 link_path: [*:0]const u8,
1890) void {
1891 io_uring_prep_rw(
1892 .SYMLINKAT,
1893 sqe,
1894 new_dir_fd,
1895 @intFromPtr(target),
1896 0,
1897 @intFromPtr(link_path),
1898 );
1899}
1900
1901pub fn io_uring_prep_linkat(
1902 sqe: *linux.io_uring_sqe,
1903 old_dir_fd: os.fd_t,
1904 old_path: [*:0]const u8,
1905 new_dir_fd: os.fd_t,
1906 new_path: [*:0]const u8,
1907 flags: u32,
1908) void {
1909 io_uring_prep_rw(
1910 .LINKAT,
1911 sqe,
1912 old_dir_fd,
1913 @intFromPtr(old_path),
1914 0,
1915 @intFromPtr(new_path),
1916 );
1917 sqe.len = @bitCast(new_dir_fd);
1918 sqe.rw_flags = flags;
1919}
1920
1921pub fn io_uring_prep_provide_buffers(
1922 sqe: *linux.io_uring_sqe,
1923 buffers: [*]u8,
1924 buffer_len: usize,
1925 num: usize,
1926 group_id: usize,
1927 buffer_id: usize,
1928) void {
1929 const ptr = @intFromPtr(buffers);
1930 io_uring_prep_rw(.PROVIDE_BUFFERS, sqe, @as(i32, @intCast(num)), ptr, buffer_len, buffer_id);
1931 sqe.buf_index = @intCast(group_id);
1932}
1933
1934pub fn io_uring_prep_remove_buffers(
1935 sqe: *linux.io_uring_sqe,
1936 num: usize,
1937 group_id: usize,
1938) void {
1939 io_uring_prep_rw(.REMOVE_BUFFERS, sqe, @as(i32, @intCast(num)), 0, 0, 0);
1940 sqe.buf_index = @intCast(group_id);
1941}
1942
1943pub fn io_uring_prep_multishot_accept(
1944 sqe: *linux.io_uring_sqe,
1945 fd: os.fd_t,
1946 addr: ?*os.sockaddr,
1947 addrlen: ?*os.socklen_t,
1948 flags: u32,
1949) void {
1950 io_uring_prep_accept(sqe, fd, addr, addrlen, flags);
1951 sqe.ioprio |= linux.IORING_ACCEPT_MULTISHOT;
1952}
1953
1954pub fn io_uring_prep_socket(
1955 sqe: *linux.io_uring_sqe,
1956 domain: u32,
1957 socket_type: u32,
1958 protocol: u32,
1959 flags: u32,
1960) void {
1961 io_uring_prep_rw(.SOCKET, sqe, @intCast(domain), 0, protocol, socket_type);
1962 sqe.rw_flags = flags;
1963}
1964
1965pub fn io_uring_prep_socket_direct(
1966 sqe: *linux.io_uring_sqe,
1967 domain: u32,
1968 socket_type: u32,
1969 protocol: u32,
1970 flags: u32,
1971 file_index: u32,
1972) void {
1973 io_uring_prep_socket(sqe, domain, socket_type, protocol, flags);
1974 __io_uring_set_target_fixed_file(sqe, file_index);
1975}
1976
1977pub fn io_uring_prep_socket_direct_alloc(
1978 sqe: *linux.io_uring_sqe,
1979 domain: u32,
1980 socket_type: u32,
1981 protocol: u32,
1982 flags: u32,
1983) void {
1984 io_uring_prep_socket(sqe, domain, socket_type, protocol, flags);
1985 __io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
1986}
1987
1988pub fn io_uring_prep_waitid(
1989 sqe: *linux.io_uring_sqe,
1990 id_type: linux.P,
1991 id: i32,
1992 infop: *linux.siginfo_t,
1993 options: u32,
1994 flags: u32,
1995) void {
1996 io_uring_prep_rw(.WAITID, sqe, id, 0, @intFromEnum(id_type), @intFromPtr(infop));
1997 sqe.rw_flags = flags;
1998 sqe.splice_fd_in = @bitCast(options);
1999}
2000
2001test "structs/offsets/entries" {
2002 if (builtin.os.tag != .linux) return error.SkipZigTest;
2003
2004 try testing.expectEqual(@as(usize, 120), @sizeOf(linux.io_uring_params));
2005 try testing.expectEqual(@as(usize, 64), @sizeOf(linux.io_uring_sqe));
2006 try testing.expectEqual(@as(usize, 16), @sizeOf(linux.io_uring_cqe));
2007
2008 try testing.expectEqual(0, linux.IORING_OFF_SQ_RING);
2009 try testing.expectEqual(0x8000000, linux.IORING_OFF_CQ_RING);
2010 try testing.expectEqual(0x10000000, linux.IORING_OFF_SQES);
2011
2012 try testing.expectError(error.EntriesZero, IO_Uring.init(0, 0));
2013 try testing.expectError(error.EntriesNotPowerOfTwo, IO_Uring.init(3, 0));
2014}
2015
2016test "nop" {
2017 if (builtin.os.tag != .linux) return error.SkipZigTest;
2018
2019 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2020 error.SystemOutdated => return error.SkipZigTest,
2021 error.PermissionDenied => return error.SkipZigTest,
2022 else => return err,
2023 };
2024 defer {
2025 ring.deinit();
2026 testing.expectEqual(@as(os.fd_t, -1), ring.fd) catch @panic("test failed");
2027 }
2028
2029 const sqe = try ring.nop(0xaaaaaaaa);
2030 try testing.expectEqual(linux.io_uring_sqe{
2031 .opcode = .NOP,
2032 .flags = 0,
2033 .ioprio = 0,
2034 .fd = 0,
2035 .off = 0,
2036 .addr = 0,
2037 .len = 0,
2038 .rw_flags = 0,
2039 .user_data = 0xaaaaaaaa,
2040 .buf_index = 0,
2041 .personality = 0,
2042 .splice_fd_in = 0,
2043 .addr3 = 0,
2044 .resv = 0,
2045 }, sqe.*);
2046
2047 try testing.expectEqual(@as(u32, 0), ring.sq.sqe_head);
2048 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
2049 try testing.expectEqual(@as(u32, 0), ring.sq.tail.*);
2050 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
2051 try testing.expectEqual(@as(u32, 1), ring.sq_ready());
2052 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2053
2054 try testing.expectEqual(@as(u32, 1), try ring.submit());
2055 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_head);
2056 try testing.expectEqual(@as(u32, 1), ring.sq.sqe_tail);
2057 try testing.expectEqual(@as(u32, 1), ring.sq.tail.*);
2058 try testing.expectEqual(@as(u32, 0), ring.cq.head.*);
2059 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2060
2061 try testing.expectEqual(linux.io_uring_cqe{
2062 .user_data = 0xaaaaaaaa,
2063 .res = 0,
2064 .flags = 0,
2065 }, try ring.copy_cqe());
2066 try testing.expectEqual(@as(u32, 1), ring.cq.head.*);
2067 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2068
2069 const sqe_barrier = try ring.nop(0xbbbbbbbb);
2070 sqe_barrier.flags |= linux.IOSQE_IO_DRAIN;
2071 try testing.expectEqual(@as(u32, 1), try ring.submit());
2072 try testing.expectEqual(linux.io_uring_cqe{
2073 .user_data = 0xbbbbbbbb,
2074 .res = 0,
2075 .flags = 0,
2076 }, try ring.copy_cqe());
2077 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_head);
2078 try testing.expectEqual(@as(u32, 2), ring.sq.sqe_tail);
2079 try testing.expectEqual(@as(u32, 2), ring.sq.tail.*);
2080 try testing.expectEqual(@as(u32, 2), ring.cq.head.*);
2081}
2082
2083test "readv" {
2084 if (builtin.os.tag != .linux) return error.SkipZigTest;
2085
2086 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2087 error.SystemOutdated => return error.SkipZigTest,
2088 error.PermissionDenied => return error.SkipZigTest,
2089 else => return err,
2090 };
2091 defer ring.deinit();
2092
2093 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2094 defer os.close(fd);
2095
2096 // Linux Kernel 5.4 supports IORING_REGISTER_FILES but not sparse fd sets (i.e. an fd of -1).
2097 // Linux Kernel 5.5 adds support for sparse fd sets.
2098 // Compare:
2099 // https://github.com/torvalds/linux/blob/v5.4/fs/io_uring.c#L3119-L3124 vs
2100 // https://github.com/torvalds/linux/blob/v5.8/fs/io_uring.c#L6687-L6691
2101 // We therefore avoid stressing sparse fd sets here:
2102 var registered_fds = [_]os.fd_t{0} ** 1;
2103 const fd_index = 0;
2104 registered_fds[fd_index] = fd;
2105 try ring.register_files(registered_fds[0..]);
2106
2107 var buffer = [_]u8{42} ** 128;
2108 var iovecs = [_]os.iovec{os.iovec{ .iov_base = &buffer, .iov_len = buffer.len }};
2109 const sqe = try ring.read(0xcccccccc, fd_index, .{ .iovecs = iovecs[0..] }, 0);
2110 try testing.expectEqual(linux.IORING_OP.READV, sqe.opcode);
2111 sqe.flags |= linux.IOSQE_FIXED_FILE;
2112
2113 try testing.expectError(error.SubmissionQueueFull, ring.nop(0));
2114 try testing.expectEqual(@as(u32, 1), try ring.submit());
2115 try testing.expectEqual(linux.io_uring_cqe{
2116 .user_data = 0xcccccccc,
2117 .res = buffer.len,
2118 .flags = 0,
2119 }, try ring.copy_cqe());
2120 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2121
2122 try ring.unregister_files();
2123}
2124
2125test "writev/fsync/readv" {
2126 if (builtin.os.tag != .linux) return error.SkipZigTest;
2127
2128 var ring = IO_Uring.init(4, 0) catch |err| switch (err) {
2129 error.SystemOutdated => return error.SkipZigTest,
2130 error.PermissionDenied => return error.SkipZigTest,
2131 else => return err,
2132 };
2133 defer ring.deinit();
2134
2135 var tmp = std.testing.tmpDir(.{});
2136 defer tmp.cleanup();
2137
2138 const path = "test_io_uring_writev_fsync_readv";
2139 const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
2140 defer file.close();
2141 const fd = file.handle;
2142
2143 const buffer_write = [_]u8{42} ** 128;
2144 const iovecs_write = [_]os.iovec_const{
2145 os.iovec_const{ .iov_base = &buffer_write, .iov_len = buffer_write.len },
2146 };
2147 var buffer_read = [_]u8{0} ** 128;
2148 var iovecs_read = [_]os.iovec{
2149 os.iovec{ .iov_base = &buffer_read, .iov_len = buffer_read.len },
2150 };
2151
2152 const sqe_writev = try ring.writev(0xdddddddd, fd, iovecs_write[0..], 17);
2153 try testing.expectEqual(linux.IORING_OP.WRITEV, sqe_writev.opcode);
2154 try testing.expectEqual(@as(u64, 17), sqe_writev.off);
2155 sqe_writev.flags |= linux.IOSQE_IO_LINK;
2156
2157 const sqe_fsync = try ring.fsync(0xeeeeeeee, fd, 0);
2158 try testing.expectEqual(linux.IORING_OP.FSYNC, sqe_fsync.opcode);
2159 try testing.expectEqual(fd, sqe_fsync.fd);
2160 sqe_fsync.flags |= linux.IOSQE_IO_LINK;
2161
2162 const sqe_readv = try ring.read(0xffffffff, fd, .{ .iovecs = iovecs_read[0..] }, 17);
2163 try testing.expectEqual(linux.IORING_OP.READV, sqe_readv.opcode);
2164 try testing.expectEqual(@as(u64, 17), sqe_readv.off);
2165
2166 try testing.expectEqual(@as(u32, 3), ring.sq_ready());
2167 try testing.expectEqual(@as(u32, 3), try ring.submit_and_wait(3));
2168 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2169 try testing.expectEqual(@as(u32, 3), ring.cq_ready());
2170
2171 try testing.expectEqual(linux.io_uring_cqe{
2172 .user_data = 0xdddddddd,
2173 .res = buffer_write.len,
2174 .flags = 0,
2175 }, try ring.copy_cqe());
2176 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2177
2178 try testing.expectEqual(linux.io_uring_cqe{
2179 .user_data = 0xeeeeeeee,
2180 .res = 0,
2181 .flags = 0,
2182 }, try ring.copy_cqe());
2183 try testing.expectEqual(@as(u32, 1), ring.cq_ready());
2184
2185 try testing.expectEqual(linux.io_uring_cqe{
2186 .user_data = 0xffffffff,
2187 .res = buffer_read.len,
2188 .flags = 0,
2189 }, try ring.copy_cqe());
2190 try testing.expectEqual(@as(u32, 0), ring.cq_ready());
2191
2192 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2193}
2194
2195test "write/read" {
2196 if (builtin.os.tag != .linux) return error.SkipZigTest;
2197
2198 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
2199 error.SystemOutdated => return error.SkipZigTest,
2200 error.PermissionDenied => return error.SkipZigTest,
2201 else => return err,
2202 };
2203 defer ring.deinit();
2204
2205 var tmp = std.testing.tmpDir(.{});
2206 defer tmp.cleanup();
2207 const path = "test_io_uring_write_read";
2208 const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
2209 defer file.close();
2210 const fd = file.handle;
2211
2212 const buffer_write = [_]u8{97} ** 20;
2213 var buffer_read = [_]u8{98} ** 20;
2214 const sqe_write = try ring.write(0x11111111, fd, buffer_write[0..], 10);
2215 try testing.expectEqual(linux.IORING_OP.WRITE, sqe_write.opcode);
2216 try testing.expectEqual(@as(u64, 10), sqe_write.off);
2217 sqe_write.flags |= linux.IOSQE_IO_LINK;
2218 const sqe_read = try ring.read(0x22222222, fd, .{ .buffer = buffer_read[0..] }, 10);
2219 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
2220 try testing.expectEqual(@as(u64, 10), sqe_read.off);
2221 try testing.expectEqual(@as(u32, 2), try ring.submit());
2222
2223 const cqe_write = try ring.copy_cqe();
2224 const cqe_read = try ring.copy_cqe();
2225 // Prior to Linux Kernel 5.6 this is the only way to test for read/write support:
2226 // https://lwn.net/Articles/809820/
2227 if (cqe_write.err() == .INVAL) return error.SkipZigTest;
2228 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
2229 try testing.expectEqual(linux.io_uring_cqe{
2230 .user_data = 0x11111111,
2231 .res = buffer_write.len,
2232 .flags = 0,
2233 }, cqe_write);
2234 try testing.expectEqual(linux.io_uring_cqe{
2235 .user_data = 0x22222222,
2236 .res = buffer_read.len,
2237 .flags = 0,
2238 }, cqe_read);
2239 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2240}
2241
2242test "splice/read" {
2243 if (builtin.os.tag != .linux) return error.SkipZigTest;
2244
2245 var ring = IO_Uring.init(4, 0) catch |err| switch (err) {
2246 error.SystemOutdated => return error.SkipZigTest,
2247 error.PermissionDenied => return error.SkipZigTest,
2248 else => return err,
2249 };
2250 defer ring.deinit();
2251
2252 var tmp = std.testing.tmpDir(.{});
2253 const path_src = "test_io_uring_splice_src";
2254 const file_src = try tmp.dir.createFile(path_src, .{ .read = true, .truncate = true });
2255 defer file_src.close();
2256 const fd_src = file_src.handle;
2257
2258 const path_dst = "test_io_uring_splice_dst";
2259 const file_dst = try tmp.dir.createFile(path_dst, .{ .read = true, .truncate = true });
2260 defer file_dst.close();
2261 const fd_dst = file_dst.handle;
2262
2263 const buffer_write = [_]u8{97} ** 20;
2264 var buffer_read = [_]u8{98} ** 20;
2265 _ = try file_src.write(&buffer_write);
2266
2267 const fds = try os.pipe();
2268 const pipe_offset: u64 = std.math.maxInt(u64);
2269
2270 const sqe_splice_to_pipe = try ring.splice(0x11111111, fd_src, 0, fds[1], pipe_offset, buffer_write.len);
2271 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_to_pipe.opcode);
2272 try testing.expectEqual(@as(u64, 0), sqe_splice_to_pipe.addr);
2273 try testing.expectEqual(pipe_offset, sqe_splice_to_pipe.off);
2274 sqe_splice_to_pipe.flags |= linux.IOSQE_IO_LINK;
2275
2276 const sqe_splice_from_pipe = try ring.splice(0x22222222, fds[0], pipe_offset, fd_dst, 10, buffer_write.len);
2277 try testing.expectEqual(linux.IORING_OP.SPLICE, sqe_splice_from_pipe.opcode);
2278 try testing.expectEqual(pipe_offset, sqe_splice_from_pipe.addr);
2279 try testing.expectEqual(@as(u64, 10), sqe_splice_from_pipe.off);
2280 sqe_splice_from_pipe.flags |= linux.IOSQE_IO_LINK;
2281
2282 const sqe_read = try ring.read(0x33333333, fd_dst, .{ .buffer = buffer_read[0..] }, 10);
2283 try testing.expectEqual(linux.IORING_OP.READ, sqe_read.opcode);
2284 try testing.expectEqual(@as(u64, 10), sqe_read.off);
2285 try testing.expectEqual(@as(u32, 3), try ring.submit());
2286
2287 const cqe_splice_to_pipe = try ring.copy_cqe();
2288 const cqe_splice_from_pipe = try ring.copy_cqe();
2289 const cqe_read = try ring.copy_cqe();
2290 // Prior to Linux Kernel 5.6 this is the only way to test for splice/read support:
2291 // https://lwn.net/Articles/809820/
2292 if (cqe_splice_to_pipe.err() == .INVAL) return error.SkipZigTest;
2293 if (cqe_splice_from_pipe.err() == .INVAL) return error.SkipZigTest;
2294 if (cqe_read.err() == .INVAL) return error.SkipZigTest;
2295 try testing.expectEqual(linux.io_uring_cqe{
2296 .user_data = 0x11111111,
2297 .res = buffer_write.len,
2298 .flags = 0,
2299 }, cqe_splice_to_pipe);
2300 try testing.expectEqual(linux.io_uring_cqe{
2301 .user_data = 0x22222222,
2302 .res = buffer_write.len,
2303 .flags = 0,
2304 }, cqe_splice_from_pipe);
2305 try testing.expectEqual(linux.io_uring_cqe{
2306 .user_data = 0x33333333,
2307 .res = buffer_read.len,
2308 .flags = 0,
2309 }, cqe_read);
2310 try testing.expectEqualSlices(u8, buffer_write[0..], buffer_read[0..]);
2311}
2312
2313test "write_fixed/read_fixed" {
2314 if (builtin.os.tag != .linux) return error.SkipZigTest;
2315
2316 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
2317 error.SystemOutdated => return error.SkipZigTest,
2318 error.PermissionDenied => return error.SkipZigTest,
2319 else => return err,
2320 };
2321 defer ring.deinit();
2322
2323 var tmp = std.testing.tmpDir(.{});
2324 defer tmp.cleanup();
2325
2326 const path = "test_io_uring_write_read_fixed";
2327 const file = try tmp.dir.createFile(path, .{ .read = true, .truncate = true });
2328 defer file.close();
2329 const fd = file.handle;
2330
2331 var raw_buffers: [2][11]u8 = undefined;
2332 // First buffer will be written to the file.
2333 @memset(&raw_buffers[0], 'z');
2334 raw_buffers[0][0.."foobar".len].* = "foobar".*;
2335
2336 var buffers = [2]os.iovec{
2337 .{ .iov_base = &raw_buffers[0], .iov_len = raw_buffers[0].len },
2338 .{ .iov_base = &raw_buffers[1], .iov_len = raw_buffers[1].len },
2339 };
2340 ring.register_buffers(&buffers) catch |err| switch (err) {
2341 error.SystemResources => {
2342 // See https://github.com/ziglang/zig/issues/15362
2343 return error.SkipZigTest;
2344 },
2345 else => |e| return e,
2346 };
2347
2348 const sqe_write = try ring.write_fixed(0x45454545, fd, &buffers[0], 3, 0);
2349 try testing.expectEqual(linux.IORING_OP.WRITE_FIXED, sqe_write.opcode);
2350 try testing.expectEqual(@as(u64, 3), sqe_write.off);
2351 sqe_write.flags |= linux.IOSQE_IO_LINK;
2352
2353 const sqe_read = try ring.read_fixed(0x12121212, fd, &buffers[1], 0, 1);
2354 try testing.expectEqual(linux.IORING_OP.READ_FIXED, sqe_read.opcode);
2355 try testing.expectEqual(@as(u64, 0), sqe_read.off);
2356
2357 try testing.expectEqual(@as(u32, 2), try ring.submit());
2358
2359 const cqe_write = try ring.copy_cqe();
2360 const cqe_read = try ring.copy_cqe();
2361
2362 try testing.expectEqual(linux.io_uring_cqe{
2363 .user_data = 0x45454545,
2364 .res = @as(i32, @intCast(buffers[0].iov_len)),
2365 .flags = 0,
2366 }, cqe_write);
2367 try testing.expectEqual(linux.io_uring_cqe{
2368 .user_data = 0x12121212,
2369 .res = @as(i32, @intCast(buffers[1].iov_len)),
2370 .flags = 0,
2371 }, cqe_read);
2372
2373 try testing.expectEqualSlices(u8, "\x00\x00\x00", buffers[1].iov_base[0..3]);
2374 try testing.expectEqualSlices(u8, "foobar", buffers[1].iov_base[3..9]);
2375 try testing.expectEqualSlices(u8, "zz", buffers[1].iov_base[9..11]);
2376}
2377
2378test "openat" {
2379 if (builtin.os.tag != .linux) return error.SkipZigTest;
2380
2381 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2382 error.SystemOutdated => return error.SkipZigTest,
2383 error.PermissionDenied => return error.SkipZigTest,
2384 else => return err,
2385 };
2386 defer ring.deinit();
2387
2388 var tmp = std.testing.tmpDir(.{});
2389 defer tmp.cleanup();
2390
2391 const path = "test_io_uring_openat";
2392
2393 // Workaround for LLVM bug: https://github.com/ziglang/zig/issues/12014
2394 const path_addr = if (builtin.zig_backend == .stage2_llvm) p: {
2395 var workaround = path;
2396 _ = &workaround;
2397 break :p @intFromPtr(workaround);
2398 } else @intFromPtr(path);
2399
2400 const flags: linux.O = .{ .CLOEXEC = true, .ACCMODE = .RDWR, .CREAT = true };
2401 const mode: os.mode_t = 0o666;
2402 const sqe_openat = try ring.openat(0x33333333, tmp.dir.fd, path, flags, mode);
2403 try testing.expectEqual(linux.io_uring_sqe{
2404 .opcode = .OPENAT,
2405 .flags = 0,
2406 .ioprio = 0,
2407 .fd = tmp.dir.fd,
2408 .off = 0,
2409 .addr = path_addr,
2410 .len = mode,
2411 .rw_flags = @bitCast(flags),
2412 .user_data = 0x33333333,
2413 .buf_index = 0,
2414 .personality = 0,
2415 .splice_fd_in = 0,
2416 .addr3 = 0,
2417 .resv = 0,
2418 }, sqe_openat.*);
2419 try testing.expectEqual(@as(u32, 1), try ring.submit());
2420
2421 const cqe_openat = try ring.copy_cqe();
2422 try testing.expectEqual(@as(u64, 0x33333333), cqe_openat.user_data);
2423 if (cqe_openat.err() == .INVAL) return error.SkipZigTest;
2424 if (cqe_openat.err() == .BADF) return error.SkipZigTest;
2425 if (cqe_openat.res <= 0) std.debug.print("\ncqe_openat.res={}\n", .{cqe_openat.res});
2426 try testing.expect(cqe_openat.res > 0);
2427 try testing.expectEqual(@as(u32, 0), cqe_openat.flags);
2428
2429 os.close(cqe_openat.res);
2430}
2431
2432test "close" {
2433 if (builtin.os.tag != .linux) return error.SkipZigTest;
2434
2435 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2436 error.SystemOutdated => return error.SkipZigTest,
2437 error.PermissionDenied => return error.SkipZigTest,
2438 else => return err,
2439 };
2440 defer ring.deinit();
2441
2442 var tmp = std.testing.tmpDir(.{});
2443 defer tmp.cleanup();
2444
2445 const path = "test_io_uring_close";
2446 const file = try tmp.dir.createFile(path, .{});
2447 errdefer file.close();
2448
2449 const sqe_close = try ring.close(0x44444444, file.handle);
2450 try testing.expectEqual(linux.IORING_OP.CLOSE, sqe_close.opcode);
2451 try testing.expectEqual(file.handle, sqe_close.fd);
2452 try testing.expectEqual(@as(u32, 1), try ring.submit());
2453
2454 const cqe_close = try ring.copy_cqe();
2455 if (cqe_close.err() == .INVAL) return error.SkipZigTest;
2456 try testing.expectEqual(linux.io_uring_cqe{
2457 .user_data = 0x44444444,
2458 .res = 0,
2459 .flags = 0,
2460 }, cqe_close);
2461}
2462
2463test "accept/connect/send/recv" {
2464 if (builtin.os.tag != .linux) return error.SkipZigTest;
2465
2466 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
2467 error.SystemOutdated => return error.SkipZigTest,
2468 error.PermissionDenied => return error.SkipZigTest,
2469 else => return err,
2470 };
2471 defer ring.deinit();
2472
2473 const socket_test_harness = try createSocketTestHarness(&ring);
2474 defer socket_test_harness.close();
2475
2476 const buffer_send = [_]u8{ 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 };
2477 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2478
2479 const send = try ring.send(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0);
2480 send.flags |= linux.IOSQE_IO_LINK;
2481 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2482 try testing.expectEqual(@as(u32, 2), try ring.submit());
2483
2484 const cqe_send = try ring.copy_cqe();
2485 if (cqe_send.err() == .INVAL) return error.SkipZigTest;
2486 try testing.expectEqual(linux.io_uring_cqe{
2487 .user_data = 0xeeeeeeee,
2488 .res = buffer_send.len,
2489 .flags = 0,
2490 }, cqe_send);
2491
2492 const cqe_recv = try ring.copy_cqe();
2493 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2494 try testing.expectEqual(linux.io_uring_cqe{
2495 .user_data = 0xffffffff,
2496 .res = buffer_recv.len,
2497 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is only set on some systems
2498 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2499 }, cqe_recv);
2500
2501 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2502}
2503
2504test "sendmsg/recvmsg" {
2505 if (builtin.os.tag != .linux) return error.SkipZigTest;
2506
2507 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
2508 error.SystemOutdated => return error.SkipZigTest,
2509 error.PermissionDenied => return error.SkipZigTest,
2510 else => return err,
2511 };
2512 defer ring.deinit();
2513
2514 var address_server = try net.Address.parseIp4("127.0.0.1", 0);
2515
2516 const server = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
2517 defer os.close(server);
2518 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1)));
2519 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
2520 try os.bind(server, &address_server.any, address_server.getOsSockLen());
2521
2522 // set address_server to the OS-chosen IP/port.
2523 var slen: os.socklen_t = address_server.getOsSockLen();
2524 try os.getsockname(server, &address_server.any, &slen);
2525
2526 const client = try os.socket(address_server.any.family, os.SOCK.DGRAM, 0);
2527 defer os.close(client);
2528
2529 const buffer_send = [_]u8{42} ** 128;
2530 const iovecs_send = [_]os.iovec_const{
2531 os.iovec_const{ .iov_base = &buffer_send, .iov_len = buffer_send.len },
2532 };
2533 const msg_send = os.msghdr_const{
2534 .name = &address_server.any,
2535 .namelen = address_server.getOsSockLen(),
2536 .iov = &iovecs_send,
2537 .iovlen = 1,
2538 .control = null,
2539 .controllen = 0,
2540 .flags = 0,
2541 };
2542 const sqe_sendmsg = try ring.sendmsg(0x11111111, client, &msg_send, 0);
2543 sqe_sendmsg.flags |= linux.IOSQE_IO_LINK;
2544 try testing.expectEqual(linux.IORING_OP.SENDMSG, sqe_sendmsg.opcode);
2545 try testing.expectEqual(client, sqe_sendmsg.fd);
2546
2547 var buffer_recv = [_]u8{0} ** 128;
2548 var iovecs_recv = [_]os.iovec{
2549 os.iovec{ .iov_base = &buffer_recv, .iov_len = buffer_recv.len },
2550 };
2551 const addr = [_]u8{0} ** 4;
2552 var address_recv = net.Address.initIp4(addr, 0);
2553 var msg_recv: os.msghdr = os.msghdr{
2554 .name = &address_recv.any,
2555 .namelen = address_recv.getOsSockLen(),
2556 .iov = &iovecs_recv,
2557 .iovlen = 1,
2558 .control = null,
2559 .controllen = 0,
2560 .flags = 0,
2561 };
2562 const sqe_recvmsg = try ring.recvmsg(0x22222222, server, &msg_recv, 0);
2563 try testing.expectEqual(linux.IORING_OP.RECVMSG, sqe_recvmsg.opcode);
2564 try testing.expectEqual(server, sqe_recvmsg.fd);
2565
2566 try testing.expectEqual(@as(u32, 2), ring.sq_ready());
2567 try testing.expectEqual(@as(u32, 2), try ring.submit_and_wait(2));
2568 try testing.expectEqual(@as(u32, 0), ring.sq_ready());
2569 try testing.expectEqual(@as(u32, 2), ring.cq_ready());
2570
2571 const cqe_sendmsg = try ring.copy_cqe();
2572 if (cqe_sendmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2573 try testing.expectEqual(linux.io_uring_cqe{
2574 .user_data = 0x11111111,
2575 .res = buffer_send.len,
2576 .flags = 0,
2577 }, cqe_sendmsg);
2578
2579 const cqe_recvmsg = try ring.copy_cqe();
2580 if (cqe_recvmsg.res == -@as(i32, @intFromEnum(linux.E.INVAL))) return error.SkipZigTest;
2581 try testing.expectEqual(linux.io_uring_cqe{
2582 .user_data = 0x22222222,
2583 .res = buffer_recv.len,
2584 // ignore IORING_CQE_F_SOCK_NONEMPTY since it is set non-deterministically
2585 .flags = cqe_recvmsg.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
2586 }, cqe_recvmsg);
2587
2588 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
2589}
2590
2591test "timeout (after a relative time)" {
2592 if (builtin.os.tag != .linux) return error.SkipZigTest;
2593
2594 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2595 error.SystemOutdated => return error.SkipZigTest,
2596 error.PermissionDenied => return error.SkipZigTest,
2597 else => return err,
2598 };
2599 defer ring.deinit();
2600
2601 const ms = 10;
2602 const margin = 5;
2603 const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = ms * 1000000 };
2604
2605 const started = std.time.milliTimestamp();
2606 const sqe = try ring.timeout(0x55555555, &ts, 0, 0);
2607 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode);
2608 try testing.expectEqual(@as(u32, 1), try ring.submit());
2609 const cqe = try ring.copy_cqe();
2610 const stopped = std.time.milliTimestamp();
2611
2612 try testing.expectEqual(linux.io_uring_cqe{
2613 .user_data = 0x55555555,
2614 .res = -@as(i32, @intFromEnum(linux.E.TIME)),
2615 .flags = 0,
2616 }, cqe);
2617
2618 // Tests should not depend on timings: skip test if outside margin.
2619 if (!std.math.approxEqAbs(f64, ms, @as(f64, @floatFromInt(stopped - started)), margin)) return error.SkipZigTest;
2620}
2621
2622test "timeout (after a number of completions)" {
2623 if (builtin.os.tag != .linux) return error.SkipZigTest;
2624
2625 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
2626 error.SystemOutdated => return error.SkipZigTest,
2627 error.PermissionDenied => return error.SkipZigTest,
2628 else => return err,
2629 };
2630 defer ring.deinit();
2631
2632 const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
2633 const count_completions: u64 = 1;
2634 const sqe_timeout = try ring.timeout(0x66666666, &ts, count_completions, 0);
2635 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2636 try testing.expectEqual(count_completions, sqe_timeout.off);
2637 _ = try ring.nop(0x77777777);
2638 try testing.expectEqual(@as(u32, 2), try ring.submit());
2639
2640 const cqe_nop = try ring.copy_cqe();
2641 try testing.expectEqual(linux.io_uring_cqe{
2642 .user_data = 0x77777777,
2643 .res = 0,
2644 .flags = 0,
2645 }, cqe_nop);
2646
2647 const cqe_timeout = try ring.copy_cqe();
2648 try testing.expectEqual(linux.io_uring_cqe{
2649 .user_data = 0x66666666,
2650 .res = 0,
2651 .flags = 0,
2652 }, cqe_timeout);
2653}
2654
2655test "timeout_remove" {
2656 if (builtin.os.tag != .linux) return error.SkipZigTest;
2657
2658 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
2659 error.SystemOutdated => return error.SkipZigTest,
2660 error.PermissionDenied => return error.SkipZigTest,
2661 else => return err,
2662 };
2663 defer ring.deinit();
2664
2665 const ts = os.linux.kernel_timespec{ .tv_sec = 3, .tv_nsec = 0 };
2666 const sqe_timeout = try ring.timeout(0x88888888, &ts, 0, 0);
2667 try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe_timeout.opcode);
2668 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout.user_data);
2669
2670 const sqe_timeout_remove = try ring.timeout_remove(0x99999999, 0x88888888, 0);
2671 try testing.expectEqual(linux.IORING_OP.TIMEOUT_REMOVE, sqe_timeout_remove.opcode);
2672 try testing.expectEqual(@as(u64, 0x88888888), sqe_timeout_remove.addr);
2673 try testing.expectEqual(@as(u64, 0x99999999), sqe_timeout_remove.user_data);
2674
2675 try testing.expectEqual(@as(u32, 2), try ring.submit());
2676
2677 // The order in which the CQE arrive is not clearly documented and it changed with kernel 5.18:
2678 // * kernel 5.10 gives user data 0x88888888 first, 0x99999999 second
2679 // * kernel 5.18 gives user data 0x99999999 first, 0x88888888 second
2680
2681 var cqes: [2]os.linux.io_uring_cqe = undefined;
2682 cqes[0] = try ring.copy_cqe();
2683 cqes[1] = try ring.copy_cqe();
2684
2685 for (cqes) |cqe| {
2686 // IORING_OP_TIMEOUT_REMOVE is not supported by this kernel version:
2687 // Timeout remove operations set the fd to -1, which results in EBADF before EINVAL.
2688 // We use IORING_FEAT_RW_CUR_POS as a safety check here to make sure we are at least pre-5.6.
2689 // We don't want to skip this test for newer kernels.
2690 if (cqe.user_data == 0x99999999 and
2691 cqe.err() == .BADF and
2692 (ring.features & linux.IORING_FEAT_RW_CUR_POS) == 0)
2693 {
2694 return error.SkipZigTest;
2695 }
2696
2697 try testing.expect(cqe.user_data == 0x88888888 or cqe.user_data == 0x99999999);
2698
2699 if (cqe.user_data == 0x88888888) {
2700 try testing.expectEqual(linux.io_uring_cqe{
2701 .user_data = 0x88888888,
2702 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2703 .flags = 0,
2704 }, cqe);
2705 } else if (cqe.user_data == 0x99999999) {
2706 try testing.expectEqual(linux.io_uring_cqe{
2707 .user_data = 0x99999999,
2708 .res = 0,
2709 .flags = 0,
2710 }, cqe);
2711 }
2712 }
2713}
2714
2715test "accept/connect/recv/link_timeout" {
2716 if (builtin.os.tag != .linux) return error.SkipZigTest;
2717
2718 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
2719 error.SystemOutdated => return error.SkipZigTest,
2720 error.PermissionDenied => return error.SkipZigTest,
2721 else => return err,
2722 };
2723 defer ring.deinit();
2724
2725 const socket_test_harness = try createSocketTestHarness(&ring);
2726 defer socket_test_harness.close();
2727
2728 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2729
2730 const sqe_recv = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2731 sqe_recv.flags |= linux.IOSQE_IO_LINK;
2732
2733 const ts = os.linux.kernel_timespec{ .tv_sec = 0, .tv_nsec = 1000000 };
2734 _ = try ring.link_timeout(0x22222222, &ts, 0);
2735
2736 const nr_wait = try ring.submit();
2737 try testing.expectEqual(@as(u32, 2), nr_wait);
2738
2739 var i: usize = 0;
2740 while (i < nr_wait) : (i += 1) {
2741 const cqe = try ring.copy_cqe();
2742 switch (cqe.user_data) {
2743 0xffffffff => {
2744 if (cqe.res != -@as(i32, @intFromEnum(linux.E.INTR)) and
2745 cqe.res != -@as(i32, @intFromEnum(linux.E.CANCELED)))
2746 {
2747 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2748 try testing.expect(false);
2749 }
2750 },
2751 0x22222222 => {
2752 if (cqe.res != -@as(i32, @intFromEnum(linux.E.ALREADY)) and
2753 cqe.res != -@as(i32, @intFromEnum(linux.E.TIME)))
2754 {
2755 std.debug.print("Req 0x{x} got {d}\n", .{ cqe.user_data, cqe.res });
2756 try testing.expect(false);
2757 }
2758 },
2759 else => @panic("should not happen"),
2760 }
2761 }
2762}
2763
2764test "fallocate" {
2765 if (builtin.os.tag != .linux) return error.SkipZigTest;
2766
2767 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2768 error.SystemOutdated => return error.SkipZigTest,
2769 error.PermissionDenied => return error.SkipZigTest,
2770 else => return err,
2771 };
2772 defer ring.deinit();
2773
2774 var tmp = std.testing.tmpDir(.{});
2775 defer tmp.cleanup();
2776
2777 const path = "test_io_uring_fallocate";
2778 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2779 defer file.close();
2780
2781 try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
2782
2783 const len: u64 = 65536;
2784 const sqe = try ring.fallocate(0xaaaaaaaa, file.handle, 0, 0, len);
2785 try testing.expectEqual(linux.IORING_OP.FALLOCATE, sqe.opcode);
2786 try testing.expectEqual(file.handle, sqe.fd);
2787 try testing.expectEqual(@as(u32, 1), try ring.submit());
2788
2789 const cqe = try ring.copy_cqe();
2790 switch (cqe.err()) {
2791 .SUCCESS => {},
2792 // This kernel's io_uring does not yet implement fallocate():
2793 .INVAL => return error.SkipZigTest,
2794 // This kernel does not implement fallocate():
2795 .NOSYS => return error.SkipZigTest,
2796 // The filesystem containing the file referred to by fd does not support this operation;
2797 // or the mode is not supported by the filesystem containing the file referred to by fd:
2798 .OPNOTSUPP => return error.SkipZigTest,
2799 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2800 }
2801 try testing.expectEqual(linux.io_uring_cqe{
2802 .user_data = 0xaaaaaaaa,
2803 .res = 0,
2804 .flags = 0,
2805 }, cqe);
2806
2807 try testing.expectEqual(len, (try file.stat()).size);
2808}
2809
2810test "statx" {
2811 if (builtin.os.tag != .linux) return error.SkipZigTest;
2812
2813 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2814 error.SystemOutdated => return error.SkipZigTest,
2815 error.PermissionDenied => return error.SkipZigTest,
2816 else => return err,
2817 };
2818 defer ring.deinit();
2819
2820 var tmp = std.testing.tmpDir(.{});
2821 defer tmp.cleanup();
2822 const path = "test_io_uring_statx";
2823 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
2824 defer file.close();
2825
2826 try testing.expectEqual(@as(u64, 0), (try file.stat()).size);
2827
2828 try file.writeAll("foobar");
2829
2830 var buf: linux.Statx = undefined;
2831 const sqe = try ring.statx(
2832 0xaaaaaaaa,
2833 tmp.dir.fd,
2834 path,
2835 0,
2836 linux.STATX_SIZE,
2837 &buf,
2838 );
2839 try testing.expectEqual(linux.IORING_OP.STATX, sqe.opcode);
2840 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
2841 try testing.expectEqual(@as(u32, 1), try ring.submit());
2842
2843 const cqe = try ring.copy_cqe();
2844 switch (cqe.err()) {
2845 .SUCCESS => {},
2846 // This kernel's io_uring does not yet implement statx():
2847 .INVAL => return error.SkipZigTest,
2848 // This kernel does not implement statx():
2849 .NOSYS => return error.SkipZigTest,
2850 // The filesystem containing the file referred to by fd does not support this operation;
2851 // or the mode is not supported by the filesystem containing the file referred to by fd:
2852 .OPNOTSUPP => return error.SkipZigTest,
2853 // not supported on older kernels (5.4)
2854 .BADF => return error.SkipZigTest,
2855 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2856 }
2857 try testing.expectEqual(linux.io_uring_cqe{
2858 .user_data = 0xaaaaaaaa,
2859 .res = 0,
2860 .flags = 0,
2861 }, cqe);
2862
2863 try testing.expect(buf.mask & os.linux.STATX_SIZE == os.linux.STATX_SIZE);
2864 try testing.expectEqual(@as(u64, 6), buf.size);
2865}
2866
2867test "accept/connect/recv/cancel" {
2868 if (builtin.os.tag != .linux) return error.SkipZigTest;
2869
2870 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
2871 error.SystemOutdated => return error.SkipZigTest,
2872 error.PermissionDenied => return error.SkipZigTest,
2873 else => return err,
2874 };
2875 defer ring.deinit();
2876
2877 const socket_test_harness = try createSocketTestHarness(&ring);
2878 defer socket_test_harness.close();
2879
2880 var buffer_recv = [_]u8{ 0, 1, 0, 1, 0 };
2881
2882 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
2883 try testing.expectEqual(@as(u32, 1), try ring.submit());
2884
2885 const sqe_cancel = try ring.cancel(0x99999999, 0xffffffff, 0);
2886 try testing.expectEqual(linux.IORING_OP.ASYNC_CANCEL, sqe_cancel.opcode);
2887 try testing.expectEqual(@as(u64, 0xffffffff), sqe_cancel.addr);
2888 try testing.expectEqual(@as(u64, 0x99999999), sqe_cancel.user_data);
2889 try testing.expectEqual(@as(u32, 1), try ring.submit());
2890
2891 var cqe_recv = try ring.copy_cqe();
2892 if (cqe_recv.err() == .INVAL) return error.SkipZigTest;
2893 var cqe_cancel = try ring.copy_cqe();
2894 if (cqe_cancel.err() == .INVAL) return error.SkipZigTest;
2895
2896 // The recv/cancel CQEs may arrive in any order, the recv CQE will sometimes come first:
2897 if (cqe_recv.user_data == 0x99999999 and cqe_cancel.user_data == 0xffffffff) {
2898 const a = cqe_recv;
2899 const b = cqe_cancel;
2900 cqe_recv = b;
2901 cqe_cancel = a;
2902 }
2903
2904 try testing.expectEqual(linux.io_uring_cqe{
2905 .user_data = 0xffffffff,
2906 .res = -@as(i32, @intFromEnum(linux.E.CANCELED)),
2907 .flags = 0,
2908 }, cqe_recv);
2909
2910 try testing.expectEqual(linux.io_uring_cqe{
2911 .user_data = 0x99999999,
2912 .res = 0,
2913 .flags = 0,
2914 }, cqe_cancel);
2915}
2916
2917test "register_files_update" {
2918 if (builtin.os.tag != .linux) return error.SkipZigTest;
2919
2920 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
2921 error.SystemOutdated => return error.SkipZigTest,
2922 error.PermissionDenied => return error.SkipZigTest,
2923 else => return err,
2924 };
2925 defer ring.deinit();
2926
2927 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2928 defer os.close(fd);
2929
2930 var registered_fds = [_]os.fd_t{0} ** 2;
2931 const fd_index = 0;
2932 const fd_index2 = 1;
2933 registered_fds[fd_index] = fd;
2934 registered_fds[fd_index2] = -1;
2935
2936 ring.register_files(registered_fds[0..]) catch |err| switch (err) {
2937 // Happens when the kernel doesn't support sparse entry (-1) in the file descriptors array.
2938 error.FileDescriptorInvalid => return error.SkipZigTest,
2939 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
2940 };
2941
2942 // Test IORING_REGISTER_FILES_UPDATE
2943 // Only available since Linux 5.5
2944
2945 const fd2 = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
2946 defer os.close(fd2);
2947
2948 registered_fds[fd_index] = fd2;
2949 registered_fds[fd_index2] = -1;
2950 try ring.register_files_update(0, registered_fds[0..]);
2951
2952 var buffer = [_]u8{42} ** 128;
2953 {
2954 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2955 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2956 sqe.flags |= linux.IOSQE_FIXED_FILE;
2957
2958 try testing.expectEqual(@as(u32, 1), try ring.submit());
2959 try testing.expectEqual(linux.io_uring_cqe{
2960 .user_data = 0xcccccccc,
2961 .res = buffer.len,
2962 .flags = 0,
2963 }, try ring.copy_cqe());
2964 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2965 }
2966
2967 // Test with a non-zero offset
2968
2969 registered_fds[fd_index] = -1;
2970 registered_fds[fd_index2] = -1;
2971 try ring.register_files_update(1, registered_fds[1..]);
2972
2973 {
2974 // Next read should still work since fd_index in the registered file descriptors hasn't been updated yet.
2975 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2976 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2977 sqe.flags |= linux.IOSQE_FIXED_FILE;
2978
2979 try testing.expectEqual(@as(u32, 1), try ring.submit());
2980 try testing.expectEqual(linux.io_uring_cqe{
2981 .user_data = 0xcccccccc,
2982 .res = buffer.len,
2983 .flags = 0,
2984 }, try ring.copy_cqe());
2985 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer.len), buffer[0..]);
2986 }
2987
2988 try ring.register_files_update(0, registered_fds[0..]);
2989
2990 {
2991 // Now this should fail since both fds are sparse (-1)
2992 const sqe = try ring.read(0xcccccccc, fd_index, .{ .buffer = &buffer }, 0);
2993 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
2994 sqe.flags |= linux.IOSQE_FIXED_FILE;
2995
2996 try testing.expectEqual(@as(u32, 1), try ring.submit());
2997 const cqe = try ring.copy_cqe();
2998 try testing.expectEqual(os.linux.E.BADF, cqe.err());
2999 }
3000
3001 try ring.unregister_files();
3002}
3003
3004test "shutdown" {
3005 if (builtin.os.tag != .linux) return error.SkipZigTest;
3006
3007 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
3008 error.SystemOutdated => return error.SkipZigTest,
3009 error.PermissionDenied => return error.SkipZigTest,
3010 else => return err,
3011 };
3012 defer ring.deinit();
3013
3014 var address = try net.Address.parseIp4("127.0.0.1", 0);
3015
3016 // Socket bound, expect shutdown to work
3017 {
3018 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3019 defer os.close(server);
3020 try os.setsockopt(server, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3021 try os.bind(server, &address.any, address.getOsSockLen());
3022 try os.listen(server, 1);
3023
3024 // set address to the OS-chosen IP/port.
3025 var slen: os.socklen_t = address.getOsSockLen();
3026 try os.getsockname(server, &address.any, &slen);
3027
3028 const shutdown_sqe = try ring.shutdown(0x445445445, server, os.linux.SHUT.RD);
3029 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
3030 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
3031
3032 try testing.expectEqual(@as(u32, 1), try ring.submit());
3033
3034 const cqe = try ring.copy_cqe();
3035 switch (cqe.err()) {
3036 .SUCCESS => {},
3037 // This kernel's io_uring does not yet implement shutdown (kernel version < 5.11)
3038 .INVAL => return error.SkipZigTest,
3039 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3040 }
3041
3042 try testing.expectEqual(linux.io_uring_cqe{
3043 .user_data = 0x445445445,
3044 .res = 0,
3045 .flags = 0,
3046 }, cqe);
3047 }
3048
3049 // Socket not bound, expect to fail with ENOTCONN
3050 {
3051 const server = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3052 defer os.close(server);
3053
3054 const shutdown_sqe = ring.shutdown(0x445445445, server, os.linux.SHUT.RD) catch |err| switch (err) {
3055 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3056 };
3057 try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode);
3058 try testing.expectEqual(@as(i32, server), shutdown_sqe.fd);
3059
3060 try testing.expectEqual(@as(u32, 1), try ring.submit());
3061
3062 const cqe = try ring.copy_cqe();
3063 try testing.expectEqual(@as(u64, 0x445445445), cqe.user_data);
3064 try testing.expectEqual(os.linux.E.NOTCONN, cqe.err());
3065 }
3066}
3067
3068test "renameat" {
3069 if (builtin.os.tag != .linux) return error.SkipZigTest;
3070
3071 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3072 error.SystemOutdated => return error.SkipZigTest,
3073 error.PermissionDenied => return error.SkipZigTest,
3074 else => return err,
3075 };
3076 defer ring.deinit();
3077
3078 const old_path = "test_io_uring_renameat_old";
3079 const new_path = "test_io_uring_renameat_new";
3080
3081 var tmp = std.testing.tmpDir(.{});
3082 defer tmp.cleanup();
3083
3084 // Write old file with data
3085
3086 const old_file = try tmp.dir.createFile(old_path, .{ .truncate = true, .mode = 0o666 });
3087 defer old_file.close();
3088 try old_file.writeAll("hello");
3089
3090 // Submit renameat
3091
3092 const sqe = try ring.renameat(
3093 0x12121212,
3094 tmp.dir.fd,
3095 old_path,
3096 tmp.dir.fd,
3097 new_path,
3098 0,
3099 );
3100 try testing.expectEqual(linux.IORING_OP.RENAMEAT, sqe.opcode);
3101 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3102 try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
3103 try testing.expectEqual(@as(u32, 1), try ring.submit());
3104
3105 const cqe = try ring.copy_cqe();
3106 switch (cqe.err()) {
3107 .SUCCESS => {},
3108 // This kernel's io_uring does not yet implement renameat (kernel version < 5.11)
3109 .BADF, .INVAL => return error.SkipZigTest,
3110 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3111 }
3112 try testing.expectEqual(linux.io_uring_cqe{
3113 .user_data = 0x12121212,
3114 .res = 0,
3115 .flags = 0,
3116 }, cqe);
3117
3118 // Validate that the old file doesn't exist anymore
3119 {
3120 _ = tmp.dir.openFile(old_path, .{}) catch |err| switch (err) {
3121 error.FileNotFound => {},
3122 else => std.debug.panic("unexpected error: {}", .{err}),
3123 };
3124 }
3125
3126 // Validate that the new file exists with the proper content
3127 {
3128 const new_file = try tmp.dir.openFile(new_path, .{});
3129 defer new_file.close();
3130
3131 var new_file_data: [16]u8 = undefined;
3132 const read = try new_file.readAll(&new_file_data);
3133 try testing.expectEqualStrings("hello", new_file_data[0..read]);
3134 }
3135}
3136
3137test "unlinkat" {
3138 if (builtin.os.tag != .linux) return error.SkipZigTest;
3139
3140 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3141 error.SystemOutdated => return error.SkipZigTest,
3142 error.PermissionDenied => return error.SkipZigTest,
3143 else => return err,
3144 };
3145 defer ring.deinit();
3146
3147 const path = "test_io_uring_unlinkat";
3148
3149 var tmp = std.testing.tmpDir(.{});
3150 defer tmp.cleanup();
3151
3152 // Write old file with data
3153
3154 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
3155 defer file.close();
3156
3157 // Submit unlinkat
3158
3159 const sqe = try ring.unlinkat(
3160 0x12121212,
3161 tmp.dir.fd,
3162 path,
3163 0,
3164 );
3165 try testing.expectEqual(linux.IORING_OP.UNLINKAT, sqe.opcode);
3166 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3167 try testing.expectEqual(@as(u32, 1), try ring.submit());
3168
3169 const cqe = try ring.copy_cqe();
3170 switch (cqe.err()) {
3171 .SUCCESS => {},
3172 // This kernel's io_uring does not yet implement unlinkat (kernel version < 5.11)
3173 .BADF, .INVAL => return error.SkipZigTest,
3174 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3175 }
3176 try testing.expectEqual(linux.io_uring_cqe{
3177 .user_data = 0x12121212,
3178 .res = 0,
3179 .flags = 0,
3180 }, cqe);
3181
3182 // Validate that the file doesn't exist anymore
3183 _ = tmp.dir.openFile(path, .{}) catch |err| switch (err) {
3184 error.FileNotFound => {},
3185 else => std.debug.panic("unexpected error: {}", .{err}),
3186 };
3187}
3188
3189test "mkdirat" {
3190 if (builtin.os.tag != .linux) return error.SkipZigTest;
3191
3192 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3193 error.SystemOutdated => return error.SkipZigTest,
3194 error.PermissionDenied => return error.SkipZigTest,
3195 else => return err,
3196 };
3197 defer ring.deinit();
3198
3199 var tmp = std.testing.tmpDir(.{});
3200 defer tmp.cleanup();
3201
3202 const path = "test_io_uring_mkdirat";
3203
3204 // Submit mkdirat
3205
3206 const sqe = try ring.mkdirat(
3207 0x12121212,
3208 tmp.dir.fd,
3209 path,
3210 0o0755,
3211 );
3212 try testing.expectEqual(linux.IORING_OP.MKDIRAT, sqe.opcode);
3213 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3214 try testing.expectEqual(@as(u32, 1), try ring.submit());
3215
3216 const cqe = try ring.copy_cqe();
3217 switch (cqe.err()) {
3218 .SUCCESS => {},
3219 // This kernel's io_uring does not yet implement mkdirat (kernel version < 5.15)
3220 .BADF, .INVAL => return error.SkipZigTest,
3221 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3222 }
3223 try testing.expectEqual(linux.io_uring_cqe{
3224 .user_data = 0x12121212,
3225 .res = 0,
3226 .flags = 0,
3227 }, cqe);
3228
3229 // Validate that the directory exist
3230 _ = try tmp.dir.openDir(path, .{});
3231}
3232
3233test "symlinkat" {
3234 if (builtin.os.tag != .linux) return error.SkipZigTest;
3235
3236 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3237 error.SystemOutdated => return error.SkipZigTest,
3238 error.PermissionDenied => return error.SkipZigTest,
3239 else => return err,
3240 };
3241 defer ring.deinit();
3242
3243 var tmp = std.testing.tmpDir(.{});
3244 defer tmp.cleanup();
3245
3246 const path = "test_io_uring_symlinkat";
3247 const link_path = "test_io_uring_symlinkat_link";
3248
3249 const file = try tmp.dir.createFile(path, .{ .truncate = true, .mode = 0o666 });
3250 defer file.close();
3251
3252 // Submit symlinkat
3253
3254 const sqe = try ring.symlinkat(
3255 0x12121212,
3256 path,
3257 tmp.dir.fd,
3258 link_path,
3259 );
3260 try testing.expectEqual(linux.IORING_OP.SYMLINKAT, sqe.opcode);
3261 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3262 try testing.expectEqual(@as(u32, 1), try ring.submit());
3263
3264 const cqe = try ring.copy_cqe();
3265 switch (cqe.err()) {
3266 .SUCCESS => {},
3267 // This kernel's io_uring does not yet implement symlinkat (kernel version < 5.15)
3268 .BADF, .INVAL => return error.SkipZigTest,
3269 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3270 }
3271 try testing.expectEqual(linux.io_uring_cqe{
3272 .user_data = 0x12121212,
3273 .res = 0,
3274 .flags = 0,
3275 }, cqe);
3276
3277 // Validate that the symlink exist
3278 _ = try tmp.dir.openFile(link_path, .{});
3279}
3280
3281test "linkat" {
3282 if (builtin.os.tag != .linux) return error.SkipZigTest;
3283
3284 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3285 error.SystemOutdated => return error.SkipZigTest,
3286 error.PermissionDenied => return error.SkipZigTest,
3287 else => return err,
3288 };
3289 defer ring.deinit();
3290
3291 var tmp = std.testing.tmpDir(.{});
3292 defer tmp.cleanup();
3293
3294 const first_path = "test_io_uring_linkat_first";
3295 const second_path = "test_io_uring_linkat_second";
3296
3297 // Write file with data
3298
3299 const first_file = try tmp.dir.createFile(first_path, .{ .truncate = true, .mode = 0o666 });
3300 defer first_file.close();
3301 try first_file.writeAll("hello");
3302
3303 // Submit linkat
3304
3305 const sqe = try ring.linkat(
3306 0x12121212,
3307 tmp.dir.fd,
3308 first_path,
3309 tmp.dir.fd,
3310 second_path,
3311 0,
3312 );
3313 try testing.expectEqual(linux.IORING_OP.LINKAT, sqe.opcode);
3314 try testing.expectEqual(@as(i32, tmp.dir.fd), sqe.fd);
3315 try testing.expectEqual(@as(i32, tmp.dir.fd), @as(i32, @bitCast(sqe.len)));
3316 try testing.expectEqual(@as(u32, 1), try ring.submit());
3317
3318 const cqe = try ring.copy_cqe();
3319 switch (cqe.err()) {
3320 .SUCCESS => {},
3321 // This kernel's io_uring does not yet implement linkat (kernel version < 5.15)
3322 .BADF, .INVAL => return error.SkipZigTest,
3323 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3324 }
3325 try testing.expectEqual(linux.io_uring_cqe{
3326 .user_data = 0x12121212,
3327 .res = 0,
3328 .flags = 0,
3329 }, cqe);
3330
3331 // Validate the second file
3332 const second_file = try tmp.dir.openFile(second_path, .{});
3333 defer second_file.close();
3334
3335 var second_file_data: [16]u8 = undefined;
3336 const read = try second_file.readAll(&second_file_data);
3337 try testing.expectEqualStrings("hello", second_file_data[0..read]);
3338}
3339
3340test "provide_buffers: read" {
3341 if (builtin.os.tag != .linux) return error.SkipZigTest;
3342
3343 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3344 error.SystemOutdated => return error.SkipZigTest,
3345 error.PermissionDenied => return error.SkipZigTest,
3346 else => return err,
3347 };
3348 defer ring.deinit();
3349
3350 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
3351 defer os.close(fd);
3352
3353 const group_id = 1337;
3354 const buffer_id = 0;
3355
3356 const buffer_len = 128;
3357
3358 var buffers: [4][buffer_len]u8 = undefined;
3359
3360 // Provide 4 buffers
3361
3362 {
3363 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3364 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3365 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3366 try testing.expectEqual(@as(u32, buffers[0].len), sqe.len);
3367 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3368 try testing.expectEqual(@as(u32, 1), try ring.submit());
3369
3370 const cqe = try ring.copy_cqe();
3371 switch (cqe.err()) {
3372 // Happens when the kernel is < 5.7
3373 .INVAL => return error.SkipZigTest,
3374 .SUCCESS => {},
3375 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3376 }
3377 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3378 }
3379
3380 // Do 4 reads which should consume all buffers
3381
3382 var i: usize = 0;
3383 while (i < buffers.len) : (i += 1) {
3384 const sqe = try ring.read(0xdededede, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3385 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3386 try testing.expectEqual(@as(i32, fd), sqe.fd);
3387 try testing.expectEqual(@as(u64, 0), sqe.addr);
3388 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3389 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3390 try testing.expectEqual(@as(u32, 1), try ring.submit());
3391
3392 const cqe = try ring.copy_cqe();
3393 switch (cqe.err()) {
3394 .SUCCESS => {},
3395 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3396 }
3397
3398 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3399 const used_buffer_id = cqe.flags >> 16;
3400 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3401 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3402
3403 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3404 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3405 }
3406
3407 // This read should fail
3408
3409 {
3410 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3411 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3412 try testing.expectEqual(@as(i32, fd), sqe.fd);
3413 try testing.expectEqual(@as(u64, 0), sqe.addr);
3414 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3415 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3416 try testing.expectEqual(@as(u32, 1), try ring.submit());
3417
3418 const cqe = try ring.copy_cqe();
3419 switch (cqe.err()) {
3420 // Expected
3421 .NOBUFS => {},
3422 .SUCCESS => std.debug.panic("unexpected success", .{}),
3423 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3424 }
3425 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3426 }
3427
3428 // Provide 1 buffer again
3429
3430 // Deliberately put something we don't expect in the buffers
3431 @memset(mem.sliceAsBytes(&buffers), 42);
3432
3433 const reprovided_buffer_id = 2;
3434
3435 {
3436 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3437 try testing.expectEqual(@as(u32, 1), try ring.submit());
3438
3439 const cqe = try ring.copy_cqe();
3440 switch (cqe.err()) {
3441 .SUCCESS => {},
3442 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3443 }
3444 }
3445
3446 // Final read which should work
3447
3448 {
3449 const sqe = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3450 try testing.expectEqual(linux.IORING_OP.READ, sqe.opcode);
3451 try testing.expectEqual(@as(i32, fd), sqe.fd);
3452 try testing.expectEqual(@as(u64, 0), sqe.addr);
3453 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3454 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3455 try testing.expectEqual(@as(u32, 1), try ring.submit());
3456
3457 const cqe = try ring.copy_cqe();
3458 switch (cqe.err()) {
3459 .SUCCESS => {},
3460 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3461 }
3462
3463 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3464 const used_buffer_id = cqe.flags >> 16;
3465 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3466 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3467 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3468 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3469 }
3470}
3471
3472test "remove_buffers" {
3473 if (builtin.os.tag != .linux) return error.SkipZigTest;
3474
3475 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3476 error.SystemOutdated => return error.SkipZigTest,
3477 error.PermissionDenied => return error.SkipZigTest,
3478 else => return err,
3479 };
3480 defer ring.deinit();
3481
3482 const fd = try os.openZ("/dev/zero", .{ .ACCMODE = .RDONLY, .CLOEXEC = true }, 0);
3483 defer os.close(fd);
3484
3485 const group_id = 1337;
3486 const buffer_id = 0;
3487
3488 const buffer_len = 128;
3489
3490 var buffers: [4][buffer_len]u8 = undefined;
3491
3492 // Provide 4 buffers
3493
3494 {
3495 _ = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3496 try testing.expectEqual(@as(u32, 1), try ring.submit());
3497
3498 const cqe = try ring.copy_cqe();
3499 switch (cqe.err()) {
3500 .INVAL => return error.SkipZigTest,
3501 .SUCCESS => {},
3502 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3503 }
3504 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3505 }
3506
3507 // Remove 3 buffers
3508
3509 {
3510 const sqe = try ring.remove_buffers(0xbababababa, 3, group_id);
3511 try testing.expectEqual(linux.IORING_OP.REMOVE_BUFFERS, sqe.opcode);
3512 try testing.expectEqual(@as(i32, 3), sqe.fd);
3513 try testing.expectEqual(@as(u64, 0), sqe.addr);
3514 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3515 try testing.expectEqual(@as(u32, 1), try ring.submit());
3516
3517 const cqe = try ring.copy_cqe();
3518 switch (cqe.err()) {
3519 .SUCCESS => {},
3520 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3521 }
3522 try testing.expectEqual(@as(u64, 0xbababababa), cqe.user_data);
3523 }
3524
3525 // This read should work
3526
3527 {
3528 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3529 try testing.expectEqual(@as(u32, 1), try ring.submit());
3530
3531 const cqe = try ring.copy_cqe();
3532 switch (cqe.err()) {
3533 .SUCCESS => {},
3534 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3535 }
3536
3537 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3538 const used_buffer_id = cqe.flags >> 16;
3539 try testing.expect(used_buffer_id >= 0 and used_buffer_id < 4);
3540 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3541 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3542 try testing.expectEqualSlices(u8, &([_]u8{0} ** buffer_len), buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))]);
3543 }
3544
3545 // Final read should _not_ work
3546
3547 {
3548 _ = try ring.read(0xdfdfdfdf, fd, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3549 try testing.expectEqual(@as(u32, 1), try ring.submit());
3550
3551 const cqe = try ring.copy_cqe();
3552 switch (cqe.err()) {
3553 // Expected
3554 .NOBUFS => {},
3555 .SUCCESS => std.debug.panic("unexpected success", .{}),
3556 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3557 }
3558 }
3559}
3560
3561test "provide_buffers: accept/connect/send/recv" {
3562 if (builtin.os.tag != .linux) return error.SkipZigTest;
3563
3564 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
3565 error.SystemOutdated => return error.SkipZigTest,
3566 error.PermissionDenied => return error.SkipZigTest,
3567 else => return err,
3568 };
3569 defer ring.deinit();
3570
3571 const group_id = 1337;
3572 const buffer_id = 0;
3573
3574 const buffer_len = 128;
3575 var buffers: [4][buffer_len]u8 = undefined;
3576
3577 // Provide 4 buffers
3578
3579 {
3580 const sqe = try ring.provide_buffers(0xcccccccc, @as([*]u8, @ptrCast(&buffers)), buffer_len, buffers.len, group_id, buffer_id);
3581 try testing.expectEqual(linux.IORING_OP.PROVIDE_BUFFERS, sqe.opcode);
3582 try testing.expectEqual(@as(i32, buffers.len), sqe.fd);
3583 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3584 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3585 try testing.expectEqual(@as(u32, 1), try ring.submit());
3586
3587 const cqe = try ring.copy_cqe();
3588 switch (cqe.err()) {
3589 // Happens when the kernel is < 5.7
3590 .INVAL => return error.SkipZigTest,
3591 // Happens on the kernel 5.4
3592 .BADF => return error.SkipZigTest,
3593 .SUCCESS => {},
3594 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3595 }
3596 try testing.expectEqual(@as(u64, 0xcccccccc), cqe.user_data);
3597 }
3598
3599 const socket_test_harness = try createSocketTestHarness(&ring);
3600 defer socket_test_harness.close();
3601
3602 // Do 4 send on the socket
3603
3604 {
3605 var i: usize = 0;
3606 while (i < buffers.len) : (i += 1) {
3607 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'z'} ** buffer_len), 0);
3608 try testing.expectEqual(@as(u32, 1), try ring.submit());
3609 }
3610
3611 var cqes: [4]linux.io_uring_cqe = undefined;
3612 try testing.expectEqual(@as(u32, 4), try ring.copy_cqes(&cqes, 4));
3613 }
3614
3615 // Do 4 recv which should consume all buffers
3616
3617 // Deliberately put something we don't expect in the buffers
3618 @memset(mem.sliceAsBytes(&buffers), 1);
3619
3620 var i: usize = 0;
3621 while (i < buffers.len) : (i += 1) {
3622 const sqe = try ring.recv(0xdededede, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3623 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3624 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3625 try testing.expectEqual(@as(u64, 0), sqe.addr);
3626 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3627 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3628 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3629 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3630 try testing.expectEqual(@as(u32, 1), try ring.submit());
3631
3632 const cqe = try ring.copy_cqe();
3633 switch (cqe.err()) {
3634 .SUCCESS => {},
3635 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3636 }
3637
3638 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3639 const used_buffer_id = cqe.flags >> 16;
3640 try testing.expect(used_buffer_id >= 0 and used_buffer_id <= 3);
3641 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3642
3643 try testing.expectEqual(@as(u64, 0xdededede), cqe.user_data);
3644 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3645 try testing.expectEqualSlices(u8, &([_]u8{'z'} ** buffer_len), buffer);
3646 }
3647
3648 // This recv should fail
3649
3650 {
3651 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3652 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3653 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3654 try testing.expectEqual(@as(u64, 0), sqe.addr);
3655 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3656 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3657 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3658 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3659 try testing.expectEqual(@as(u32, 1), try ring.submit());
3660
3661 const cqe = try ring.copy_cqe();
3662 switch (cqe.err()) {
3663 // Expected
3664 .NOBUFS => {},
3665 .SUCCESS => std.debug.panic("unexpected success", .{}),
3666 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3667 }
3668 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3669 }
3670
3671 // Provide 1 buffer again
3672
3673 const reprovided_buffer_id = 2;
3674
3675 {
3676 _ = try ring.provide_buffers(0xabababab, @as([*]u8, @ptrCast(&buffers[reprovided_buffer_id])), buffer_len, 1, group_id, reprovided_buffer_id);
3677 try testing.expectEqual(@as(u32, 1), try ring.submit());
3678
3679 const cqe = try ring.copy_cqe();
3680 switch (cqe.err()) {
3681 .SUCCESS => {},
3682 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3683 }
3684 }
3685
3686 // Redo 1 send on the server socket
3687
3688 {
3689 _ = try ring.send(0xdeaddead, socket_test_harness.server, &([_]u8{'w'} ** buffer_len), 0);
3690 try testing.expectEqual(@as(u32, 1), try ring.submit());
3691
3692 _ = try ring.copy_cqe();
3693 }
3694
3695 // Final recv which should work
3696
3697 // Deliberately put something we don't expect in the buffers
3698 @memset(mem.sliceAsBytes(&buffers), 1);
3699
3700 {
3701 const sqe = try ring.recv(0xdfdfdfdf, socket_test_harness.client, .{ .buffer_selection = .{ .group_id = group_id, .len = buffer_len } }, 0);
3702 try testing.expectEqual(linux.IORING_OP.RECV, sqe.opcode);
3703 try testing.expectEqual(@as(i32, socket_test_harness.client), sqe.fd);
3704 try testing.expectEqual(@as(u64, 0), sqe.addr);
3705 try testing.expectEqual(@as(u32, buffer_len), sqe.len);
3706 try testing.expectEqual(@as(u16, group_id), sqe.buf_index);
3707 try testing.expectEqual(@as(u32, 0), sqe.rw_flags);
3708 try testing.expectEqual(@as(u32, linux.IOSQE_BUFFER_SELECT), sqe.flags);
3709 try testing.expectEqual(@as(u32, 1), try ring.submit());
3710
3711 const cqe = try ring.copy_cqe();
3712 switch (cqe.err()) {
3713 .SUCCESS => {},
3714 else => |errno| std.debug.panic("unhandled errno: {}", .{errno}),
3715 }
3716
3717 try testing.expect(cqe.flags & linux.IORING_CQE_F_BUFFER == linux.IORING_CQE_F_BUFFER);
3718 const used_buffer_id = cqe.flags >> 16;
3719 try testing.expectEqual(used_buffer_id, reprovided_buffer_id);
3720 try testing.expectEqual(@as(i32, buffer_len), cqe.res);
3721 try testing.expectEqual(@as(u64, 0xdfdfdfdf), cqe.user_data);
3722 const buffer = buffers[used_buffer_id][0..@as(usize, @intCast(cqe.res))];
3723 try testing.expectEqualSlices(u8, &([_]u8{'w'} ** buffer_len), buffer);
3724 }
3725}
3726
3727/// Used for testing server/client interactions.
3728const SocketTestHarness = struct {
3729 listener: os.socket_t,
3730 server: os.socket_t,
3731 client: os.socket_t,
3732
3733 fn close(self: SocketTestHarness) void {
3734 posix.close(self.client);
3735 posix.close(self.listener);
3736 }
3737};
3738
3739fn createSocketTestHarness(ring: *IO_Uring) !SocketTestHarness {
3740 // Create a TCP server socket
3741 var address = try net.Address.parseIp4("127.0.0.1", 0);
3742 const listener_socket = try createListenerSocket(&address);
3743 errdefer posix.close(listener_socket);
3744
3745 // Submit 1 accept
3746 var accept_addr: os.sockaddr = undefined;
3747 var accept_addr_len: os.socklen_t = @sizeOf(@TypeOf(accept_addr));
3748 _ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0);
3749
3750 // Create a TCP client socket
3751 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3752 errdefer posix.close(client);
3753 _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen());
3754
3755 try testing.expectEqual(@as(u32, 2), try ring.submit());
3756
3757 var cqe_accept = try ring.copy_cqe();
3758 if (cqe_accept.err() == .INVAL) return error.SkipZigTest;
3759 var cqe_connect = try ring.copy_cqe();
3760 if (cqe_connect.err() == .INVAL) return error.SkipZigTest;
3761
3762 // The accept/connect CQEs may arrive in any order, the connect CQE will sometimes come first:
3763 if (cqe_accept.user_data == 0xcccccccc and cqe_connect.user_data == 0xaaaaaaaa) {
3764 const a = cqe_accept;
3765 const b = cqe_connect;
3766 cqe_accept = b;
3767 cqe_connect = a;
3768 }
3769
3770 try testing.expectEqual(@as(u64, 0xaaaaaaaa), cqe_accept.user_data);
3771 if (cqe_accept.res <= 0) std.debug.print("\ncqe_accept.res={}\n", .{cqe_accept.res});
3772 try testing.expect(cqe_accept.res > 0);
3773 try testing.expectEqual(@as(u32, 0), cqe_accept.flags);
3774 try testing.expectEqual(linux.io_uring_cqe{
3775 .user_data = 0xcccccccc,
3776 .res = 0,
3777 .flags = 0,
3778 }, cqe_connect);
3779
3780 // All good
3781
3782 return SocketTestHarness{
3783 .listener = listener_socket,
3784 .server = cqe_accept.res,
3785 .client = client,
3786 };
3787}
3788
3789fn createListenerSocket(address: *net.Address) !os.socket_t {
3790 const kernel_backlog = 1;
3791 const listener_socket = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3792 errdefer posix.close(listener_socket);
3793
3794 try os.setsockopt(listener_socket, os.SOL.SOCKET, os.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1)));
3795 try os.bind(listener_socket, &address.any, address.getOsSockLen());
3796 try os.listen(listener_socket, kernel_backlog);
3797
3798 // set address to the OS-chosen IP/port.
3799 var slen: os.socklen_t = address.getOsSockLen();
3800 try os.getsockname(listener_socket, &address.any, &slen);
3801
3802 return listener_socket;
3803}
3804
3805test "accept multishot" {
3806 if (builtin.os.tag != .linux) return error.SkipZigTest;
3807
3808 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
3809 error.SystemOutdated => return error.SkipZigTest,
3810 error.PermissionDenied => return error.SkipZigTest,
3811 else => return err,
3812 };
3813 defer ring.deinit();
3814
3815 var address = try net.Address.parseIp4("127.0.0.1", 0);
3816 const listener_socket = try createListenerSocket(&address);
3817 defer posix.close(listener_socket);
3818
3819 // submit multishot accept operation
3820 var addr: os.sockaddr = undefined;
3821 var addr_len: os.socklen_t = @sizeOf(@TypeOf(addr));
3822 const userdata: u64 = 0xaaaaaaaa;
3823 _ = try ring.accept_multishot(userdata, listener_socket, &addr, &addr_len, 0);
3824 try testing.expectEqual(@as(u32, 1), try ring.submit());
3825
3826 var nr: usize = 4; // number of clients to connect
3827 while (nr > 0) : (nr -= 1) {
3828 // connect client
3829 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3830 errdefer posix.close(client);
3831 try os.connect(client, &address.any, address.getOsSockLen());
3832
3833 // test accept completion
3834 var cqe = try ring.copy_cqe();
3835 if (cqe.err() == .INVAL) return error.SkipZigTest;
3836 try testing.expect(cqe.res > 0);
3837 try testing.expect(cqe.user_data == userdata);
3838 try testing.expect(cqe.flags & linux.IORING_CQE_F_MORE > 0); // more flag is set
3839
3840 posix.close(client);
3841 }
3842}
3843
3844test "accept/connect/send_zc/recv" {
3845 try skipKernelLessThan(.{ .major = 6, .minor = 0, .patch = 0 });
3846
3847 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
3848 error.SystemOutdated => return error.SkipZigTest,
3849 error.PermissionDenied => return error.SkipZigTest,
3850 else => return err,
3851 };
3852 defer ring.deinit();
3853
3854 const socket_test_harness = try createSocketTestHarness(&ring);
3855 defer socket_test_harness.close();
3856
3857 const buffer_send = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3858 var buffer_recv = [_]u8{0} ** 10;
3859
3860 // zero-copy send
3861 const send = try ring.send_zc(0xeeeeeeee, socket_test_harness.client, buffer_send[0..], 0, 0);
3862 send.flags |= linux.IOSQE_IO_LINK;
3863 _ = try ring.recv(0xffffffff, socket_test_harness.server, .{ .buffer = buffer_recv[0..] }, 0);
3864 try testing.expectEqual(@as(u32, 2), try ring.submit());
3865
3866 // First completion of zero-copy send.
3867 // IORING_CQE_F_MORE, means that there
3868 // will be a second completion event / notification for the
3869 // request, with the user_data field set to the same value.
3870 // buffer_send must be keep alive until second cqe.
3871 var cqe_send = try ring.copy_cqe();
3872 try testing.expectEqual(linux.io_uring_cqe{
3873 .user_data = 0xeeeeeeee,
3874 .res = buffer_send.len,
3875 .flags = linux.IORING_CQE_F_MORE,
3876 }, cqe_send);
3877
3878 const cqe_recv = try ring.copy_cqe();
3879 try testing.expectEqual(linux.io_uring_cqe{
3880 .user_data = 0xffffffff,
3881 .res = buffer_recv.len,
3882 .flags = cqe_recv.flags & linux.IORING_CQE_F_SOCK_NONEMPTY,
3883 }, cqe_recv);
3884
3885 try testing.expectEqualSlices(u8, buffer_send[0..buffer_recv.len], buffer_recv[0..]);
3886
3887 // Second completion of zero-copy send.
3888 // IORING_CQE_F_NOTIF in flags signals that kernel is done with send_buffer
3889 cqe_send = try ring.copy_cqe();
3890 try testing.expectEqual(linux.io_uring_cqe{
3891 .user_data = 0xeeeeeeee,
3892 .res = 0,
3893 .flags = linux.IORING_CQE_F_NOTIF,
3894 }, cqe_send);
3895}
3896
3897test "accept_direct" {
3898 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3899
3900 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3901 error.SystemOutdated => return error.SkipZigTest,
3902 error.PermissionDenied => return error.SkipZigTest,
3903 else => return err,
3904 };
3905 defer ring.deinit();
3906 var address = try net.Address.parseIp4("127.0.0.1", 0);
3907
3908 // register direct file descriptors
3909 var registered_fds = [_]os.fd_t{-1} ** 2;
3910 try ring.register_files(registered_fds[0..]);
3911
3912 const listener_socket = try createListenerSocket(&address);
3913 defer posix.close(listener_socket);
3914
3915 const accept_userdata: u64 = 0xaaaaaaaa;
3916 const read_userdata: u64 = 0xbbbbbbbb;
3917 const data = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xa, 0xb, 0xc, 0xd, 0xe };
3918
3919 for (0..2) |_| {
3920 for (registered_fds, 0..) |_, i| {
3921 var buffer_recv = [_]u8{0} ** 16;
3922 const buffer_send: []const u8 = data[0 .. data.len - i]; // make it different at each loop
3923
3924 // submit accept, will chose registered fd and return index in cqe
3925 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3926 try testing.expectEqual(@as(u32, 1), try ring.submit());
3927
3928 // connect
3929 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3930 try os.connect(client, &address.any, address.getOsSockLen());
3931 defer posix.close(client);
3932
3933 // accept completion
3934 const cqe_accept = try ring.copy_cqe();
3935 try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
3936 const fd_index = cqe_accept.res;
3937 try testing.expect(fd_index < registered_fds.len);
3938 try testing.expect(cqe_accept.user_data == accept_userdata);
3939
3940 // send data
3941 _ = try os.send(client, buffer_send, 0);
3942
3943 // Example of how to use registered fd:
3944 // Submit receive to fixed file returned by accept (fd_index).
3945 // Fd field is set to registered file index, returned by accept.
3946 // Flag linux.IOSQE_FIXED_FILE must be set.
3947 const recv_sqe = try ring.recv(read_userdata, fd_index, .{ .buffer = &buffer_recv }, 0);
3948 recv_sqe.flags |= linux.IOSQE_FIXED_FILE;
3949 try testing.expectEqual(@as(u32, 1), try ring.submit());
3950
3951 // accept receive
3952 const recv_cqe = try ring.copy_cqe();
3953 try testing.expect(recv_cqe.user_data == read_userdata);
3954 try testing.expect(recv_cqe.res == buffer_send.len);
3955 try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]);
3956 }
3957 // no more available fds, accept will get NFILE error
3958 {
3959 // submit accept
3960 _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0);
3961 try testing.expectEqual(@as(u32, 1), try ring.submit());
3962 // connect
3963 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
3964 try os.connect(client, &address.any, address.getOsSockLen());
3965 defer posix.close(client);
3966 // completion with error
3967 const cqe_accept = try ring.copy_cqe();
3968 try testing.expect(cqe_accept.user_data == accept_userdata);
3969 try testing.expectEqual(os.E.NFILE, cqe_accept.err());
3970 }
3971 // return file descriptors to kernel
3972 try ring.register_files_update(0, registered_fds[0..]);
3973 }
3974 try ring.unregister_files();
3975}
3976
3977test "accept_multishot_direct" {
3978 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
3979
3980 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
3981 error.SystemOutdated => return error.SkipZigTest,
3982 error.PermissionDenied => return error.SkipZigTest,
3983 else => return err,
3984 };
3985 defer ring.deinit();
3986
3987 var address = try net.Address.parseIp4("127.0.0.1", 0);
3988
3989 var registered_fds = [_]os.fd_t{-1} ** 2;
3990 try ring.register_files(registered_fds[0..]);
3991
3992 const listener_socket = try createListenerSocket(&address);
3993 defer posix.close(listener_socket);
3994
3995 const accept_userdata: u64 = 0xaaaaaaaa;
3996
3997 for (0..2) |_| {
3998 // submit multishot accept
3999 // Will chose registered fd and return index of the selected registered file in cqe.
4000 _ = try ring.accept_multishot_direct(accept_userdata, listener_socket, null, null, 0);
4001 try testing.expectEqual(@as(u32, 1), try ring.submit());
4002
4003 for (registered_fds) |_| {
4004 // connect
4005 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
4006 try os.connect(client, &address.any, address.getOsSockLen());
4007 defer posix.close(client);
4008
4009 // accept completion
4010 const cqe_accept = try ring.copy_cqe();
4011 const fd_index = cqe_accept.res;
4012 try testing.expect(fd_index < registered_fds.len);
4013 try testing.expect(cqe_accept.user_data == accept_userdata);
4014 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE > 0); // has more is set
4015 }
4016 // No more available fds, accept will get NFILE error.
4017 // Multishot is terminated (more flag is not set).
4018 {
4019 // connect
4020 const client = try os.socket(address.any.family, os.SOCK.STREAM | os.SOCK.CLOEXEC, 0);
4021 try os.connect(client, &address.any, address.getOsSockLen());
4022 defer posix.close(client);
4023 // completion with error
4024 const cqe_accept = try ring.copy_cqe();
4025 try testing.expect(cqe_accept.user_data == accept_userdata);
4026 try testing.expectEqual(os.E.NFILE, cqe_accept.err());
4027 try testing.expect(cqe_accept.flags & linux.IORING_CQE_F_MORE == 0); // has more is not set
4028 }
4029 // return file descriptors to kernel
4030 try ring.register_files_update(0, registered_fds[0..]);
4031 }
4032 try ring.unregister_files();
4033}
4034
4035test "socket" {
4036 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4037
4038 var ring = IO_Uring.init(1, 0) catch |err| switch (err) {
4039 error.SystemOutdated => return error.SkipZigTest,
4040 error.PermissionDenied => return error.SkipZigTest,
4041 else => return err,
4042 };
4043 defer ring.deinit();
4044
4045 // prepare, submit socket operation
4046 _ = try ring.socket(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
4047 try testing.expectEqual(@as(u32, 1), try ring.submit());
4048
4049 // test completion
4050 var cqe = try ring.copy_cqe();
4051 try testing.expectEqual(os.E.SUCCESS, cqe.err());
4052 const fd: os.fd_t = @intCast(cqe.res);
4053 try testing.expect(fd > 2);
4054
4055 os.close(fd);
4056}
4057
4058test "socket_direct/socket_direct_alloc/close_direct" {
4059 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4060
4061 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
4062 error.SystemOutdated => return error.SkipZigTest,
4063 error.PermissionDenied => return error.SkipZigTest,
4064 else => return err,
4065 };
4066 defer ring.deinit();
4067
4068 var registered_fds = [_]os.fd_t{-1} ** 3;
4069 try ring.register_files(registered_fds[0..]);
4070
4071 // create socket in registered file descriptor at index 0 (last param)
4072 _ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 0);
4073 try testing.expectEqual(@as(u32, 1), try ring.submit());
4074 var cqe_socket = try ring.copy_cqe();
4075 try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
4076 try testing.expect(cqe_socket.res == 0);
4077
4078 // create socket in registered file descriptor at index 1 (last param)
4079 _ = try ring.socket_direct(0, linux.AF.INET, os.SOCK.STREAM, 0, 0, 1);
4080 try testing.expectEqual(@as(u32, 1), try ring.submit());
4081 cqe_socket = try ring.copy_cqe();
4082 try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
4083 try testing.expect(cqe_socket.res == 0); // res is 0 when index is specified
4084
4085 // create socket in kernel chosen file descriptor index (_alloc version)
4086 // completion res has index from registered files
4087 _ = try ring.socket_direct_alloc(0, linux.AF.INET, os.SOCK.STREAM, 0, 0);
4088 try testing.expectEqual(@as(u32, 1), try ring.submit());
4089 cqe_socket = try ring.copy_cqe();
4090 try testing.expectEqual(os.E.SUCCESS, cqe_socket.err());
4091 try testing.expect(cqe_socket.res == 2); // returns registered file index
4092
4093 // use sockets from registered_fds in connect operation
4094 var address = try net.Address.parseIp4("127.0.0.1", 0);
4095 const listener_socket = try createListenerSocket(&address);
4096 defer posix.close(listener_socket);
4097 const accept_userdata: u64 = 0xaaaaaaaa;
4098 const connect_userdata: u64 = 0xbbbbbbbb;
4099 const close_userdata: u64 = 0xcccccccc;
4100 for (registered_fds, 0..) |_, fd_index| {
4101 // prepare accept
4102 _ = try ring.accept(accept_userdata, listener_socket, null, null, 0);
4103 // prepare connect with fixed socket
4104 const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), &address.any, address.getOsSockLen());
4105 connect_sqe.flags |= linux.IOSQE_FIXED_FILE; // fd is fixed file index
4106 // submit both
4107 try testing.expectEqual(@as(u32, 2), try ring.submit());
4108 // get completions
4109 var cqe_connect = try ring.copy_cqe();
4110 var cqe_accept = try ring.copy_cqe();
4111 // ignore order
4112 if (cqe_connect.user_data == accept_userdata and cqe_accept.user_data == connect_userdata) {
4113 const a = cqe_accept;
4114 const b = cqe_connect;
4115 cqe_accept = b;
4116 cqe_connect = a;
4117 }
4118 // test connect completion
4119 try testing.expect(cqe_connect.user_data == connect_userdata);
4120 try testing.expectEqual(os.E.SUCCESS, cqe_connect.err());
4121 // test accept completion
4122 try testing.expect(cqe_accept.user_data == accept_userdata);
4123 try testing.expectEqual(os.E.SUCCESS, cqe_accept.err());
4124
4125 // submit and test close_direct
4126 _ = try ring.close_direct(close_userdata, @intCast(fd_index));
4127 try testing.expectEqual(@as(u32, 1), try ring.submit());
4128 var cqe_close = try ring.copy_cqe();
4129 try testing.expect(cqe_close.user_data == close_userdata);
4130 try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
4131 }
4132
4133 try ring.unregister_files();
4134}
4135
4136test "openat_direct/close_direct" {
4137 try skipKernelLessThan(.{ .major = 5, .minor = 19, .patch = 0 });
4138
4139 var ring = IO_Uring.init(2, 0) catch |err| switch (err) {
4140 error.SystemOutdated => return error.SkipZigTest,
4141 error.PermissionDenied => return error.SkipZigTest,
4142 else => return err,
4143 };
4144 defer ring.deinit();
4145
4146 var registered_fds = [_]os.fd_t{-1} ** 3;
4147 try ring.register_files(registered_fds[0..]);
4148
4149 var tmp = std.testing.tmpDir(.{});
4150 defer tmp.cleanup();
4151 const path = "test_io_uring_close_direct";
4152 const flags: linux.O = .{ .ACCMODE = .RDWR, .CREAT = true };
4153 const mode: os.mode_t = 0o666;
4154 const user_data: u64 = 0;
4155
4156 // use registered file at index 0 (last param)
4157 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 0);
4158 try testing.expectEqual(@as(u32, 1), try ring.submit());
4159 var cqe = try ring.copy_cqe();
4160 try testing.expectEqual(os.E.SUCCESS, cqe.err());
4161 try testing.expect(cqe.res == 0);
4162
4163 // use registered file at index 1
4164 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, 1);
4165 try testing.expectEqual(@as(u32, 1), try ring.submit());
4166 cqe = try ring.copy_cqe();
4167 try testing.expectEqual(os.E.SUCCESS, cqe.err());
4168 try testing.expect(cqe.res == 0); // res is 0 when we specify index
4169
4170 // let kernel choose registered file index
4171 _ = try ring.openat_direct(user_data, tmp.dir.fd, path, flags, mode, linux.IORING_FILE_INDEX_ALLOC);
4172 try testing.expectEqual(@as(u32, 1), try ring.submit());
4173 cqe = try ring.copy_cqe();
4174 try testing.expectEqual(os.E.SUCCESS, cqe.err());
4175 try testing.expect(cqe.res == 2); // chosen index is in res
4176
4177 // close all open file descriptors
4178 for (registered_fds, 0..) |_, fd_index| {
4179 _ = try ring.close_direct(user_data, @intCast(fd_index));
4180 try testing.expectEqual(@as(u32, 1), try ring.submit());
4181 var cqe_close = try ring.copy_cqe();
4182 try testing.expectEqual(os.E.SUCCESS, cqe_close.err());
4183 }
4184 try ring.unregister_files();
4185}
4186
4187test "waitid" {
4188 try skipKernelLessThan(.{ .major = 6, .minor = 7, .patch = 0 });
4189
4190 var ring = IO_Uring.init(16, 0) catch |err| switch (err) {
4191 error.SystemOutdated => return error.SkipZigTest,
4192 error.PermissionDenied => return error.SkipZigTest,
4193 else => return err,
4194 };
4195 defer ring.deinit();
4196
4197 const pid = try os.fork();
4198 if (pid == 0) {
4199 os.exit(7);
4200 }
4201
4202 var siginfo: os.siginfo_t = undefined;
4203 _ = try ring.waitid(0, .PID, pid, &siginfo, os.W.EXITED, 0);
4204
4205 try testing.expectEqual(1, try ring.submit());
4206
4207 const cqe_waitid = try ring.copy_cqe();
4208 try testing.expectEqual(0, cqe_waitid.res);
4209 try testing.expectEqual(pid, siginfo.fields.common.first.piduid.pid);
4210 try testing.expectEqual(7, siginfo.fields.common.second.sigchld.status);
4211}
4212
4213/// For use in tests. Returns SkipZigTest is kernel version is less than required.
4214inline fn skipKernelLessThan(required: std.SemanticVersion) !void {
4215 if (builtin.os.tag != .linux) return error.SkipZigTest;
4216
4217 var uts: linux.utsname = undefined;
4218 const res = linux.uname(&uts);
4219 switch (linux.getErrno(res)) {
4220 .SUCCESS => {},
4221 else => |errno| return os.unexpectedErrno(errno),
4222 }
4223
4224 const release = mem.sliceTo(&uts.release, 0);
4225 var current = try std.SemanticVersion.parse(release);
4226 current.pre = null; // don't check pre field
4227 if (required.order(current) == .gt) return error.SkipZigTest;
4228}
lib/std/os/linux/io_uring_sqe.zig created+579
...@@ -0,0 +1,579 @@
1//! Contains only the definition of `io_uring_sqe`.
2//! Split into its own file to compartmentalize the initialization methods.
3
4const std = @import("../../std.zig");
5const os = std.os;
6const linux = os.linux;
7
8pub const io_uring_sqe = extern struct {
9 opcode: linux.IORING_OP,
10 flags: u8,
11 ioprio: u16,
12 fd: i32,
13 off: u64,
14 addr: u64,
15 len: u32,
16 rw_flags: u32,
17 user_data: u64,
18 buf_index: u16,
19 personality: u16,
20 splice_fd_in: i32,
21 addr3: u64,
22 resv: u64,
23
24 pub fn prep_nop(sqe: *linux.io_uring_sqe) void {
25 sqe.* = .{
26 .opcode = .NOP,
27 .flags = 0,
28 .ioprio = 0,
29 .fd = 0,
30 .off = 0,
31 .addr = 0,
32 .len = 0,
33 .rw_flags = 0,
34 .user_data = 0,
35 .buf_index = 0,
36 .personality = 0,
37 .splice_fd_in = 0,
38 .addr3 = 0,
39 .resv = 0,
40 };
41 }
42
43 pub fn prep_fsync(sqe: *linux.io_uring_sqe, fd: os.fd_t, flags: u32) void {
44 sqe.* = .{
45 .opcode = .FSYNC,
46 .flags = 0,
47 .ioprio = 0,
48 .fd = fd,
49 .off = 0,
50 .addr = 0,
51 .len = 0,
52 .rw_flags = flags,
53 .user_data = 0,
54 .buf_index = 0,
55 .personality = 0,
56 .splice_fd_in = 0,
57 .addr3 = 0,
58 .resv = 0,
59 };
60 }
61
62 pub fn prep_rw(
63 sqe: *linux.io_uring_sqe,
64 op: linux.IORING_OP,
65 fd: os.fd_t,
66 addr: u64,
67 len: usize,
68 offset: u64,
69 ) void {
70 sqe.* = .{
71 .opcode = op,
72 .flags = 0,
73 .ioprio = 0,
74 .fd = fd,
75 .off = offset,
76 .addr = addr,
77 .len = @intCast(len),
78 .rw_flags = 0,
79 .user_data = 0,
80 .buf_index = 0,
81 .personality = 0,
82 .splice_fd_in = 0,
83 .addr3 = 0,
84 .resv = 0,
85 };
86 }
87
88 pub fn prep_read(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, offset: u64) void {
89 sqe.prep_rw(.READ, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
90 }
91
92 pub fn prep_write(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, offset: u64) void {
93 sqe.prep_rw(.WRITE, fd, @intFromPtr(buffer.ptr), buffer.len, offset);
94 }
95
96 pub fn prep_splice(sqe: *linux.io_uring_sqe, fd_in: os.fd_t, off_in: u64, fd_out: os.fd_t, off_out: u64, len: usize) void {
97 sqe.prep_rw(.SPLICE, fd_out, undefined, len, off_out);
98 sqe.addr = off_in;
99 sqe.splice_fd_in = fd_in;
100 }
101
102 pub fn prep_readv(
103 sqe: *linux.io_uring_sqe,
104 fd: os.fd_t,
105 iovecs: []const os.iovec,
106 offset: u64,
107 ) void {
108 sqe.prep_rw(.READV, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
109 }
110
111 pub fn prep_writev(
112 sqe: *linux.io_uring_sqe,
113 fd: os.fd_t,
114 iovecs: []const os.iovec_const,
115 offset: u64,
116 ) void {
117 sqe.prep_rw(.WRITEV, fd, @intFromPtr(iovecs.ptr), iovecs.len, offset);
118 }
119
120 pub fn prep_read_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: *os.iovec, offset: u64, buffer_index: u16) void {
121 sqe.prep_rw(.READ_FIXED, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
122 sqe.buf_index = buffer_index;
123 }
124
125 pub fn prep_write_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: *os.iovec, offset: u64, buffer_index: u16) void {
126 sqe.prep_rw(.WRITE_FIXED, fd, @intFromPtr(buffer.iov_base), buffer.iov_len, offset);
127 sqe.buf_index = buffer_index;
128 }
129
130 pub fn prep_accept(
131 sqe: *linux.io_uring_sqe,
132 fd: os.fd_t,
133 addr: ?*os.sockaddr,
134 addrlen: ?*os.socklen_t,
135 flags: u32,
136 ) void {
137 // `addr` holds a pointer to `sockaddr`, and `addr2` holds a pointer to socklen_t`.
138 // `addr2` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
139 sqe.prep_rw(.ACCEPT, fd, @intFromPtr(addr), 0, @intFromPtr(addrlen));
140 sqe.rw_flags = flags;
141 }
142
143 pub fn prep_accept_direct(
144 sqe: *linux.io_uring_sqe,
145 fd: os.fd_t,
146 addr: ?*os.sockaddr,
147 addrlen: ?*os.socklen_t,
148 flags: u32,
149 file_index: u32,
150 ) void {
151 prep_accept(sqe, fd, addr, addrlen, flags);
152 __io_uring_set_target_fixed_file(sqe, file_index);
153 }
154
155 pub fn prep_multishot_accept_direct(
156 sqe: *linux.io_uring_sqe,
157 fd: os.fd_t,
158 addr: ?*os.sockaddr,
159 addrlen: ?*os.socklen_t,
160 flags: u32,
161 ) void {
162 prep_multishot_accept(sqe, fd, addr, addrlen, flags);
163 __io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
164 }
165
166 fn __io_uring_set_target_fixed_file(sqe: *linux.io_uring_sqe, file_index: u32) void {
167 const sqe_file_index: u32 = if (file_index == linux.IORING_FILE_INDEX_ALLOC)
168 linux.IORING_FILE_INDEX_ALLOC
169 else
170 // 0 means no fixed files, indexes should be encoded as "index + 1"
171 file_index + 1;
172 // This filed is overloaded in liburing:
173 // splice_fd_in: i32
174 // sqe_file_index: u32
175 sqe.splice_fd_in = @bitCast(sqe_file_index);
176 }
177
178 pub fn prep_connect(
179 sqe: *linux.io_uring_sqe,
180 fd: os.fd_t,
181 addr: *const os.sockaddr,
182 addrlen: os.socklen_t,
183 ) void {
184 // `addrlen` maps to `sqe.off` (u64) instead of `sqe.len` (which is only a u32).
185 sqe.prep_rw(.CONNECT, fd, @intFromPtr(addr), 0, addrlen);
186 }
187
188 pub fn prep_epoll_ctl(
189 sqe: *linux.io_uring_sqe,
190 epfd: os.fd_t,
191 fd: os.fd_t,
192 op: u32,
193 ev: ?*linux.epoll_event,
194 ) void {
195 sqe.prep_rw(.EPOLL_CTL, epfd, @intFromPtr(ev), op, @intCast(fd));
196 }
197
198 pub fn prep_recv(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []u8, flags: u32) void {
199 sqe.prep_rw(.RECV, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
200 sqe.rw_flags = flags;
201 }
202
203 pub fn prep_send(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32) void {
204 sqe.prep_rw(.SEND, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
205 sqe.rw_flags = flags;
206 }
207
208 pub fn prep_send_zc(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16) void {
209 sqe.prep_rw(.SEND_ZC, fd, @intFromPtr(buffer.ptr), buffer.len, 0);
210 sqe.rw_flags = flags;
211 sqe.ioprio = zc_flags;
212 }
213
214 pub fn prep_send_zc_fixed(sqe: *linux.io_uring_sqe, fd: os.fd_t, buffer: []const u8, flags: u32, zc_flags: u16, buf_index: u16) void {
215 prep_send_zc(sqe, fd, buffer, flags, zc_flags);
216 sqe.ioprio |= linux.IORING_RECVSEND_FIXED_BUF;
217 sqe.buf_index = buf_index;
218 }
219
220 pub fn prep_sendmsg_zc(
221 sqe: *linux.io_uring_sqe,
222 fd: os.fd_t,
223 msg: *const os.msghdr_const,
224 flags: u32,
225 ) void {
226 prep_sendmsg(sqe, fd, msg, flags);
227 sqe.opcode = .SENDMSG_ZC;
228 }
229
230 pub fn prep_recvmsg(
231 sqe: *linux.io_uring_sqe,
232 fd: os.fd_t,
233 msg: *os.msghdr,
234 flags: u32,
235 ) void {
236 sqe.prep_rw(.RECVMSG, fd, @intFromPtr(msg), 1, 0);
237 sqe.rw_flags = flags;
238 }
239
240 pub fn prep_sendmsg(
241 sqe: *linux.io_uring_sqe,
242 fd: os.fd_t,
243 msg: *const os.msghdr_const,
244 flags: u32,
245 ) void {
246 sqe.prep_rw(.SENDMSG, fd, @intFromPtr(msg), 1, 0);
247 sqe.rw_flags = flags;
248 }
249
250 pub fn prep_openat(
251 sqe: *linux.io_uring_sqe,
252 fd: os.fd_t,
253 path: [*:0]const u8,
254 flags: linux.O,
255 mode: os.mode_t,
256 ) void {
257 sqe.prep_rw(.OPENAT, fd, @intFromPtr(path), mode, 0);
258 sqe.rw_flags = @bitCast(flags);
259 }
260
261 pub fn prep_openat_direct(
262 sqe: *linux.io_uring_sqe,
263 fd: os.fd_t,
264 path: [*:0]const u8,
265 flags: linux.O,
266 mode: os.mode_t,
267 file_index: u32,
268 ) void {
269 prep_openat(sqe, fd, path, flags, mode);
270 __io_uring_set_target_fixed_file(sqe, file_index);
271 }
272
273 pub fn prep_close(sqe: *linux.io_uring_sqe, fd: os.fd_t) void {
274 sqe.* = .{
275 .opcode = .CLOSE,
276 .flags = 0,
277 .ioprio = 0,
278 .fd = fd,
279 .off = 0,
280 .addr = 0,
281 .len = 0,
282 .rw_flags = 0,
283 .user_data = 0,
284 .buf_index = 0,
285 .personality = 0,
286 .splice_fd_in = 0,
287 .addr3 = 0,
288 .resv = 0,
289 };
290 }
291
292 pub fn prep_close_direct(sqe: *linux.io_uring_sqe, file_index: u32) void {
293 prep_close(sqe, 0);
294 __io_uring_set_target_fixed_file(sqe, file_index);
295 }
296
297 pub fn prep_timeout(
298 sqe: *linux.io_uring_sqe,
299 ts: *const os.linux.kernel_timespec,
300 count: u32,
301 flags: u32,
302 ) void {
303 sqe.prep_rw(.TIMEOUT, -1, @intFromPtr(ts), 1, count);
304 sqe.rw_flags = flags;
305 }
306
307 pub fn prep_timeout_remove(sqe: *linux.io_uring_sqe, timeout_user_data: u64, flags: u32) void {
308 sqe.* = .{
309 .opcode = .TIMEOUT_REMOVE,
310 .flags = 0,
311 .ioprio = 0,
312 .fd = -1,
313 .off = 0,
314 .addr = timeout_user_data,
315 .len = 0,
316 .rw_flags = flags,
317 .user_data = 0,
318 .buf_index = 0,
319 .personality = 0,
320 .splice_fd_in = 0,
321 .addr3 = 0,
322 .resv = 0,
323 };
324 }
325
326 pub fn prep_link_timeout(
327 sqe: *linux.io_uring_sqe,
328 ts: *const os.linux.kernel_timespec,
329 flags: u32,
330 ) void {
331 sqe.prep_rw(.LINK_TIMEOUT, -1, @intFromPtr(ts), 1, 0);
332 sqe.rw_flags = flags;
333 }
334
335 pub fn prep_poll_add(
336 sqe: *linux.io_uring_sqe,
337 fd: os.fd_t,
338 poll_mask: u32,
339 ) void {
340 sqe.prep_rw(.POLL_ADD, fd, @intFromPtr(@as(?*anyopaque, null)), 0, 0);
341 // Poll masks previously used to comprise of 16 bits in the flags union of
342 // a SQE, but were then extended to comprise of 32 bits in order to make
343 // room for additional option flags. To ensure that the correct bits of
344 // poll masks are consistently and properly read across multiple kernel
345 // versions, poll masks are enforced to be little-endian.
346 // https://www.spinics.net/lists/io-uring/msg02848.html
347 sqe.rw_flags = std.mem.nativeToLittle(u32, poll_mask);
348 }
349
350 pub fn prep_poll_remove(
351 sqe: *linux.io_uring_sqe,
352 target_user_data: u64,
353 ) void {
354 sqe.prep_rw(.POLL_REMOVE, -1, target_user_data, 0, 0);
355 }
356
357 pub fn prep_poll_update(
358 sqe: *linux.io_uring_sqe,
359 old_user_data: u64,
360 new_user_data: u64,
361 poll_mask: u32,
362 flags: u32,
363 ) void {
364 sqe.prep_rw(.POLL_REMOVE, -1, old_user_data, flags, new_user_data);
365 // Poll masks previously used to comprise of 16 bits in the flags union of
366 // a SQE, but were then extended to comprise of 32 bits in order to make
367 // room for additional option flags. To ensure that the correct bits of
368 // poll masks are consistently and properly read across multiple kernel
369 // versions, poll masks are enforced to be little-endian.
370 // https://www.spinics.net/lists/io-uring/msg02848.html
371 sqe.rw_flags = std.mem.nativeToLittle(u32, poll_mask);
372 }
373
374 pub fn prep_fallocate(
375 sqe: *linux.io_uring_sqe,
376 fd: os.fd_t,
377 mode: i32,
378 offset: u64,
379 len: u64,
380 ) void {
381 sqe.* = .{
382 .opcode = .FALLOCATE,
383 .flags = 0,
384 .ioprio = 0,
385 .fd = fd,
386 .off = offset,
387 .addr = len,
388 .len = @intCast(mode),
389 .rw_flags = 0,
390 .user_data = 0,
391 .buf_index = 0,
392 .personality = 0,
393 .splice_fd_in = 0,
394 .addr3 = 0,
395 .resv = 0,
396 };
397 }
398
399 pub fn prep_statx(
400 sqe: *linux.io_uring_sqe,
401 fd: os.fd_t,
402 path: [*:0]const u8,
403 flags: u32,
404 mask: u32,
405 buf: *linux.Statx,
406 ) void {
407 sqe.prep_rw(.STATX, fd, @intFromPtr(path), mask, @intFromPtr(buf));
408 sqe.rw_flags = flags;
409 }
410
411 pub fn prep_cancel(
412 sqe: *linux.io_uring_sqe,
413 cancel_user_data: u64,
414 flags: u32,
415 ) void {
416 sqe.prep_rw(.ASYNC_CANCEL, -1, cancel_user_data, 0, 0);
417 sqe.rw_flags = flags;
418 }
419
420 pub fn prep_shutdown(
421 sqe: *linux.io_uring_sqe,
422 sockfd: os.socket_t,
423 how: u32,
424 ) void {
425 sqe.prep_rw(.SHUTDOWN, sockfd, 0, how, 0);
426 }
427
428 pub fn prep_renameat(
429 sqe: *linux.io_uring_sqe,
430 old_dir_fd: os.fd_t,
431 old_path: [*:0]const u8,
432 new_dir_fd: os.fd_t,
433 new_path: [*:0]const u8,
434 flags: u32,
435 ) void {
436 sqe.prep_rw(
437 .RENAMEAT,
438 old_dir_fd,
439 @intFromPtr(old_path),
440 0,
441 @intFromPtr(new_path),
442 );
443 sqe.len = @bitCast(new_dir_fd);
444 sqe.rw_flags = flags;
445 }
446
447 pub fn prep_unlinkat(
448 sqe: *linux.io_uring_sqe,
449 dir_fd: os.fd_t,
450 path: [*:0]const u8,
451 flags: u32,
452 ) void {
453 sqe.prep_rw(.UNLINKAT, dir_fd, @intFromPtr(path), 0, 0);
454 sqe.rw_flags = flags;
455 }
456
457 pub fn prep_mkdirat(
458 sqe: *linux.io_uring_sqe,
459 dir_fd: os.fd_t,
460 path: [*:0]const u8,
461 mode: os.mode_t,
462 ) void {
463 sqe.prep_rw(.MKDIRAT, dir_fd, @intFromPtr(path), mode, 0);
464 }
465
466 pub fn prep_symlinkat(
467 sqe: *linux.io_uring_sqe,
468 target: [*:0]const u8,
469 new_dir_fd: os.fd_t,
470 link_path: [*:0]const u8,
471 ) void {
472 sqe.prep_rw(
473 .SYMLINKAT,
474 new_dir_fd,
475 @intFromPtr(target),
476 0,
477 @intFromPtr(link_path),
478 );
479 }
480
481 pub fn prep_linkat(
482 sqe: *linux.io_uring_sqe,
483 old_dir_fd: os.fd_t,
484 old_path: [*:0]const u8,
485 new_dir_fd: os.fd_t,
486 new_path: [*:0]const u8,
487 flags: u32,
488 ) void {
489 sqe.prep_rw(
490 .LINKAT,
491 old_dir_fd,
492 @intFromPtr(old_path),
493 0,
494 @intFromPtr(new_path),
495 );
496 sqe.len = @bitCast(new_dir_fd);
497 sqe.rw_flags = flags;
498 }
499
500 pub fn prep_provide_buffers(
501 sqe: *linux.io_uring_sqe,
502 buffers: [*]u8,
503 buffer_len: usize,
504 num: usize,
505 group_id: usize,
506 buffer_id: usize,
507 ) void {
508 const ptr = @intFromPtr(buffers);
509 sqe.prep_rw(.PROVIDE_BUFFERS, @intCast(num), ptr, buffer_len, buffer_id);
510 sqe.buf_index = @intCast(group_id);
511 }
512
513 pub fn prep_remove_buffers(
514 sqe: *linux.io_uring_sqe,
515 num: usize,
516 group_id: usize,
517 ) void {
518 sqe.prep_rw(.REMOVE_BUFFERS, @intCast(num), 0, 0, 0);
519 sqe.buf_index = @intCast(group_id);
520 }
521
522 pub fn prep_multishot_accept(
523 sqe: *linux.io_uring_sqe,
524 fd: os.fd_t,
525 addr: ?*os.sockaddr,
526 addrlen: ?*os.socklen_t,
527 flags: u32,
528 ) void {
529 prep_accept(sqe, fd, addr, addrlen, flags);
530 sqe.ioprio |= linux.IORING_ACCEPT_MULTISHOT;
531 }
532
533 pub fn prep_socket(
534 sqe: *linux.io_uring_sqe,
535 domain: u32,
536 socket_type: u32,
537 protocol: u32,
538 flags: u32,
539 ) void {
540 sqe.prep_rw(.SOCKET, @intCast(domain), 0, protocol, socket_type);
541 sqe.rw_flags = flags;
542 }
543
544 pub fn prep_socket_direct(
545 sqe: *linux.io_uring_sqe,
546 domain: u32,
547 socket_type: u32,
548 protocol: u32,
549 flags: u32,
550 file_index: u32,
551 ) void {
552 prep_socket(sqe, domain, socket_type, protocol, flags);
553 __io_uring_set_target_fixed_file(sqe, file_index);
554 }
555
556 pub fn prep_socket_direct_alloc(
557 sqe: *linux.io_uring_sqe,
558 domain: u32,
559 socket_type: u32,
560 protocol: u32,
561 flags: u32,
562 ) void {
563 prep_socket(sqe, domain, socket_type, protocol, flags);
564 __io_uring_set_target_fixed_file(sqe, linux.IORING_FILE_INDEX_ALLOC);
565 }
566
567 pub fn prep_waitid(
568 sqe: *linux.io_uring_sqe,
569 id_type: linux.P,
570 id: i32,
571 infop: *linux.siginfo_t,
572 options: u32,
573 flags: u32,
574 ) void {
575 sqe.prep_rw(.WAITID, id, 0, @intFromEnum(id_type), @intFromPtr(infop));
576 sqe.rw_flags = flags;
577 sqe.splice_fd_in = @bitCast(options);
578 }
579};
lib/std/os/linux/test.zig+4
...@@ -120,3 +120,7 @@ test "fadvise" {...@@ -120,3 +120,7 @@ test "fadvise" {
120 const ret = linux.fadvise(file.handle, 0, 0, linux.POSIX_FADV.SEQUENTIAL);120 const ret = linux.fadvise(file.handle, 0, 0, linux.POSIX_FADV.SEQUENTIAL);
121 try expectEqual(@as(usize, 0), ret);121 try expectEqual(@as(usize, 0), ret);
122}122}
123
124test {
125 _ = linux.IoUring;
126}
lib/std/os/windows.zig+278-268
...@@ -3796,290 +3796,300 @@ pub const EXCEPTION_RECORD = extern struct {...@@ -3796,290 +3796,300 @@ pub const EXCEPTION_RECORD = extern struct {
3796 ExceptionInformation: [15]usize,3796 ExceptionInformation: [15]usize,
3797};3797};
37983798
3799pub usingnamespace switch (native_arch) {3799pub const FLOATING_SAVE_AREA = switch (native_arch) {
3800 .x86 => struct {3800 .x86 => extern struct {
3801 pub const FLOATING_SAVE_AREA = extern struct {3801 ControlWord: DWORD,
3802 ControlWord: DWORD,3802 StatusWord: DWORD,
3803 StatusWord: DWORD,3803 TagWord: DWORD,
3804 TagWord: DWORD,3804 ErrorOffset: DWORD,
3805 ErrorOffset: DWORD,3805 ErrorSelector: DWORD,
3806 ErrorSelector: DWORD,3806 DataOffset: DWORD,
3807 DataOffset: DWORD,3807 DataSelector: DWORD,
3808 DataSelector: DWORD,3808 RegisterArea: [80]BYTE,
3809 RegisterArea: [80]BYTE,3809 Cr0NpxState: DWORD,
3810 Cr0NpxState: DWORD,3810 },
3811 };3811 else => @compileError("FLOATING_SAVE_AREA only defined on x86"),
3812};
38123813
3813 pub const CONTEXT = extern struct {3814pub const M128A = switch (native_arch) {
3814 ContextFlags: DWORD,3815 .x86_64 => extern struct {
3815 Dr0: DWORD,3816 Low: ULONGLONG,
3816 Dr1: DWORD,3817 High: LONGLONG,
3817 Dr2: DWORD,3818 },
3818 Dr3: DWORD,3819 else => @compileError("M128A only defined on x86_64"),
3819 Dr6: DWORD,3820};
3820 Dr7: DWORD,3821
3821 FloatSave: FLOATING_SAVE_AREA,3822pub const XMM_SAVE_AREA32 = switch (native_arch) {
3822 SegGs: DWORD,3823 .x86_64 => extern struct {
3823 SegFs: DWORD,3824 ControlWord: WORD,
3824 SegEs: DWORD,3825 StatusWord: WORD,
3825 SegDs: DWORD,3826 TagWord: BYTE,
3826 Edi: DWORD,3827 Reserved1: BYTE,
3827 Esi: DWORD,3828 ErrorOpcode: WORD,
3828 Ebx: DWORD,3829 ErrorOffset: DWORD,
3829 Edx: DWORD,3830 ErrorSelector: WORD,
3830 Ecx: DWORD,3831 Reserved2: WORD,
3831 Eax: DWORD,3832 DataOffset: DWORD,
3832 Ebp: DWORD,3833 DataSelector: WORD,
3833 Eip: DWORD,3834 Reserved3: WORD,
3834 SegCs: DWORD,3835 MxCsr: DWORD,
3835 EFlags: DWORD,3836 MxCsr_Mask: DWORD,
3836 Esp: DWORD,3837 FloatRegisters: [8]M128A,
3837 SegSs: DWORD,3838 XmmRegisters: [16]M128A,
3838 ExtendedRegisters: [512]BYTE,3839 Reserved4: [96]BYTE,
3839
3840 pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize } {
3841 return .{ .bp = ctx.Ebp, .ip = ctx.Eip };
3842 }
3843 };
3844 },3840 },
3845 .x86_64 => struct {3841 else => @compileError("XMM_SAVE_AREA32 only defined on x86_64"),
3846 pub const M128A = extern struct {3842};
3843
3844pub const NEON128 = switch (native_arch) {
3845 .aarch64 => extern union {
3846 DUMMYSTRUCTNAME: extern struct {
3847 Low: ULONGLONG,3847 Low: ULONGLONG,
3848 High: LONGLONG,3848 High: LONGLONG,
3849 };3849 },
38503850 D: [2]f64,
3851 pub const XMM_SAVE_AREA32 = extern struct {3851 S: [4]f32,
3852 ControlWord: WORD,3852 H: [8]WORD,
3853 StatusWord: WORD,3853 B: [16]BYTE,
3854 TagWord: BYTE,3854 },
3855 Reserved1: BYTE,3855 else => @compileError("NEON128 only defined on aarch64"),
3856 ErrorOpcode: WORD,3856};
3857 ErrorOffset: DWORD,3857
3858 ErrorSelector: WORD,3858pub const CONTEXT = switch (native_arch) {
3859 Reserved2: WORD,3859 .x86 => extern struct {
3860 DataOffset: DWORD,3860 ContextFlags: DWORD,
3861 DataSelector: WORD,3861 Dr0: DWORD,
3862 Reserved3: WORD,3862 Dr1: DWORD,
3863 MxCsr: DWORD,3863 Dr2: DWORD,
3864 MxCsr_Mask: DWORD,3864 Dr3: DWORD,
3865 FloatRegisters: [8]M128A,3865 Dr6: DWORD,
3866 XmmRegisters: [16]M128A,3866 Dr7: DWORD,
3867 Reserved4: [96]BYTE,3867 FloatSave: FLOATING_SAVE_AREA,
3868 };3868 SegGs: DWORD,
38693869 SegFs: DWORD,
3870 pub const CONTEXT = extern struct {3870 SegEs: DWORD,
3871 P1Home: DWORD64 align(16),3871 SegDs: DWORD,
3872 P2Home: DWORD64,3872 Edi: DWORD,
3873 P3Home: DWORD64,3873 Esi: DWORD,
3874 P4Home: DWORD64,3874 Ebx: DWORD,
3875 P5Home: DWORD64,3875 Edx: DWORD,
3876 P6Home: DWORD64,3876 Ecx: DWORD,
3877 ContextFlags: DWORD,3877 Eax: DWORD,
3878 MxCsr: DWORD,3878 Ebp: DWORD,
3879 SegCs: WORD,3879 Eip: DWORD,
3880 SegDs: WORD,3880 SegCs: DWORD,
3881 SegEs: WORD,3881 EFlags: DWORD,
3882 SegFs: WORD,3882 Esp: DWORD,
3883 SegGs: WORD,3883 SegSs: DWORD,
3884 SegSs: WORD,3884 ExtendedRegisters: [512]BYTE,
3885 EFlags: DWORD,3885
3886 Dr0: DWORD64,3886 pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize } {
3887 Dr1: DWORD64,3887 return .{ .bp = ctx.Ebp, .ip = ctx.Eip };
3888 Dr2: DWORD64,3888 }
3889 Dr3: DWORD64,3889 },
3890 Dr6: DWORD64,3890 .x86_64 => extern struct {
3891 Dr7: DWORD64,3891 P1Home: DWORD64 align(16),
3892 Rax: DWORD64,3892 P2Home: DWORD64,
3893 Rcx: DWORD64,3893 P3Home: DWORD64,
3894 Rdx: DWORD64,3894 P4Home: DWORD64,
3895 Rbx: DWORD64,3895 P5Home: DWORD64,
3896 Rsp: DWORD64,3896 P6Home: DWORD64,
3897 Rbp: DWORD64,3897 ContextFlags: DWORD,
3898 Rsi: DWORD64,3898 MxCsr: DWORD,
3899 Rdi: DWORD64,3899 SegCs: WORD,
3900 R8: DWORD64,3900 SegDs: WORD,
3901 R9: DWORD64,3901 SegEs: WORD,
3902 R10: DWORD64,3902 SegFs: WORD,
3903 R11: DWORD64,3903 SegGs: WORD,
3904 R12: DWORD64,3904 SegSs: WORD,
3905 R13: DWORD64,3905 EFlags: DWORD,
3906 R14: DWORD64,3906 Dr0: DWORD64,
3907 R15: DWORD64,3907 Dr1: DWORD64,
3908 Rip: DWORD64,3908 Dr2: DWORD64,
3909 DUMMYUNIONNAME: extern union {3909 Dr3: DWORD64,
3910 FltSave: XMM_SAVE_AREA32,3910 Dr6: DWORD64,
3911 FloatSave: XMM_SAVE_AREA32,3911 Dr7: DWORD64,
3912 DUMMYSTRUCTNAME: extern struct {3912 Rax: DWORD64,
3913 Header: [2]M128A,3913 Rcx: DWORD64,
3914 Legacy: [8]M128A,3914 Rdx: DWORD64,
3915 Xmm0: M128A,3915 Rbx: DWORD64,
3916 Xmm1: M128A,3916 Rsp: DWORD64,
3917 Xmm2: M128A,3917 Rbp: DWORD64,
3918 Xmm3: M128A,3918 Rsi: DWORD64,
3919 Xmm4: M128A,3919 Rdi: DWORD64,
3920 Xmm5: M128A,3920 R8: DWORD64,
3921 Xmm6: M128A,3921 R9: DWORD64,
3922 Xmm7: M128A,3922 R10: DWORD64,
3923 Xmm8: M128A,3923 R11: DWORD64,
3924 Xmm9: M128A,3924 R12: DWORD64,
3925 Xmm10: M128A,3925 R13: DWORD64,
3926 Xmm11: M128A,3926 R14: DWORD64,
3927 Xmm12: M128A,3927 R15: DWORD64,
3928 Xmm13: M128A,3928 Rip: DWORD64,
3929 Xmm14: M128A,3929 DUMMYUNIONNAME: extern union {
3930 Xmm15: M128A,3930 FltSave: XMM_SAVE_AREA32,
3931 },3931 FloatSave: XMM_SAVE_AREA32,
3932 DUMMYSTRUCTNAME: extern struct {
3933 Header: [2]M128A,
3934 Legacy: [8]M128A,
3935 Xmm0: M128A,
3936 Xmm1: M128A,
3937 Xmm2: M128A,
3938 Xmm3: M128A,
3939 Xmm4: M128A,
3940 Xmm5: M128A,
3941 Xmm6: M128A,
3942 Xmm7: M128A,
3943 Xmm8: M128A,
3944 Xmm9: M128A,
3945 Xmm10: M128A,
3946 Xmm11: M128A,
3947 Xmm12: M128A,
3948 Xmm13: M128A,
3949 Xmm14: M128A,
3950 Xmm15: M128A,
3932 },3951 },
3933 VectorRegister: [26]M128A,3952 },
3934 VectorControl: DWORD64,3953 VectorRegister: [26]M128A,
3935 DebugControl: DWORD64,3954 VectorControl: DWORD64,
3936 LastBranchToRip: DWORD64,3955 DebugControl: DWORD64,
3937 LastBranchFromRip: DWORD64,3956 LastBranchToRip: DWORD64,
3938 LastExceptionToRip: DWORD64,3957 LastBranchFromRip: DWORD64,
3939 LastExceptionFromRip: DWORD64,3958 LastExceptionToRip: DWORD64,
39403959 LastExceptionFromRip: DWORD64,
3941 pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize, sp: usize } {3960
3942 return .{ .bp = ctx.Rbp, .ip = ctx.Rip, .sp = ctx.Rsp };3961 pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize, sp: usize } {
3943 }3962 return .{ .bp = ctx.Rbp, .ip = ctx.Rip, .sp = ctx.Rsp };
39443963 }
3945 pub fn setIp(ctx: *CONTEXT, ip: usize) void {
3946 ctx.Rip = ip;
3947 }
3948
3949 pub fn setSp(ctx: *CONTEXT, sp: usize) void {
3950 ctx.Rsp = sp;
3951 }
3952 };
39533964
3954 pub const RUNTIME_FUNCTION = extern struct {3965 pub fn setIp(ctx: *CONTEXT, ip: usize) void {
3955 BeginAddress: DWORD,3966 ctx.Rip = ip;
3956 EndAddress: DWORD,3967 }
3957 UnwindData: DWORD,
3958 };
39593968
3960 pub const KNONVOLATILE_CONTEXT_POINTERS = extern struct {3969 pub fn setSp(ctx: *CONTEXT, sp: usize) void {
3961 FloatingContext: [16]?*M128A,3970 ctx.Rsp = sp;
3962 IntegerContext: [16]?*ULONG64,3971 }
3963 };
3964 },3972 },
3965 .aarch64 => struct {3973 .aarch64 => extern struct {
3966 pub const NEON128 = extern union {3974 ContextFlags: ULONG align(16),
3975 Cpsr: ULONG,
3976 DUMMYUNIONNAME: extern union {
3967 DUMMYSTRUCTNAME: extern struct {3977 DUMMYSTRUCTNAME: extern struct {
3968 Low: ULONGLONG,3978 X0: DWORD64,
3969 High: LONGLONG,3979 X1: DWORD64,
3970 },3980 X2: DWORD64,
3971 D: [2]f64,3981 X3: DWORD64,
3972 S: [4]f32,3982 X4: DWORD64,
3973 H: [8]WORD,3983 X5: DWORD64,
3974 B: [16]BYTE,3984 X6: DWORD64,
3975 };3985 X7: DWORD64,
39763986 X8: DWORD64,
3977 pub const CONTEXT = extern struct {3987 X9: DWORD64,
3978 ContextFlags: ULONG align(16),3988 X10: DWORD64,
3979 Cpsr: ULONG,3989 X11: DWORD64,
3980 DUMMYUNIONNAME: extern union {3990 X12: DWORD64,
3981 DUMMYSTRUCTNAME: extern struct {3991 X13: DWORD64,
3982 X0: DWORD64,3992 X14: DWORD64,
3983 X1: DWORD64,3993 X15: DWORD64,
3984 X2: DWORD64,3994 X16: DWORD64,
3985 X3: DWORD64,3995 X17: DWORD64,
3986 X4: DWORD64,3996 X18: DWORD64,
3987 X5: DWORD64,3997 X19: DWORD64,
3988 X6: DWORD64,3998 X20: DWORD64,
3989 X7: DWORD64,3999 X21: DWORD64,
3990 X8: DWORD64,4000 X22: DWORD64,
3991 X9: DWORD64,4001 X23: DWORD64,
3992 X10: DWORD64,4002 X24: DWORD64,
3993 X11: DWORD64,4003 X25: DWORD64,
3994 X12: DWORD64,4004 X26: DWORD64,
3995 X13: DWORD64,4005 X27: DWORD64,
3996 X14: DWORD64,4006 X28: DWORD64,
3997 X15: DWORD64,4007 Fp: DWORD64,
3998 X16: DWORD64,4008 Lr: DWORD64,
3999 X17: DWORD64,
4000 X18: DWORD64,
4001 X19: DWORD64,
4002 X20: DWORD64,
4003 X21: DWORD64,
4004 X22: DWORD64,
4005 X23: DWORD64,
4006 X24: DWORD64,
4007 X25: DWORD64,
4008 X26: DWORD64,
4009 X27: DWORD64,
4010 X28: DWORD64,
4011 Fp: DWORD64,
4012 Lr: DWORD64,
4013 },
4014 X: [31]DWORD64,
4015 },4009 },
4016 Sp: DWORD64,4010 X: [31]DWORD64,
4017 Pc: DWORD64,4011 },
4018 V: [32]NEON128,4012 Sp: DWORD64,
4019 Fpcr: DWORD,4013 Pc: DWORD64,
4020 Fpsr: DWORD,4014 V: [32]NEON128,
4021 Bcr: [8]DWORD,4015 Fpcr: DWORD,
4022 Bvr: [8]DWORD64,4016 Fpsr: DWORD,
4023 Wcr: [2]DWORD,4017 Bcr: [8]DWORD,
4024 Wvr: [2]DWORD64,4018 Bvr: [8]DWORD64,
40254019 Wcr: [2]DWORD,
4026 pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize, sp: usize } {4020 Wvr: [2]DWORD64,
4027 return .{4021
4028 .bp = ctx.DUMMYUNIONNAME.DUMMYSTRUCTNAME.Fp,4022 pub fn getRegs(ctx: *const CONTEXT) struct { bp: usize, ip: usize, sp: usize } {
4029 .ip = ctx.Pc,4023 return .{
4030 .sp = ctx.Sp,4024 .bp = ctx.DUMMYUNIONNAME.DUMMYSTRUCTNAME.Fp,
4031 };4025 .ip = ctx.Pc,
4032 }4026 .sp = ctx.Sp,
4027 };
4028 }
40334029
4034 pub fn setIp(ctx: *CONTEXT, ip: usize) void {4030 pub fn setIp(ctx: *CONTEXT, ip: usize) void {
4035 ctx.Pc = ip;4031 ctx.Pc = ip;
4036 }4032 }
40374033
4038 pub fn setSp(ctx: *CONTEXT, sp: usize) void {4034 pub fn setSp(ctx: *CONTEXT, sp: usize) void {
4039 ctx.Sp = sp;4035 ctx.Sp = sp;
4040 }4036 }
4041 };4037 },
4038 else => @compileError("CONTEXT is not defined for this architecture"),
4039};
40424040
4043 pub const RUNTIME_FUNCTION = extern struct {4041pub const RUNTIME_FUNCTION = switch (native_arch) {
4044 BeginAddress: DWORD,4042 .x86_64 => extern struct {
4045 DUMMYUNIONNAME: extern union {4043 BeginAddress: DWORD,
4046 UnwindData: DWORD,4044 EndAddress: DWORD,
4047 DUMMYSTRUCTNAME: packed struct {4045 UnwindData: DWORD,
4048 Flag: u2,4046 },
4049 FunctionLength: u11,4047 .aarch64 => extern struct {
4050 RegF: u3,4048 BeginAddress: DWORD,
4051 RegI: u4,4049 DUMMYUNIONNAME: extern union {
4052 H: u1,4050 UnwindData: DWORD,
4053 CR: u2,4051 DUMMYSTRUCTNAME: packed struct {
4054 FrameSize: u9,4052 Flag: u2,
4055 },4053 FunctionLength: u11,
4054 RegF: u3,
4055 RegI: u4,
4056 H: u1,
4057 CR: u2,
4058 FrameSize: u9,
4056 },4059 },
4057 };4060 },
4061 },
4062 else => @compileError("RUNTIME_FUNCTION is not defined for this architecture"),
4063};
40584064
4059 pub const KNONVOLATILE_CONTEXT_POINTERS = extern struct {4065pub const KNONVOLATILE_CONTEXT_POINTERS = switch (native_arch) {
4060 X19: ?*DWORD64,4066 .x86_64 => extern struct {
4061 X20: ?*DWORD64,4067 FloatingContext: [16]?*M128A,
4062 X21: ?*DWORD64,4068 IntegerContext: [16]?*ULONG64,
4063 X22: ?*DWORD64,4069 },
4064 X23: ?*DWORD64,4070 .aarch64 => extern struct {
4065 X24: ?*DWORD64,4071 X19: ?*DWORD64,
4066 X25: ?*DWORD64,4072 X20: ?*DWORD64,
4067 X26: ?*DWORD64,4073 X21: ?*DWORD64,
4068 X27: ?*DWORD64,4074 X22: ?*DWORD64,
4069 X28: ?*DWORD64,4075 X23: ?*DWORD64,
4070 Fp: ?*DWORD64,4076 X24: ?*DWORD64,
4071 Lr: ?*DWORD64,4077 X25: ?*DWORD64,
4072 D8: ?*DWORD64,4078 X26: ?*DWORD64,
4073 D9: ?*DWORD64,4079 X27: ?*DWORD64,
4074 D10: ?*DWORD64,4080 X28: ?*DWORD64,
4075 D11: ?*DWORD64,4081 Fp: ?*DWORD64,
4076 D12: ?*DWORD64,4082 Lr: ?*DWORD64,
4077 D13: ?*DWORD64,4083 D8: ?*DWORD64,
4078 D14: ?*DWORD64,4084 D9: ?*DWORD64,
4079 D15: ?*DWORD64,4085 D10: ?*DWORD64,
4080 };4086 D11: ?*DWORD64,
4087 D12: ?*DWORD64,
4088 D13: ?*DWORD64,
4089 D14: ?*DWORD64,
4090 D15: ?*DWORD64,
4081 },4091 },
4082 else => struct {},4092 else => @compileError("KNONVOLATILE_CONTEXT_POINTERS is not defined for this architecture"),
4083};4093};
40844094
4085pub const EXCEPTION_POINTERS = extern struct {4095pub const EXCEPTION_POINTERS = extern struct {