authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-27 20:43:13+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-27 20:43:13+01:00
log6bf9499c0c5566962b76a767667212e5e20544d6
tree2c2791ba5eaf97b090a2d5c634aa75f173a8526a
parentef77cc0de1f4453d89d8841116eaa65a42a9c89f
parent03a3269d48db03f1998166f63ad5195cbba2dd88

Merge pull request 'Optimize {write,take}Leb128' (#30012) from NicoElbers/zig:leb-perf into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/30012 Reviewed-by: Andrew Kelley <andrewrk@noreply.codeberg.org>

2 files changed, 469 insertions(+), 119 deletions(-)

lib/std/Io/Reader.zig+319-96
...@@ -1271,38 +1271,113 @@ pub fn takeEnumNonexhaustive(r: *Reader, comptime Enum: type, endian: std.builti...@@ -1271,38 +1271,113 @@ pub fn takeEnumNonexhaustive(r: *Reader, comptime Enum: type, endian: std.builti
1271pub const TakeLeb128Error = Error || error{Overflow};1271pub const TakeLeb128Error = Error || error{Overflow};
12721272
1273/// Read a single LEB128 value as type T, or `error.Overflow` if the value cannot fit.1273/// Read a single LEB128 value as type T, or `error.Overflow` if the value cannot fit.
1274pub fn takeLeb128(r: *Reader, comptime Result: type) TakeLeb128Error!Result {1274pub fn takeLeb128(r: *Reader, comptime T: type) TakeLeb128Error!T {
1275 const result_info = @typeInfo(Result).int;1275 const info = switch (@typeInfo(T)) {
1276 return std.math.cast(Result, try r.takeMultipleOf7Leb128(@Int(1276 .int => |info| info,
1277 result_info.signedness,1277 else => @compileError(@typeName(T) ++ " not supported"),
1278 std.mem.alignForwardAnyAlign(u16, result_info.bits, 7),1278 };
1279 ))) orelse error.Overflow;1279 const Byte = packed struct { bits: u7, more: bool };
1280}1280
12811281 if (info.bits <= 7) {
1282fn takeMultipleOf7Leb128(r: *Reader, comptime Result: type) TakeLeb128Error!Result {1282 var byte: Byte = undefined;
1283 const result_info = @typeInfo(Result).int;1283 const Bits = @Int(info.signedness, 7);
1284 comptime assert(result_info.bits % 7 == 0);1284
1285 var remaining_bits: std.math.Log2IntCeil(Result) = result_info.bits;1285 byte = @bitCast(try r.takeByte());
1286 const UnsignedResult = @Int(.unsigned, result_info.bits);1286 const val = std.math.cast(T, @as(Bits, @bitCast(byte.bits))) orelse error.Overflow;
1287 var result: UnsignedResult = 0;1287
1288 var fits = true;1288 const allowed_bits: u7 = switch (info.signedness) {
1289 while (true) {1289 .unsigned => 0,
1290 const buffer: []const packed struct(u8) { bits: u7, more: bool } = @ptrCast(try r.peekGreedy(1));1290 .signed => @bitCast(@as(i7, @bitCast(byte.bits)) >> 6),
1291 for (buffer, 1..) |byte, len| {1291 };
1292 if (remaining_bits > 0) {1292
1293 result = @shlExact(@as(UnsignedResult, byte.bits), result_info.bits - 7) |1293 var fits = true;
1294 if (result_info.bits > 7) @shrExact(result, 7) else 0;1294 while (byte.more) {
1295 remaining_bits -= 7;1295 byte = @bitCast(try r.takeByte());
1296 } else if (fits) fits = switch (result_info.signedness) {1296
1297 .signed => @as(i7, @bitCast(byte.bits)) ==1297 if (byte.bits != allowed_bits) fits = false;
1298 @as(i7, @truncate(@as(Result, @bitCast(result)) >> (result_info.bits - 1))),1298 }
1299 .unsigned => byte.bits == 0,1299
1300 return if (fits) blk: {
1301 @branchHint(.likely);
1302 break :blk val;
1303 } else error.Overflow;
1304 }
1305
1306 const Unsigned = @Int(.unsigned, info.bits);
1307 const UInt = std.math.ByteAlignedInt(Unsigned);
1308 const Int = std.math.ByteAlignedInt(T);
1309
1310 const uint_bits = @typeInfo(UInt).int.bits;
1311
1312 var byte: Byte = undefined;
1313 var val: UInt = 0;
1314 const max_bytes = @divFloor(info.bits - 1, 7) + 1;
1315 inline for (0..max_bytes) |iteration| {
1316 const shift = iteration * 7;
1317
1318 byte = @bitCast(try r.takeByte());
1319
1320 const extended: UInt = byte.bits;
1321 val |= extended << shift;
1322
1323 const bits_written = shift + 7;
1324
1325 if (bits_written >= info.bits) {
1326 const bits_overflowed = bits_written - info.bits;
1327 const bits_remaining = @mod(info.bits, 7);
1328
1329 const allowed_bits: u7, var fits: bool = switch (info.signedness) {
1330 .unsigned => blk: {
1331 const fits = bits_remaining == 0 or byte.bits >> bits_remaining == 0;
1332
1333 break :blk .{ 0, fits };
1334 },
1335 .signed => blk: {
1336 const bits: i7 = @bitCast(byte.bits);
1337
1338 // Move the sign bit into the MSB
1339 const shifted_bits: i7 = bits << bits_overflowed;
1340
1341 const value_sign: i7 = shifted_bits >> 6; // sign extends
1342 const bits_sign: i7 = bits >> bits_remaining; // sign extends
1343
1344 const fits = bits_remaining == 0 or bits_sign == value_sign;
1345
1346 if (uint_bits != info.bits and value_sign != 0) {
1347 const sign_extend_mask = @as(UInt, std.math.maxInt(UInt)) << info.bits;
1348 val |= sign_extend_mask;
1349 }
1350
1351 break :blk .{ @bitCast(value_sign), fits };
1352 },
1300 };1353 };
1301 if (byte.more) continue;1354
1302 r.toss(len);1355 switch (info.signedness) {
1303 return if (fits) @as(Result, @bitCast(result)) >> remaining_bits else error.Overflow;1356 .signed => assert(allowed_bits == 0 or allowed_bits == 0x7F),
1357 .unsigned => comptime assert(allowed_bits == 0),
1358 }
1359
1360 while (byte.more) {
1361 byte = @bitCast(try r.takeByte());
1362 if (byte.bits != allowed_bits) fits = false;
1363 }
1364
1365 return if (fits) blk: {
1366 @branchHint(.likely);
1367 break :blk std.math.cast(T, @as(Int, @bitCast(val))) orelse error.Overflow;
1368 } else error.Overflow;
1369 }
1370
1371 comptime assert(bits_written < info.bits);
1372 if (!byte.more) {
1373 if (info.signedness == .signed and // can be negative
1374 byte.bits & 0x40 != 0) // is negative
1375 {
1376 const sign_extend_mask = @as(UInt, std.math.maxInt(UInt)) << bits_written;
1377 val |= sign_extend_mask;
1378 }
1379 return std.math.cast(T, @as(Int, @bitCast(val))) orelse error.Overflow;
1304 }1380 }
1305 r.toss(buffer.len);
1306 }1381 }
1307}1382}
13081383
...@@ -1610,13 +1685,6 @@ test takeEnum {...@@ -1610,13 +1685,6 @@ test takeEnum {
1610 try testing.expectEqual(@as(E2, @enumFromInt(0x0001)), try r.takeEnum(E2, .big));1685 try testing.expectEqual(@as(E2, @enumFromInt(0x0001)), try r.takeEnum(E2, .big));
1611}1686}
16121687
1613test takeLeb128 {
1614 var r: Reader = .fixed("\xc7\x9f\x7f\x80");
1615 try testing.expectEqual(-12345, try r.takeLeb128(i64));
1616 try testing.expectEqual(0x80, try r.peekByte());
1617 try testing.expectError(error.EndOfStream, r.takeLeb128(i64));
1618}
1619
1620test readSliceShort {1688test readSliceShort {
1621 var r: Reader = .fixed("HelloFren");1689 var r: Reader = .fixed("HelloFren");
1622 var buf: [5]u8 = undefined;1690 var buf: [5]u8 = undefined;
...@@ -1978,90 +2046,245 @@ pub fn writableVector(r: *Reader, buffer: [][]u8, data: []const []u8) Error!stru...@@ -1978,90 +2046,245 @@ pub fn writableVector(r: *Reader, buffer: [][]u8, data: []const []u8) Error!stru
1978}2046}
19792047
1980test "deserialize signed LEB128" {2048test "deserialize signed LEB128" {
2049 // Small values
2050 try testing.expectEqual(5, testLeb128(i7, "\x05"));
2051 try testing.expectEqual(53, testLeb128(i64, "\x35"));
2052
2053 try testing.expectEqual(-6, testLeb128(i7, "\x7A"));
2054 try testing.expectEqual(-23, testLeb128(i64, "\x69"));
2055
2056 // Random values
2057 try testing.expectEqual(90, testLeb128(i8, "\xDA\x00"));
2058 try testing.expectEqual(3434, testLeb128(i16, "\xEA\x1A"));
2059 try testing.expectEqual(1505683543, testLeb128(i32, "\xD7\xD0\xFB\xCD\x05"));
2060 try testing.expectEqual(105721575804011595, testLeb128(i64, "\xCB\x88\x92\xD7\xE8\xA6\xE6\xBB\x01"));
2061 try testing.expectEqual(51316697993548595875823294343650416388, testLeb128(i128, "\x84\xAE\xAC\xFC\xE4\xA0\xCD\xE2\x87\xED\x83\xB2\x87\xAA\xA3\x9E\x9B\xCD\x00"));
2062
2063 try testing.expectEqual(-68, testLeb128(i8, "\xBC\x7F"));
2064 try testing.expectEqual(-20174, testLeb128(i16, "\xB2\xE2\x7E"));
2065 try testing.expectEqual(-166511141, testLeb128(i32, "\xDB\xFB\xCC\xB0\x7F"));
2066 try testing.expectEqual(-4368809844285451825, testLeb128(i64, "\xCF\xA3\x8B\xA1\xFF\xD2\xB7\xAF\x43"));
2067 try testing.expectEqual(-43250117698642799010758201165100952046, testLeb128(i128, "\x92\xAC\xDB\xA4\xEC\xDE\xB9\x95\xD1\xBA\xEC\xB0\xD7\x80\xA4\xAA\xF6\xBE\x7F"));
2068
2069 // {min,max} values
2070 try testing.expectEqual(std.math.maxInt(i8), testLeb128(i8, "\xFF\x00"));
2071 try testing.expectEqual(std.math.maxInt(i16), testLeb128(i16, "\xFF\xFF\x01"));
2072 try testing.expectEqual(std.math.maxInt(i32), testLeb128(i32, "\xFF\xFF\xFF\xFF\x07"));
2073 try testing.expectEqual(std.math.maxInt(i64), testLeb128(i64, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00"));
2074 try testing.expectEqual(std.math.maxInt(i128), testLeb128(i128, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"));
2075
2076 try testing.expectEqual(std.math.minInt(i8), testLeb128(i8, "\x80\x7F"));
2077 try testing.expectEqual(std.math.minInt(i16), testLeb128(i16, "\x80\x80\x7E"));
2078 try testing.expectEqual(std.math.minInt(i32), testLeb128(i32, "\x80\x80\x80\x80\x78"));
2079 try testing.expectEqual(std.math.minInt(i64), testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x7F"));
2080 try testing.expectEqual(std.math.minInt(i128), testLeb128(i128, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x7E"));
2081
2082 // Specific cases
2083 try testing.expectEqual(0, testLeb128(i0, "\x00"));
2084 try testing.expectEqual(0, testLeb128(i2, "\x00"));
2085 try testing.expectEqual(0, testLeb128(i8, "\x00"));
2086
2087 try testing.expectEqual(1, testLeb128(i2, "\x01"));
2088 try testing.expectEqual(1, testLeb128(i8, "\x01"));
2089
2090 try testing.expectEqual(-1, testLeb128(i2, "\x7F"));
2091 try testing.expectEqual(-1, testLeb128(i8, "\x7F"));
2092
2093 const end_of_stream: [20]u8 = @splat(0x80);
2094 const overflow: [21]u8 = end_of_stream ++ .{0x01};
2095 const long_zero: [21]u8 = end_of_stream ++ .{0x00};
2096 const long_one: [22]u8 = .{0x81} ++ end_of_stream ++ .{0x00};
2097 const long_minus_one: [20]u8 = @as([19]u8, @splat(0xFF)) ++ .{0x7F};
2098
1981 // Truncated2099 // Truncated
1982 try testing.expectError(error.EndOfStream, testLeb128(i64, "\x80"));2100 try testing.expectError(error.EndOfStream, testLeb128(i16, "\x80\x80\x84\x80"));
2101 try testing.expectError(error.EndOfStream, testLeb128(i16, "\x80\x80\x80\x84\x80"));
2102 try testing.expectError(error.EndOfStream, testLeb128(i32, "\x80\x80\x80\x80\x90"));
2103
2104 try testing.expectError(error.EndOfStream, testLeb128(i7, ""));
2105 try testing.expectError(error.EndOfStream, testLeb128(i8, ""));
2106 try testing.expectError(error.EndOfStream, testLeb128(i14, ""));
2107 try testing.expectError(error.EndOfStream, testLeb128(i128, ""));
2108
2109 try testing.expectError(error.EndOfStream, testLeb128(i7, "\x80"));
2110 try testing.expectError(error.EndOfStream, testLeb128(i8, "\x80"));
2111 try testing.expectError(error.EndOfStream, testLeb128(i14, "\x80"));
2112 try testing.expectError(error.EndOfStream, testLeb128(i128, "\x80"));
2113
2114 try testing.expectError(error.EndOfStream, testLeb128(i7, &end_of_stream));
2115 try testing.expectError(error.EndOfStream, testLeb128(i8, &end_of_stream));
2116 try testing.expectError(error.EndOfStream, testLeb128(i14, &end_of_stream));
2117 try testing.expectError(error.EndOfStream, testLeb128(i128, &end_of_stream));
19832118
1984 // Overflow2119 // Overflow
2120 try testing.expectError(error.Overflow, testLeb128(i0, "\x01"));
2121 try testing.expectError(error.Overflow, testLeb128(i0, "\x7F"));
2122 try testing.expectError(error.Overflow, testLeb128(i8, "\x80\x01"));
2123 try testing.expectError(error.Overflow, testLeb128(i8, "\xFF\x7E"));
1985 try testing.expectError(error.Overflow, testLeb128(i8, "\x80\x80\x40"));2124 try testing.expectError(error.Overflow, testLeb128(i8, "\x80\x80\x40"));
1986 try testing.expectError(error.Overflow, testLeb128(i16, "\x80\x80\x80\x40"));2125 try testing.expectError(error.Overflow, testLeb128(i16, "\x80\x80\x80\x40"));
1987 try testing.expectError(error.Overflow, testLeb128(i32, "\x80\x80\x80\x80\x40"));
1988 try testing.expectError(error.Overflow, testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x40"));
1989 try testing.expectError(error.Overflow, testLeb128(i8, "\xff\x7e"));
1990 try testing.expectError(error.Overflow, testLeb128(i32, "\x80\x80\x80\x80\x08"));2126 try testing.expectError(error.Overflow, testLeb128(i32, "\x80\x80\x80\x80\x08"));
2127 try testing.expectError(error.Overflow, testLeb128(i32, "\x80\x80\x80\x80\x40"));
1991 try testing.expectError(error.Overflow, testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01"));2128 try testing.expectError(error.Overflow, testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01"));
2129 try testing.expectError(error.Overflow, testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x40"));
19922130
1993 // Decode SLEB1282131 try testing.expectError(error.Overflow, testLeb128(i0, &overflow));
1994 try testing.expect((try testLeb128(i64, "\x00")) == 0);2132 try testing.expectError(error.Overflow, testLeb128(i7, &overflow));
1995 try testing.expect((try testLeb128(i64, "\x01")) == 1);2133 try testing.expectError(error.Overflow, testLeb128(i8, &overflow));
1996 try testing.expect((try testLeb128(i64, "\x3f")) == 63);2134 try testing.expectError(error.Overflow, testLeb128(i14, &overflow));
1997 try testing.expect((try testLeb128(i64, "\x40")) == -64);2135 try testing.expectError(error.Overflow, testLeb128(i128, &overflow));
1998 try testing.expect((try testLeb128(i64, "\x41")) == -63);2136
1999 try testing.expect((try testLeb128(i64, "\x7f")) == -1);2137 // Extra padding
2000 try testing.expect((try testLeb128(i64, "\x80\x01")) == 128);2138 try testing.expectEqual(-1, testLeb128(i32, "\xFF\xFF\xFF\xFF\x7F"));
2001 try testing.expect((try testLeb128(i64, "\x81\x01")) == 129);2139 try testing.expectEqual(-1, testLeb128(i64, "\xFF\x7F"));
2002 try testing.expect((try testLeb128(i64, "\xff\x7e")) == -129);2140 try testing.expectEqual(0x7F, testLeb128(i64, "\xFF\x00"));
2003 try testing.expect((try testLeb128(i64, "\x80\x7f")) == -128);2141 try testing.expectEqual(0x7F, testLeb128(i64, "\xFF\x80\x00"));
2004 try testing.expect((try testLeb128(i64, "\x81\x7f")) == -127);2142 try testing.expectEqual(0x80, testLeb128(i64, "\x80\x81\x00"));
2005 try testing.expect((try testLeb128(i64, "\xc0\x00")) == 64);2143 try testing.expectEqual(0x80, testLeb128(i64, "\x80\x81\x80\x00"));
2006 try testing.expect((try testLeb128(i64, "\xc7\x9f\x7f")) == -12345);2144
2007 try testing.expect((try testLeb128(i8, "\xff\x7f")) == -1);2145 try testing.expectEqual(0, testLeb128(i0, &long_zero));
2008 try testing.expect((try testLeb128(i16, "\xff\xff\x7f")) == -1);2146 try testing.expectEqual(0, testLeb128(i7, &long_zero));
2009 try testing.expect((try testLeb128(i32, "\xff\xff\xff\xff\x7f")) == -1);2147 try testing.expectEqual(0, testLeb128(i8, &long_zero));
2010 try testing.expect((try testLeb128(i32, "\x80\x80\x80\x80\x78")) == -0x80000000);2148 try testing.expectEqual(0, testLeb128(i14, &long_zero));
2011 try testing.expect((try testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x7f")) == @as(i64, @bitCast(@as(u64, @intCast(0x8000000000000000)))));2149 try testing.expectEqual(0, testLeb128(i128, &long_zero));
2012 try testing.expect((try testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x40")) == -0x4000000000000000);2150
2013 try testing.expect((try testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x7f")) == -0x8000000000000000);2151 try testing.expectEqual(1, testLeb128(i2, &long_one));
20142152 try testing.expectEqual(1, testLeb128(i7, &long_one));
2015 // Decode unnormalized SLEB128 with extra padding bytes.2153 try testing.expectEqual(1, testLeb128(i8, &long_one));
2016 try testing.expect((try testLeb128(i64, "\x80\x00")) == 0);2154 try testing.expectEqual(1, testLeb128(i14, &long_one));
2017 try testing.expect((try testLeb128(i64, "\x80\x80\x00")) == 0);2155 try testing.expectEqual(1, testLeb128(i128, &long_one));
2018 try testing.expect((try testLeb128(i64, "\xff\x00")) == 0x7f);2156
2019 try testing.expect((try testLeb128(i64, "\xff\x80\x00")) == 0x7f);2157 try testing.expectEqual(-1, testLeb128(i2, &long_minus_one));
2020 try testing.expect((try testLeb128(i64, "\x80\x81\x00")) == 0x80);2158 try testing.expectEqual(-1, testLeb128(i7, &long_minus_one));
2021 try testing.expect((try testLeb128(i64, "\x80\x81\x80\x00")) == 0x80);2159 try testing.expectEqual(-1, testLeb128(i8, &long_minus_one));
2160 try testing.expectEqual(-1, testLeb128(i14, &long_minus_one));
2161 try testing.expectEqual(-1, testLeb128(i128, &long_minus_one));
2162
2163 // Decode byte boundaries
2164 try testing.expectEqual(std.math.maxInt(i7), testLeb128(i7, "\x3F"));
2165 try testing.expectEqual(std.math.maxInt(i7) + 1, testLeb128(i8, "\xC0\x00"));
2166 try testing.expectEqual(std.math.maxInt(i14), testLeb128(i14, "\xFF\x3F"));
2167 try testing.expectEqual(std.math.maxInt(i14) + 1, testLeb128(i15, "\x80\xC0\x00"));
2168 try testing.expectEqual(std.math.maxInt(i49), testLeb128(i49, "\xFF\xFF\xFF\xFF\xFF\xFF\x3F"));
2169 try testing.expectEqual(std.math.maxInt(i49) + 1, testLeb128(i50, "\x80\x80\x80\x80\x80\x80\xC0\x00"));
2170 try testing.expectEqual(std.math.maxInt(i56), testLeb128(i56, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x3F"));
2171 try testing.expectEqual(std.math.maxInt(i56) + 1, testLeb128(i57, "\x80\x80\x80\x80\x80\x80\x80\xC0\x00"));
2172 try testing.expectEqual(std.math.maxInt(i63), testLeb128(i63, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x3F"));
2173 try testing.expectEqual(std.math.maxInt(i63) + 1, testLeb128(i64, "\x80\x80\x80\x80\x80\x80\x80\x80\xC0\x00"));
2174
2175 try testing.expectEqual(std.math.minInt(i7), testLeb128(i7, "\x40"));
2176 try testing.expectEqual(std.math.minInt(i7) - 1, testLeb128(i8, "\xBF\x7F"));
2177 try testing.expectEqual(std.math.minInt(i14), testLeb128(i14, "\x80\x40"));
2178 try testing.expectEqual(std.math.minInt(i14) - 1, testLeb128(i15, "\xFF\xBF\x7F"));
2179 try testing.expectEqual(std.math.minInt(i49), testLeb128(i49, "\x80\x80\x80\x80\x80\x80\x40"));
2180 try testing.expectEqual(std.math.minInt(i49) - 1, testLeb128(i50, "\xFF\xFF\xFF\xFF\xFF\xFF\xBF\x7F"));
2181 try testing.expectEqual(std.math.minInt(i56), testLeb128(i56, "\x80\x80\x80\x80\x80\x80\x80\x40"));
2182 try testing.expectEqual(std.math.minInt(i56) - 1, testLeb128(i57, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xBF\x7F"));
2183 try testing.expectEqual(std.math.minInt(i63), testLeb128(i63, "\x80\x80\x80\x80\x80\x80\x80\x80\x40"));
2184 try testing.expectEqual(std.math.minInt(i63) - 1, testLeb128(i64, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xBF\x7F"));
2022}2185}
20232186
2024test "deserialize unsigned LEB128" {2187test "deserialize unsigned LEB128" {
2188 // Small values
2189 try testing.expectEqual(46, testLeb128(u7, "\x2E"));
2190 try testing.expectEqual(117, testLeb128(u64, "\x75"));
2191
2192 // Random values
2193 try testing.expectEqual(224, testLeb128(u8, "\xE0\x01"));
2194 try testing.expectEqual(53023, testLeb128(u16, "\x9F\x9E\x03"));
2195 try testing.expectEqual(2609971022, testLeb128(u32, "\xCE\xFE\xC3\xDC\x09"));
2196 try testing.expectEqual(10223253173206528843, testLeb128(u64, "\xCB\xE6\xF0\xEE\x88\xD3\x92\xF0\x8D\x01"));
2197 try testing.expectEqual(67831258924174241363439488509570048548, testLeb128(u128, "\xA4\xC4\xD7\xE9\x8C\xD2\x86\x80\xBC\xAC\xE5\xAB\xB4\xA2\xD1\xE9\x87\x66"));
2198
2199 // max values
2200 try testing.expectEqual(std.math.maxInt(u8), testLeb128(u8, "\xFF\x01"));
2201 try testing.expectEqual(std.math.maxInt(u16), testLeb128(u16, "\xFF\xFF\x03"));
2202 try testing.expectEqual(std.math.maxInt(u32), testLeb128(u32, "\xFF\xFF\xFF\xFF\x0F"));
2203 try testing.expectEqual(std.math.maxInt(u64), testLeb128(u64, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"));
2204 try testing.expectEqual(std.math.maxInt(u128), testLeb128(u128, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x03"));
2205
2206 // Specific cases
2207 try testing.expectEqual(0, testLeb128(u0, "\x00"));
2208 try testing.expectEqual(0, testLeb128(u1, "\x00"));
2209 try testing.expectEqual(0, testLeb128(u8, "\x00"));
2210
2211 try testing.expectEqual(1, testLeb128(u1, "\x01"));
2212 try testing.expectEqual(1, testLeb128(u8, "\x01"));
2213
2214 const end_of_stream: [20]u8 = @splat(0x80);
2215 const overflow: [21]u8 = end_of_stream ++ .{0x01};
2216 const long_zero: [21]u8 = end_of_stream ++ .{0x00};
2217 const long_one: [22]u8 = .{0x81} ++ end_of_stream ++ .{0x00};
2218
2025 // Truncated2219 // Truncated
2026 try testing.expectError(error.EndOfStream, testLeb128(u64, "\x80"));2220 try testing.expectError(error.EndOfStream, testLeb128(u16, "\x80\x80\x84\x80"));
2027 try testing.expectError(error.EndOfStream, testLeb128(u16, "\x80\x80\x84"));2221 try testing.expectError(error.EndOfStream, testLeb128(u16, "\x80\x80\x80\x84\x80"));
2028 try testing.expectError(error.EndOfStream, testLeb128(u32, "\x80\x80\x80\x80\x90"));2222 try testing.expectError(error.EndOfStream, testLeb128(u32, "\x80\x80\x80\x80\x90"));
20292223
2224 try testing.expectError(error.EndOfStream, testLeb128(u7, ""));
2225 try testing.expectError(error.EndOfStream, testLeb128(u8, ""));
2226 try testing.expectError(error.EndOfStream, testLeb128(u14, ""));
2227 try testing.expectError(error.EndOfStream, testLeb128(u128, ""));
2228
2229 try testing.expectError(error.EndOfStream, testLeb128(u7, "\x80"));
2230 try testing.expectError(error.EndOfStream, testLeb128(u8, "\x80"));
2231 try testing.expectError(error.EndOfStream, testLeb128(u14, "\x80"));
2232 try testing.expectError(error.EndOfStream, testLeb128(u128, "\x80"));
2233
2234 try testing.expectError(error.EndOfStream, testLeb128(u7, &end_of_stream));
2235 try testing.expectError(error.EndOfStream, testLeb128(u8, &end_of_stream));
2236 try testing.expectError(error.EndOfStream, testLeb128(u14, &end_of_stream));
2237 try testing.expectError(error.EndOfStream, testLeb128(u128, &end_of_stream));
2238
2030 // Overflow2239 // Overflow
2240 try testing.expectError(error.Overflow, testLeb128(u0, "\x01"));
2241 try testing.expectError(error.Overflow, testLeb128(u1, "\x02"));
2031 try testing.expectError(error.Overflow, testLeb128(u8, "\x80\x02"));2242 try testing.expectError(error.Overflow, testLeb128(u8, "\x80\x02"));
2032 try testing.expectError(error.Overflow, testLeb128(u8, "\x80\x80\x40"));2243 try testing.expectError(error.Overflow, testLeb128(u8, "\x80\x80\x40"));
2033 try testing.expectError(error.Overflow, testLeb128(u16, "\x80\x80\x80\x40"));2244 try testing.expectError(error.Overflow, testLeb128(u16, "\x80\x80\x80\x40"));
2034 try testing.expectError(error.Overflow, testLeb128(u32, "\x80\x80\x80\x80\x40"));2245 try testing.expectError(error.Overflow, testLeb128(u32, "\x80\x80\x80\x80\x40"));
2035 try testing.expectError(error.Overflow, testLeb128(u64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x40"));2246 try testing.expectError(error.Overflow, testLeb128(u64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x40"));
20362247
2037 // Decode ULEB1282248 try testing.expectError(error.Overflow, testLeb128(u7, &overflow));
2038 try testing.expect((try testLeb128(u64, "\x00")) == 0);2249 try testing.expectError(error.Overflow, testLeb128(u8, &overflow));
2039 try testing.expect((try testLeb128(u64, "\x01")) == 1);2250 try testing.expectError(error.Overflow, testLeb128(u14, &overflow));
2040 try testing.expect((try testLeb128(u64, "\x3f")) == 63);2251 try testing.expectError(error.Overflow, testLeb128(u128, &overflow));
2041 try testing.expect((try testLeb128(u64, "\x40")) == 64);2252
2042 try testing.expect((try testLeb128(u64, "\x7f")) == 0x7f);2253 // Extra padding
2043 try testing.expect((try testLeb128(u64, "\x80\x01")) == 0x80);2254 try testing.expectEqual(0x7F, testLeb128(u64, "\xFF\x00"));
2044 try testing.expect((try testLeb128(u64, "\x81\x01")) == 0x81);2255 try testing.expectEqual(0x7F, testLeb128(u64, "\xFF\x80\x00"));
2045 try testing.expect((try testLeb128(u64, "\x90\x01")) == 0x90);2256 try testing.expectEqual(0x80, testLeb128(u64, "\x80\x81\x00"));
2046 try testing.expect((try testLeb128(u64, "\xff\x01")) == 0xff);2257 try testing.expectEqual(0x80, testLeb128(u64, "\x80\x81\x80\x80\x00"));
2047 try testing.expect((try testLeb128(u64, "\x80\x02")) == 0x100);2258
2048 try testing.expect((try testLeb128(u64, "\x81\x02")) == 0x101);2259 try testing.expectEqual(0, testLeb128(u0, &long_zero));
2049 try testing.expect((try testLeb128(u64, "\x80\xc1\x80\x80\x10")) == 4294975616);2260 try testing.expectEqual(0, testLeb128(u7, &long_zero));
2050 try testing.expect((try testLeb128(u64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01")) == 0x8000000000000000);2261 try testing.expectEqual(0, testLeb128(u8, &long_zero));
20512262 try testing.expectEqual(0, testLeb128(u14, &long_zero));
2052 // Decode ULEB128 with extra padding bytes2263 try testing.expectEqual(0, testLeb128(u128, &long_zero));
2053 try testing.expect((try testLeb128(u64, "\x80\x00")) == 0);2264
2054 try testing.expect((try testLeb128(u64, "\x80\x80\x00")) == 0);2265 try testing.expectEqual(1, testLeb128(u1, &long_one));
2055 try testing.expect((try testLeb128(u64, "\xff\x00")) == 0x7f);2266 try testing.expectEqual(1, testLeb128(u7, &long_one));
2056 try testing.expect((try testLeb128(u64, "\xff\x80\x00")) == 0x7f);2267 try testing.expectEqual(1, testLeb128(u8, &long_one));
2057 try testing.expect((try testLeb128(u64, "\x80\x81\x00")) == 0x80);2268 try testing.expectEqual(1, testLeb128(u14, &long_one));
2058 try testing.expect((try testLeb128(u64, "\x80\x81\x80\x00")) == 0x80);2269 try testing.expectEqual(1, testLeb128(u128, &long_one));
2270
2271 // Decode byte boundaries
2272 try testing.expectEqual(std.math.maxInt(u7), testLeb128(u7, "\x7F"));
2273 try testing.expectEqual(std.math.maxInt(u7) + 1, testLeb128(u8, "\x80\x01"));
2274 try testing.expectEqual(std.math.maxInt(u14), testLeb128(u14, "\xFF\x7F"));
2275 try testing.expectEqual(std.math.maxInt(u14) + 1, testLeb128(u15, "\x80\x80\x01"));
2276 try testing.expectEqual(std.math.maxInt(u49), testLeb128(u49, "\xFF\xFF\xFF\xFF\xFF\xFF\x7F"));
2277 try testing.expectEqual(std.math.maxInt(u49) + 1, testLeb128(u50, "\x80\x80\x80\x80\x80\x80\x80\x01"));
2278 try testing.expectEqual(std.math.maxInt(u56), testLeb128(u56, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"));
2279 try testing.expectEqual(std.math.maxInt(u56) + 1, testLeb128(u57, "\x80\x80\x80\x80\x80\x80\x80\x80\x01"));
2280 try testing.expectEqual(std.math.maxInt(u63), testLeb128(u63, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"));
2281 try testing.expectEqual(std.math.maxInt(u63) + 1, testLeb128(u64, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01"));
2059}2282}
20602283
2061fn testLeb128(comptime T: type, encoded: []const u8) !T {2284fn testLeb128(comptime T: type, encoded: []const u8) !T {
2062 var reader: std.Io.Reader = .fixed(encoded);2285 var reader: std.Io.Reader = .fixed(encoded);
2063 const result = try reader.takeLeb128(T);2286 const result = reader.takeLeb128(T);
2064 try testing.expect(reader.seek == reader.end);2287 try testing.expectEqual(reader.seek, reader.end);
2065 return result;2288 return result;
2066}2289}
20672290
lib/std/Io/Writer.zig+150-23
...@@ -1908,35 +1908,162 @@ pub fn writeSleb128(w: *Writer, value: anytype) Error!void {...@@ -1908,35 +1908,162 @@ pub fn writeSleb128(w: *Writer, value: anytype) Error!void {
19081908
1909/// Write a single integer as LEB128 to the given writer.1909/// Write a single integer as LEB128 to the given writer.
1910pub fn writeLeb128(w: *Writer, value: anytype) Error!void {1910pub fn writeLeb128(w: *Writer, value: anytype) Error!void {
1911 const value_info = @typeInfo(@TypeOf(value)).int;1911 const T = @TypeOf(value);
1912 try w.writeMultipleOf7Leb128(@as(@Int(1912 const info = switch (@typeInfo(T)) {
1913 value_info.signedness,1913 .int => |info| info,
1914 @max(std.mem.alignForwardAnyAlign(u16, value_info.bits, 7), 7),1914 else => @compileError(@typeName(T) ++ " not supported"),
1915 ), value));1915 };
1916}
19171916
1918fn writeMultipleOf7Leb128(w: *Writer, value: anytype) Error!void {1917 const BoundInt = @Int(info.signedness, 7);
1919 const value_info = @typeInfo(@TypeOf(value)).int;1918 if (info.bits <= 7 or (value >= std.math.minInt(BoundInt) and value <= std.math.maxInt(BoundInt))) {
1920 const Byte = packed struct(u8) { bits: u7, more: bool };1919 const Bits = @Int(info.signedness, 8);
1921 var bytes: [@divExact(value_info.bits, 7)]Byte = undefined;1920 const byte = switch (info.signedness) {
1922 var remaining = value;1921 .signed => @as(Bits, @intCast(value)) & 0x7F,
1923 for (&bytes, 1..) |*byte, len| {1922 .unsigned => @as(Bits, @intCast(value)),
1924 const more = switch (value_info.signedness) {
1925 .signed => remaining >> 6 != remaining >> (value_info.bits - 1),
1926 .unsigned => remaining > std.math.maxInt(u7),
1927 };1923 };
1928 byte.* = .{1924 try w.writeByte(@bitCast(byte));
1929 .bits = @bitCast(@as(1925 return;
1930 @Int(value_info.signedness, 7),1926 }
1931 @truncate(remaining),1927
1932 )),1928 const Byte = packed struct { bits: u7, more: bool };
1933 .more = more,1929 const Int = std.math.ByteAlignedInt(T);
1930
1931 const max_bytes = @divFloor(info.bits - 1, 7) + 1;
1932
1933 const sign_value = value >> (info.bits - 1);
1934 var val: Int = value;
1935 for (0..max_bytes) |_| {
1936 const more = switch (info.signedness) {
1937 .signed => val >> 6 != sign_value,
1938 .unsigned => val > std.math.maxInt(u7),
1934 };1939 };
1935 if (value_info.bits > 7) remaining >>= 7;1940
1936 if (!more) return w.writeAll(@ptrCast(bytes[0..len]));1941 try w.writeByte(@bitCast(@as(Byte, .{
1942 .bits = @intCast(val & 0x7F),
1943 .more = more,
1944 })));
1945
1946 if (!more) return;
1947
1948 val >>= 7;
1937 } else unreachable;1949 } else unreachable;
1938}1950}
19391951
1952test "serialize signed LEB128" {
1953 // Small values
1954 try testLeb128Encoding(i7, 9, "\x09");
1955 try testLeb128Encoding(i64, 125, "\xFD\x00");
1956
1957 try testLeb128Encoding(i7, -34, "\x5E");
1958 try testLeb128Encoding(i64, -3, "\x7D");
1959
1960 // Random values
1961 try testLeb128Encoding(i16, 19373, "\xAD\x97\x01");
1962 try testLeb128Encoding(i32, 1628839242, "\xCA\xBA\xD8\x88\x06");
1963 try testLeb128Encoding(i64, 3789169920125966546, "\xD2\xB1\xD0\xD5\xF6\xBE\xF5\xCA\x34");
1964 try testLeb128Encoding(i128, 704622239050934257305893323522763588, "\xC4\xD6\x83\xC7\xE3\x91\x95\xC3\x96\x80\x8D\xA5\xF5\xDF\xA3\xDA\x87\x01");
1965
1966 try testLeb128Encoding(i16, -14558, "\xA2\x8E\x7F");
1967 try testLeb128Encoding(i32, -1702738165, "\x8B\x8E\x89\xD4\x79");
1968 try testLeb128Encoding(i64, -1709126996960612298, "\xB6\xE0\x87\xB1\xD3\xC1\xFD\xA3\x68");
1969 try testLeb128Encoding(i128, -113498719181566012704681230050325944039, "\x99\xD2\x80\xBC\xE6\x95\xBC\xC8\xDE\xB4\x9D\x81\x9F\xCA\xC6\xF8\x9C\xD5\x7E");
1970
1971 // {min,max} values
1972 try testLeb128Encoding(i16, std.math.maxInt(i16), "\xFF\xFF\x01");
1973 try testLeb128Encoding(i32, std.math.maxInt(i32), "\xFF\xFF\xFF\xFF\x07");
1974 try testLeb128Encoding(i64, std.math.maxInt(i64), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x00");
1975 try testLeb128Encoding(i128, std.math.maxInt(i128), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01");
1976
1977 try testLeb128Encoding(i16, std.math.minInt(i16), "\x80\x80\x7E");
1978 try testLeb128Encoding(i32, std.math.minInt(i32), "\x80\x80\x80\x80\x78");
1979 try testLeb128Encoding(i64, std.math.minInt(i64), "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x7F");
1980 try testLeb128Encoding(i128, std.math.minInt(i128), "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x7E");
1981
1982 // Specific cases
1983 try testLeb128Encoding(i0, 0, "\x00");
1984 try testLeb128Encoding(i8, 0, "\x00");
1985
1986 try testLeb128Encoding(i2, -1, "\x7F");
1987 try testLeb128Encoding(i8, -1, "\x7F");
1988
1989 try testLeb128Encoding(i2, 1, "\x01");
1990 try testLeb128Encoding(i8, 1, "\x01");
1991
1992 // Encode byte boundaries
1993 try testLeb128Encoding(i7, std.math.maxInt(i7), "\x3F");
1994 try testLeb128Encoding(i8, std.math.maxInt(i7) + 1, "\xC0\x00");
1995 try testLeb128Encoding(i14, std.math.maxInt(i14), "\xFF\x3F");
1996 try testLeb128Encoding(i15, std.math.maxInt(i14) + 1, "\x80\xC0\x00");
1997 try testLeb128Encoding(i49, std.math.maxInt(i49), "\xFF\xFF\xFF\xFF\xFF\xFF\x3F");
1998 try testLeb128Encoding(i50, std.math.maxInt(i49) + 1, "\x80\x80\x80\x80\x80\x80\xC0\x00");
1999 try testLeb128Encoding(i56, std.math.maxInt(i56), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x3F");
2000 try testLeb128Encoding(i57, std.math.maxInt(i56) + 1, "\x80\x80\x80\x80\x80\x80\x80\xC0\x00");
2001 try testLeb128Encoding(i63, std.math.maxInt(i63), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x3F");
2002 try testLeb128Encoding(i64, std.math.maxInt(i63) + 1, "\x80\x80\x80\x80\x80\x80\x80\x80\xC0\x00");
2003
2004 try testLeb128Encoding(i7, std.math.minInt(i7), "\x40");
2005 try testLeb128Encoding(i8, std.math.minInt(i7) - 1, "\xBF\x7F");
2006 try testLeb128Encoding(i14, std.math.minInt(i14), "\x80\x40");
2007 try testLeb128Encoding(i15, std.math.minInt(i14) - 1, "\xFF\xBF\x7F");
2008 try testLeb128Encoding(i49, std.math.minInt(i49), "\x80\x80\x80\x80\x80\x80\x40");
2009 try testLeb128Encoding(i50, std.math.minInt(i49) - 1, "\xFF\xFF\xFF\xFF\xFF\xFF\xBF\x7F");
2010 try testLeb128Encoding(i56, std.math.minInt(i56), "\x80\x80\x80\x80\x80\x80\x80\x40");
2011 try testLeb128Encoding(i57, std.math.minInt(i56) - 1, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xBF\x7F");
2012 try testLeb128Encoding(i63, std.math.minInt(i63), "\x80\x80\x80\x80\x80\x80\x80\x80\x40");
2013 try testLeb128Encoding(i64, std.math.minInt(i63) - 1, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xBF\x7F");
2014}
2015
2016test "serialize unsigned LEB128" {
2017 // Small values
2018 try testLeb128Encoding(u7, 12, "\x0C");
2019 try testLeb128Encoding(u64, 201, "\xC9\x01");
2020
2021 // Random values
2022 try testLeb128Encoding(u8, 254, "\xFE\x01");
2023 try testLeb128Encoding(u16, 30241, "\xA1\xEC\x01");
2024 try testLeb128Encoding(u32, 2173531193, "\xB9\xE8\xB5\x8C\x08");
2025 try testLeb128Encoding(u64, 18321125691115744902, "\x86\xDD\xF2\x81\xF2\xD7\xED\xA0\xFE\x01");
2026 try testLeb128Encoding(u128, 122619209508942982841456325819614676193, "\xE1\x89\xF3\xD9\xE3\xAD\xEC\xF4\x98\x95\xF8\xBB\xD7\xB8\xF2\xCC\xBF\xB8\x01");
2027
2028 // Max values
2029 try testLeb128Encoding(u8, std.math.maxInt(u8), "\xFF\x01");
2030 try testLeb128Encoding(u16, std.math.maxInt(u16), "\xFF\xFF\x03");
2031 try testLeb128Encoding(u32, std.math.maxInt(u32), "\xFF\xFF\xFF\xFF\x0F");
2032 try testLeb128Encoding(u64, std.math.maxInt(u64), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01");
2033 try testLeb128Encoding(u128, std.math.maxInt(u128), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x03");
2034
2035 // Specific cases
2036 try testLeb128Encoding(u0, 0, "\x00");
2037 try testLeb128Encoding(u1, 0, "\x00");
2038 try testLeb128Encoding(u8, 0, "\x00");
2039
2040 try testLeb128Encoding(u1, 1, "\x01");
2041 try testLeb128Encoding(u8, 1, "\x01");
2042
2043 // Encode byte boundaries
2044 try testLeb128Encoding(u7, std.math.maxInt(u7), "\x7F");
2045 try testLeb128Encoding(u8, std.math.maxInt(u7) + 1, "\x80\x01");
2046 try testLeb128Encoding(u14, std.math.maxInt(u14), "\xFF\x7F");
2047 try testLeb128Encoding(u15, std.math.maxInt(u14) + 1, "\x80\x80\x01");
2048 try testLeb128Encoding(u49, std.math.maxInt(u49), "\xFF\xFF\xFF\xFF\xFF\xFF\x7F");
2049 try testLeb128Encoding(u50, std.math.maxInt(u49) + 1, "\x80\x80\x80\x80\x80\x80\x80\x01");
2050 try testLeb128Encoding(u56, std.math.maxInt(u56), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F");
2051 try testLeb128Encoding(u57, std.math.maxInt(u56) + 1, "\x80\x80\x80\x80\x80\x80\x80\x80\x01");
2052 try testLeb128Encoding(u63, std.math.maxInt(u63), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F");
2053 try testLeb128Encoding(u64, std.math.maxInt(u63) + 1, "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x01");
2054}
2055
2056fn testLeb128Encoding(comptime T: type, value: T, encoding: []const u8) !void {
2057 const info = @typeInfo(T).int;
2058 const max_bytes = @divFloor(info.bits -| 1, 7) + 1;
2059 var bytes: [max_bytes]u8 = undefined;
2060
2061 var fw: Writer = .fixed(&bytes);
2062 try writeLeb128(&fw, value);
2063
2064 try std.testing.expectEqualSlices(u8, encoding, fw.buffered());
2065}
2066
1940test "printValue max_depth" {2067test "printValue max_depth" {
1941 const Vec2 = struct {2068 const Vec2 = struct {
1942 const SelfType = @This();2069 const SelfType = @This();