authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-25 19:10:31-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-06-25 19:10:31-04:00
loge820678ca1c1a66b561e760fb6aae4df4babf8ba
tree81bafbcb350e36ce8bd329bc132b4fbdd881feaa
parent061c8be049508368fba8f9c466fad38ae826c77e
parent38e69a9e6a6d69a48d130382dcb61ba2516863d0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5588 from tgschultz/leb128-output

LEB128 overhaul and output

1 files changed, 243 insertions(+), 109 deletions(-)

lib/std/debug/leb128.zig+243-109
...@@ -1,171 +1,211 @@...@@ -1,171 +1,211 @@
1const std = @import("std");1const std = @import("std");
2const testing = std.testing;2const testing = std.testing;
33
4pub fn readULEB128(comptime T: type, in_stream: var) !T {4/// Read a single unsigned LEB128 value from the given reader as type T,
5 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));5/// or error.Overflow if the value cannot fit.
6pub fn readULEB128(comptime T: type, reader: var) !T {
7 const U = if (T.bit_count < 8) u8 else T;
8 const ShiftT = std.math.Log2Int(U);
69
7 var result: T = 0;10 const max_group = (U.bit_count + 6) / 7;
8 var shift: usize = 0;
911
10 while (true) {12 var value = @as(U, 0);
11 const byte = try in_stream.readByte();13 var group = @as(ShiftT, 0);
12
13 if (shift > T.bit_count)
14 return error.Overflow;
15
16 var operand: T = undefined;
17 if (@shlWithOverflow(T, byte & 0x7f, @intCast(ShiftT, shift), &operand))
18 return error.Overflow;
1914
20 result |= operand;15 while (group < max_group) : (group += 1) {
16 const byte = try reader.readByte();
17 var temp = @as(U, byte & 0x7f);
2118
22 if ((byte & 0x80) == 0)19 if (@shlWithOverflow(U, temp, group * 7, &temp)) return error.Overflow;
23 return result;
2420
25 shift += 7;21 value |= temp;
22 if (byte & 0x80 == 0) break;
23 } else {
24 return error.Overflow;
26 }25 }
27}
28
29pub fn readULEB128Mem(comptime T: type, ptr: *[*]const u8) !T {
30 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));
3126
32 var result: T = 0;27 // only applies in the case that we extended to u8
33 var shift: usize = 0;28 if (U != T) {
34 var i: usize = 0;29 if (value > std.math.maxInt(T)) return error.Overflow;
3530 }
36 while (true) : (i += 1) {
37 const byte = ptr.*[i];
38
39 if (shift > T.bit_count)
40 return error.Overflow;
4131
42 var operand: T = undefined;32 return @truncate(T, value);
43 if (@shlWithOverflow(T, byte & 0x7f, @intCast(ShiftT, shift), &operand))33}
44 return error.Overflow;
4534
46 result |= operand;35/// Write a single unsigned integer as unsigned LEB128 to the given writer.
36pub fn writeULEB128(writer: var, uint_value: var) !void {
37 const T = @TypeOf(uint_value);
38 const U = if (T.bit_count < 8) u8 else T;
39 var value = @intCast(U, uint_value);
4740
48 if ((byte & 0x80) == 0) {41 while (true) {
49 ptr.* += i + 1;42 const byte = @truncate(u8, value & 0x7f);
50 return result;43 value >>= 7;
44 if (value == 0) {
45 try writer.writeByte(byte);
46 break;
47 } else {
48 try writer.writeByte(byte | 0x80);
51 }49 }
52
53 shift += 7;
54 }50 }
55}51}
5652
57pub fn readILEB128(comptime T: type, in_stream: var) !T {53/// Read a single unsinged integer from the given memory as type T.
58 const UT = std.meta.Int(false, T.bit_count);54/// The provided slice reference will be updated to point to the byte after the last byte read.
59 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));55pub fn readULEB128Mem(comptime T: type, ptr: *[]const u8) !T {
56 var buf = std.io.fixedBufferStream(ptr.*);
57 const value = try readULEB128(T, buf.reader());
58 ptr.*.ptr += buf.pos;
59 return value;
60}
6061
61 var result: UT = 0;62/// Write a single unsigned LEB128 integer to the given memory as unsigned LEB128,
62 var shift: usize = 0;63/// returning the number of bytes written.
64pub fn writeULEB128Mem(ptr: []u8, uint_value: var) !usize {
65 const T = @TypeOf(uint_value);
66 const max_group = (T.bit_count + 6) / 7;
67 var buf = std.io.fixedBufferStream(ptr);
68 try writeULEB128(buf.writer(), uint_value);
69 return buf.pos;
70}
6371
64 while (true) {72/// Read a single signed LEB128 value from the given reader as type T,
65 const byte: u8 = try in_stream.readByte();73/// or error.Overflow if the value cannot fit.
74pub fn readILEB128(comptime T: type, reader: var) !T {
75 const S = if (T.bit_count < 8) i8 else T;
76 const U = std.meta.Int(false, S.bit_count);
77 const ShiftU = std.math.Log2Int(U);
6678
67 if (shift > T.bit_count)79 const max_group = (U.bit_count + 6) / 7;
68 return error.Overflow;
6980
70 var operand: UT = undefined;81 var value = @as(U, 0);
71 if (@shlWithOverflow(UT, @as(UT, byte & 0x7f), @intCast(ShiftT, shift), &operand)) {82 var group = @as(ShiftU, 0);
72 if (byte != 0x7f)
73 return error.Overflow;
74 }
7583
76 result |= operand;84 while (group < max_group) : (group += 1) {
85 const byte = try reader.readByte();
86 var temp = @as(U, byte & 0x7f);
7787
78 shift += 7;88 const shift = group * 7;
89 if (@shlWithOverflow(U, temp, shift, &temp)) {
90 // Overflow is ok so long as the sign bit is set and this is the last byte
91 if (byte & 0x80 != 0) return error.Overflow;
92 if (@bitCast(S, temp) >= 0) return error.Overflow;
7993
80 if ((byte & 0x80) == 0) {94 // and all the overflowed bits are 1
81 if (shift < T.bit_count and (byte & 0x40) != 0) {95 const remaining_shift = @intCast(u3, U.bit_count - @as(u16, shift));
82 result |= @bitCast(UT, @intCast(T, -1)) << @intCast(ShiftT, shift);96 const remaining_bits = @bitCast(i8, byte | 0x80) >> remaining_shift;
97 if (remaining_bits != -1) return error.Overflow;
98 }
99
100 value |= temp;
101 if (byte & 0x80 == 0) {
102 const needs_sign_ext = group + 1 < max_group;
103 if (byte & 0x40 != 0 and needs_sign_ext) {
104 const ones = @as(S, -1);
105 value |= @bitCast(U, ones) << (shift + 7);
83 }106 }
84 return @bitCast(T, result);107 break;
85 }108 }
109 } else {
110 return error.Overflow;
86 }111 }
87}
88112
89pub fn readILEB128Mem(comptime T: type, ptr: *[*]const u8) !T {113 const result = @bitCast(S, value);
90 const UT = std.meta.Int(false, T.bit_count);114 // Only applies if we extended to i8
91 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));115 if (S != T) {
116 if (result > std.math.maxInt(T) or result < std.math.minInt(T)) return error.Overflow;
117 }
92118
93 var result: UT = 0;119 return @truncate(T, result);
94 var shift: usize = 0;120}
95 var i: usize = 0;
96121
97 while (true) : (i += 1) {122/// Write a single signed integer as signed LEB128 to the given writer.
98 const byte = ptr.*[i];123pub fn writeILEB128(writer: var, int_value: var) !void {
124 const T = @TypeOf(int_value);
125 const S = if (T.bit_count < 8) i8 else T;
126 const U = std.meta.Int(false, S.bit_count);
99127
100 if (shift > T.bit_count)128 var value = @intCast(S, int_value);
101 return error.Overflow;
102129
103 var operand: UT = undefined;130 while (true) {
104 if (@shlWithOverflow(UT, @as(UT, byte & 0x7f), @intCast(ShiftT, shift), &operand)) {131 const uvalue = @bitCast(U, value);
105 if (byte != 0x7f)132 const byte = @truncate(u8, uvalue);
106 return error.Overflow;133 value >>= 6;
134 if (value == -1 or value == 0) {
135 try writer.writeByte(byte & 0x7F);
136 break;
137 } else {
138 value >>= 1;
139 try writer.writeByte(byte | 0x80);
107 }140 }
141 }
142}
108143
109 result |= operand;144/// Read a single singed LEB128 integer from the given memory as type T.
110145/// The provided slice reference will be updated to point to the byte after the last byte read.
111 shift += 7;146pub fn readILEB128Mem(comptime T: type, ptr: *[]const u8) !T {
147 var buf = std.io.fixedBufferStream(ptr.*);
148 const value = try readILEB128(T, buf.reader());
149 ptr.*.ptr += buf.pos;
150 return value;
151}
112152
113 if ((byte & 0x80) == 0) {153/// Write a single signed LEB128 integer to the given memory as unsigned LEB128,
114 if (shift < T.bit_count and (byte & 0x40) != 0) {154/// returning the number of bytes written.
115 result |= @bitCast(UT, @intCast(T, -1)) << @intCast(ShiftT, shift);155pub fn writeILEB128Mem(ptr: []u8, int_value: var) !usize {
116 }156 const T = @TypeOf(int_value);
117 ptr.* += i + 1;157 var buf = std.io.fixedBufferStream(ptr);
118 return @bitCast(T, result);158 try writeILEB128(buf.writer(), int_value);
119 }159 return buf.pos;
120 }
121}160}
122161
162// tests
123fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {163fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {
124 var in_stream = std.io.fixedBufferStream(encoded);164 var reader = std.io.fixedBufferStream(encoded);
125 return try readILEB128(T, in_stream.inStream());165 return try readILEB128(T, reader.reader());
126}166}
127167
128fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {168fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {
129 var in_stream = std.io.fixedBufferStream(encoded);169 var reader = std.io.fixedBufferStream(encoded);
130 return try readULEB128(T, in_stream.inStream());170 return try readULEB128(T, reader.reader());
131}171}
132172
133fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {173fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {
134 var in_stream = std.io.fixedBufferStream(encoded);174 var reader = std.io.fixedBufferStream(encoded);
135 const v1 = readILEB128(T, in_stream.inStream());175 const v1 = try readILEB128(T, reader.reader());
136 var in_ptr = encoded.ptr;176 var in_ptr = encoded;
137 const v2 = readILEB128Mem(T, &in_ptr);177 const v2 = try readILEB128Mem(T, &in_ptr);
138 testing.expectEqual(v1, v2);178 testing.expectEqual(v1, v2);
139 return v1;179 return v1;
140}180}
141181
142fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {182fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {
143 var in_stream = std.io.fixedBufferStream(encoded);183 var reader = std.io.fixedBufferStream(encoded);
144 const v1 = readULEB128(T, in_stream.inStream());184 const v1 = try readULEB128(T, reader.reader());
145 var in_ptr = encoded.ptr;185 var in_ptr = encoded;
146 const v2 = readULEB128Mem(T, &in_ptr);186 const v2 = try readULEB128Mem(T, &in_ptr);
147 testing.expectEqual(v1, v2);187 testing.expectEqual(v1, v2);
148 return v1;188 return v1;
149}189}
150190
151fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) void {191fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
152 var in_stream = std.io.fixedBufferStream(encoded);192 var reader = std.io.fixedBufferStream(encoded);
153 var in_ptr = encoded.ptr;193 var in_ptr = encoded;
154 var i: usize = 0;194 var i: usize = 0;
155 while (i < N) : (i += 1) {195 while (i < N) : (i += 1) {
156 const v1 = readILEB128(T, in_stream.inStream());196 const v1 = try readILEB128(T, reader.reader());
157 const v2 = readILEB128Mem(T, &in_ptr);197 const v2 = try readILEB128Mem(T, &in_ptr);
158 testing.expectEqual(v1, v2);198 testing.expectEqual(v1, v2);
159 }199 }
160}200}
161201
162fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) void {202fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
163 var in_stream = std.io.fixedBufferStream(encoded);203 var reader = std.io.fixedBufferStream(encoded);
164 var in_ptr = encoded.ptr;204 var in_ptr = encoded;
165 var i: usize = 0;205 var i: usize = 0;
166 while (i < N) : (i += 1) {206 while (i < N) : (i += 1) {
167 const v1 = readULEB128(T, in_stream.inStream());207 const v1 = try readULEB128(T, reader.reader());
168 const v2 = readULEB128Mem(T, &in_ptr);208 const v2 = try readULEB128Mem(T, &in_ptr);
169 testing.expectEqual(v1, v2);209 testing.expectEqual(v1, v2);
170 }210 }
171}211}
...@@ -212,7 +252,7 @@ test "deserialize signed LEB128" {...@@ -212,7 +252,7 @@ test "deserialize signed LEB128" {
212 testing.expect((try test_read_ileb128(i64, "\x80\x81\x80\x00")) == 0x80);252 testing.expect((try test_read_ileb128(i64, "\x80\x81\x80\x00")) == 0x80);
213253
214 // Decode sequence of SLEB128 values254 // Decode sequence of SLEB128 values
215 test_read_ileb128_seq(i64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");255 try test_read_ileb128_seq(i64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");
216}256}
217257
218test "deserialize unsigned LEB128" {258test "deserialize unsigned LEB128" {
...@@ -252,5 +292,99 @@ test "deserialize unsigned LEB128" {...@@ -252,5 +292,99 @@ test "deserialize unsigned LEB128" {
252 testing.expect((try test_read_uleb128(u64, "\x80\x81\x80\x00")) == 0x80);292 testing.expect((try test_read_uleb128(u64, "\x80\x81\x80\x00")) == 0x80);
253293
254 // Decode sequence of ULEB128 values294 // Decode sequence of ULEB128 values
255 test_read_uleb128_seq(u64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");295 try test_read_uleb128_seq(u64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");
296}
297
298fn test_write_leb128(value: var) !void {
299 const T = @TypeOf(value);
300
301 const writeStream = if (T.is_signed) writeILEB128 else writeULEB128;
302 const writeMem = if (T.is_signed) writeILEB128Mem else writeULEB128Mem;
303 const readStream = if (T.is_signed) readILEB128 else readULEB128;
304 const readMem = if (T.is_signed) readILEB128Mem else readULEB128Mem;
305
306 // decode to a larger bit size too, to ensure sign extension
307 // is working as expected
308 const larger_type_bits = ((T.bit_count + 8) / 8) * 8;
309 const B = std.meta.Int(T.is_signed, larger_type_bits);
310
311 const bytes_needed = bn: {
312 const S = std.meta.Int(T.is_signed, @sizeOf(T) * 8);
313 if (T.bit_count <= 7) break :bn @as(u16, 1);
314
315 const unused_bits = if (value < 0) @clz(T, ~value) else @clz(T, value);
316 const used_bits: u16 = (T.bit_count - unused_bits) + @boolToInt(T.is_signed);
317 if (used_bits <= 7) break :bn @as(u16, 1);
318 break :bn ((used_bits + 6) / 7);
319 };
320
321 const max_groups = if (T.bit_count == 0) 1 else (T.bit_count + 6) / 7;
322
323 var buf: [max_groups]u8 = undefined;
324 var fbs = std.io.fixedBufferStream(&buf);
325
326 // stream write
327 try writeStream(fbs.writer(), value);
328 const w1_pos = fbs.pos;
329 testing.expect(w1_pos == bytes_needed);
330
331 // stream read
332 fbs.pos = 0;
333 const sr = try readStream(T, fbs.reader());
334 testing.expect(fbs.pos == w1_pos);
335 testing.expect(sr == value);
336
337 // bigger type stream read
338 fbs.pos = 0;
339 const bsr = try readStream(B, fbs.reader());
340 testing.expect(fbs.pos == w1_pos);
341 testing.expect(bsr == value);
342
343 // mem write
344 const w2_pos = try writeMem(&buf, value);
345 testing.expect(w2_pos == w1_pos);
346
347 // mem read
348 var buf_ref: []u8 = buf[0..];
349 const mr = try readMem(T, &buf_ref);
350 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);
351 testing.expect(mr == value);
352
353 // bigger type mem read
354 buf_ref = buf[0..];
355 const bmr = try readMem(T, &buf_ref);
356 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);
357 testing.expect(bmr == value);
358}
359
360test "serialize unsigned LEB128" {
361 const max_bits = 18;
362
363 comptime var t = 0;
364 inline while (t <= max_bits) : (t += 1) {
365 const T = std.meta.Int(false, t);
366 const min = std.math.minInt(T);
367 const max = std.math.maxInt(T);
368 var i = @as(std.meta.Int(false, T.bit_count + 1), min);
369
370 while (i <= max) : (i += 1) try test_write_leb128(@intCast(T, i));
371 }
372}
373
374test "serialize signed LEB128" {
375 // explicitly test i0 because starting `t` at 0
376 // will break the while loop
377 try test_write_leb128(@as(i0, 0));
378
379 const max_bits = 18;
380
381 comptime var t = 1;
382 inline while (t <= max_bits) : (t += 1) {
383 const T = std.meta.Int(true, t);
384 const min = std.math.minInt(T);
385 const max = std.math.maxInt(T);
386 var i = @as(std.meta.Int(true, T.bit_count + 1), min);
387
388 while (i <= max) : (i += 1) try test_write_leb128(@intCast(T, i));
389 }
256}390}