authorgravatar for tgschultz@gmail.comtgschultz <tgschultz@gmail.com> 2020-06-11 18:42:56+00:00
committergravatar for tgschultz@gmail.comtgschultz <tgschultz@gmail.com> 2020-06-16 16:20:58+00:00
log8978fe94cfe08d4140bd968b1910348136b4b77d
tree3ed3a04950071666925397249077922af97ba3b3
parentf595545c10a35b85879edfa3c002ce308ffeb6c2

Overhauled leb128:

handles integers < 8 bits incorrect overflow bugs fixed simplified *mem implementations added wrte* functions added thurough write/read testing

1 files changed, 216 insertions(+), 104 deletions(-)

lib/std/debug/leb128.zig+216-104
...@@ -1,171 +1,198 @@...@@ -1,171 +1,198 @@
1const std = @import("std");1const std = @import("std");
2const testing = std.testing;2const testing = std.testing;
33
4pub fn readULEB128(comptime T: type, in_stream: var) !T {4//@TODO: you can take *slice and alter slice.ptr
5 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));5// make sign bits check more efficient
6// add wrapper readLEB128 and write LEB128 that infer from type?
7// or use assertions?
68
7 var result: T = 0;9pub fn readULEB128(comptime T: type, reader: var) !T {
8 var shift: usize = 0;10 const U = if (T.bit_count < 8) u8 else T;
11 const ShiftT = std.math.Log2Int(U);
912
10 while (true) {13 const max_group = (U.bit_count + 6) / 7;
11 const byte = try in_stream.readByte();
12
13 if (shift > T.bit_count)
14 return error.Overflow;
1514
16 var operand: T = undefined;15 var value = @as(U, 0);
17 if (@shlWithOverflow(T, byte & 0x7f, @intCast(ShiftT, shift), &operand))16 var group = @as(ShiftT, 0);
18 return error.Overflow;
1917
20 result |= operand;18 while (group < max_group) : (group += 1) {
19 const byte = try reader.readByte();
20 var temp = @as(U, byte & 0x7f);
2121
22 if ((byte & 0x80) == 0)22 if (@shlWithOverflow(U, temp, group * 7, &temp)) return error.Overflow;
23 return result;
2423
25 shift += 7;24 value |= temp;
25 if (byte & 0x80 == 0) break;
26 } else {
27 return error.Overflow;
26 }28 }
27}
2829
29pub fn readULEB128Mem(comptime T: type, ptr: *[*]const u8) !T {30 //only applies in the case that we extended to u8
30 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));31 if (value > std.math.maxInt(T)) return error.Overflow;
31
32 var result: T = 0;
33 var shift: usize = 0;
34 var i: usize = 0;
3532
36 while (true) : (i += 1) {33 return @truncate(T, value);
37 const byte = ptr.*[i];34}
38
39 if (shift > T.bit_count)
40 return error.Overflow;
41
42 var operand: T = undefined;
43 if (@shlWithOverflow(T, byte & 0x7f, @intCast(ShiftT, shift), &operand))
44 return error.Overflow;
4535
46 result |= operand;36pub fn writeULEB128(writer: var, uint_value: var) !void {
37 const T = @TypeOf(uint_value);
38 const U = if (T.bit_count < 8) u8 else T;
39 var value = @intCast(U, uint_value);
4740
48 if ((byte & 0x80) == 0) {41 while (true) {
49 ptr.* += i + 1;42 const byte = @truncate(u8, value & 0x7f);
50 return result;43 value >>= 7;
44 if (value == 0) {
45 try writer.writeByte(byte);
46 break;
47 } else {
48 try writer.writeByte(byte | 0x80);
51 }49 }
52
53 shift += 7;
54 }50 }
55}51}
5652
57pub fn readILEB128(comptime T: type, in_stream: var) !T {53pub fn readULEB128Mem(comptime T: type, ptr: *[*]const u8) !T {
58 const UT = std.meta.Int(false, T.bit_count);54 const max_group = (T.bit_count + 6) / 7;
59 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));55 var buf = std.io.fixedBufferStream(ptr.*[0 .. max_group + 1]);
56 const value = try readULEB128(T, buf.reader());
57 ptr.* += @intCast(usize, try buf.getPos());
58 return value;
59}
6060
61 var result: UT = 0;61pub fn writeULEB128Mem(ptr: []u8, uint_value: var) !usize {
62 var shift: usize = 0;62 const T = @TypeOf(uint_value);
63 const max_group = (T.bit_count + 6) / 7;
64 var buf = std.io.fixedBufferStream(ptr);
65 try writeULEB128(buf.writer(), uint_value);
66 return try buf.getPos();
67}
6368
64 while (true) {69pub fn readILEB128(comptime T: type, reader: var) !T {
65 const byte: u8 = try in_stream.readByte();70 const S = if (T.bit_count < 8) i8 else T;
71 const U = std.meta.Int(false, S.bit_count);
72 const ShiftU = std.math.Log2Int(U);
6673
67 if (shift > T.bit_count)74 const max_group = (U.bit_count + 6) / 7;
68 return error.Overflow;
6975
70 var operand: UT = undefined;76 var value = @as(U, 0);
71 if (@shlWithOverflow(UT, @as(UT, byte & 0x7f), @intCast(ShiftT, shift), &operand)) {77 var group = @as(ShiftU, 0);
72 if (byte != 0x7f)
73 return error.Overflow;
74 }
7578
76 result |= operand;79 while (group < max_group) : (group += 1) {
80 const byte = try reader.readByte();
81 var temp = @as(U, byte & 0x7f);
7782
78 shift += 7;83 if (@shlWithOverflow(U, temp, group * 7, &temp)) {
84 //Overflow is ok so long as the sign bit is set and this is the last byte
85 if (byte & 0x80 != 0) return error.Overflow;
86 if (@bitCast(S, temp) >= 0) return error.Overflow;
7987
80 if ((byte & 0x80) == 0) {88 //and all the overflowed bits are 1
81 if (shift < T.bit_count and (byte & 0x40) != 0) {89 const check_bits_shift = @intCast(u3, U.bit_count - @as(u16, group * 7));
82 result |= @bitCast(UT, @intCast(T, -1)) << @intCast(ShiftT, shift);90 const check_bits_remaining = 7 - check_bits_shift;
91 const check_bits = byte >> check_bits_shift;
92 const num_consecutive_ones = @ctz(u8, ~check_bits);
93 if (num_consecutive_ones < check_bits_remaining) return error.Overflow;
94 }
95
96 value |= temp;
97 if (byte & 0x80 == 0) {
98 if (byte & 0x40 != 0 and group + 1 < max_group) {
99 value |= @bitCast(U, @as(S, -1)) << ((group + 1) * 7);
83 }100 }
84 return @bitCast(T, result);101 break;
85 }102 }
103 } else {
104 return error.Overflow;
86 }105 }
87}
88106
89pub fn readILEB128Mem(comptime T: type, ptr: *[*]const u8) !T {107 //Only applies if we extended to i8
90 const UT = std.meta.Int(false, T.bit_count);108 if (@bitCast(S, value) > std.math.maxInt(T) or @bitCast(S, value) < std.math.minInt(T)) return error.Overflow;
91 const ShiftT = std.meta.Int(false, std.math.log2(T.bit_count));
92109
93 var result: UT = 0;110 return @truncate(T, @bitCast(S, value));
94 var shift: usize = 0;111}
95 var i: usize = 0;
96112
97 while (true) : (i += 1) {113pub fn writeILEB128(writer: var, int_value: var) !void {
98 const byte = ptr.*[i];114 const T = @TypeOf(int_value);
115 const S = if (T.bit_count < 8) i8 else T;
116 const U = std.meta.Int(false, S.bit_count);
99117
100 if (shift > T.bit_count)118 var value = @intCast(S, int_value);
101 return error.Overflow;
102119
103 var operand: UT = undefined;120 while (true) {
104 if (@shlWithOverflow(UT, @as(UT, byte & 0x7f), @intCast(ShiftT, shift), &operand)) {121 const uvalue = @bitCast(U, value);
105 if (byte != 0x7f)122 const byte = @truncate(u8, uvalue);
106 return error.Overflow;123 value >>= 6;
124 if (value == -1 or value == 0) {
125 try writer.writeByte(byte & 0x7F);
126 break;
127 } else {
128 value >>= 1;
129 try writer.writeByte(byte | 0x80);
107 }130 }
131 }
132}
108133
109 result |= operand;134pub fn readILEB128Mem(comptime T: type, ptr: *[*]const u8) !T {
110135 const max_group = (T.bit_count + 6) / 7;
111 shift += 7;136 var buf = std.io.fixedBufferStream(ptr.*[0 .. max_group + 1]);
137 const value = try readILEB128(T, buf.reader());
138 ptr.* += @intCast(usize, try buf.getPos());
139 return value;
140}
112141
113 if ((byte & 0x80) == 0) {142pub fn writeILEB128Mem(ptr: []u8, int_value: var) !usize {
114 if (shift < T.bit_count and (byte & 0x40) != 0) {143 const T = @TypeOf(int_value);
115 result |= @bitCast(UT, @intCast(T, -1)) << @intCast(ShiftT, shift);144 var buf = std.io.fixedBufferStream(ptr);
116 }145 try writeILEB128(buf.writer(), int_value);
117 ptr.* += i + 1;146 return try buf.getPos();
118 return @bitCast(T, result);
119 }
120 }
121}147}
122148
149//tests
123fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {150fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {
124 var in_stream = std.io.fixedBufferStream(encoded);151 var reader = std.io.fixedBufferStream(encoded);
125 return try readILEB128(T, in_stream.inStream());152 return try readILEB128(T, reader.reader());
126}153}
127154
128fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {155fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {
129 var in_stream = std.io.fixedBufferStream(encoded);156 var reader = std.io.fixedBufferStream(encoded);
130 return try readULEB128(T, in_stream.inStream());157 return try readULEB128(T, reader.reader());
131}158}
132159
133fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {160fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {
134 var in_stream = std.io.fixedBufferStream(encoded);161 var reader = std.io.fixedBufferStream(encoded);
135 const v1 = readILEB128(T, in_stream.inStream());162 const v1 = try readILEB128(T, reader.reader());
136 var in_ptr = encoded.ptr;163 var in_ptr = encoded.ptr;
137 const v2 = readILEB128Mem(T, &in_ptr);164 const v2 = try readILEB128Mem(T, &in_ptr);
138 testing.expectEqual(v1, v2);165 testing.expectEqual(v1, v2);
139 return v1;166 return v1;
140}167}
141168
142fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {169fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {
143 var in_stream = std.io.fixedBufferStream(encoded);170 var reader = std.io.fixedBufferStream(encoded);
144 const v1 = readULEB128(T, in_stream.inStream());171 const v1 = try readULEB128(T, reader.reader());
145 var in_ptr = encoded.ptr;172 var in_ptr = encoded.ptr;
146 const v2 = readULEB128Mem(T, &in_ptr);173 const v2 = try readULEB128Mem(T, &in_ptr);
147 testing.expectEqual(v1, v2);174 testing.expectEqual(v1, v2);
148 return v1;175 return v1;
149}176}
150177
151fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) void {178fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
152 var in_stream = std.io.fixedBufferStream(encoded);179 var reader = std.io.fixedBufferStream(encoded);
153 var in_ptr = encoded.ptr;180 var in_ptr = encoded.ptr;
154 var i: usize = 0;181 var i: usize = 0;
155 while (i < N) : (i += 1) {182 while (i < N) : (i += 1) {
156 const v1 = readILEB128(T, in_stream.inStream());183 const v1 = try readILEB128(T, reader.reader());
157 const v2 = readILEB128Mem(T, &in_ptr);184 const v2 = try readILEB128Mem(T, &in_ptr);
158 testing.expectEqual(v1, v2);185 testing.expectEqual(v1, v2);
159 }186 }
160}187}
161188
162fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) void {189fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
163 var in_stream = std.io.fixedBufferStream(encoded);190 var reader = std.io.fixedBufferStream(encoded);
164 var in_ptr = encoded.ptr;191 var in_ptr = encoded.ptr;
165 var i: usize = 0;192 var i: usize = 0;
166 while (i < N) : (i += 1) {193 while (i < N) : (i += 1) {
167 const v1 = readULEB128(T, in_stream.inStream());194 const v1 = try readULEB128(T, reader.reader());
168 const v2 = readULEB128Mem(T, &in_ptr);195 const v2 = try readULEB128Mem(T, &in_ptr);
169 testing.expectEqual(v1, v2);196 testing.expectEqual(v1, v2);
170 }197 }
171}198}
...@@ -212,7 +239,7 @@ test "deserialize signed LEB128" {...@@ -212,7 +239,7 @@ test "deserialize signed LEB128" {
212 testing.expect((try test_read_ileb128(i64, "\x80\x81\x80\x00")) == 0x80);239 testing.expect((try test_read_ileb128(i64, "\x80\x81\x80\x00")) == 0x80);
213240
214 // Decode sequence of SLEB128 values241 // Decode sequence of SLEB128 values
215 test_read_ileb128_seq(i64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");242 try test_read_ileb128_seq(i64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");
216}243}
217244
218test "deserialize unsigned LEB128" {245test "deserialize unsigned LEB128" {
...@@ -252,5 +279,90 @@ test "deserialize unsigned LEB128" {...@@ -252,5 +279,90 @@ test "deserialize unsigned LEB128" {
252 testing.expect((try test_read_uleb128(u64, "\x80\x81\x80\x00")) == 0x80);279 testing.expect((try test_read_uleb128(u64, "\x80\x81\x80\x00")) == 0x80);
253280
254 // Decode sequence of ULEB128 values281 // Decode sequence of ULEB128 values
255 test_read_uleb128_seq(u64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");282 try test_read_uleb128_seq(u64, 4, "\x81\x01\x3f\x80\x7f\x80\x80\x80\x00");
283}
284
285fn test_write_leb128(value: var) !void {
286 const T = @TypeOf(value);
287
288 if (T.bit_count == 0) std.debug.warn("{}\n", .{@typeName(T)});
289
290 const writeStream = if (T.is_signed) writeILEB128 else writeULEB128;
291 const writeMem = if (T.is_signed) writeILEB128Mem else writeULEB128Mem;
292 const readStream = if (T.is_signed) readILEB128 else readULEB128;
293 const readMem = if (T.is_signed) readILEB128Mem else readULEB128Mem;
294
295 //decode to a larger bit size too, to ensure sign extension
296 // is working as expected
297 const larger_type_bits = ((T.bit_count + 8) / 8) * 8;
298 const B = std.meta.Int(T.is_signed, larger_type_bits);
299 const max_groups = if (T.bit_count == 0) 1 else (T.bit_count + 6) / 7;
300
301 var buf: [max_groups]u8 = undefined;
302 var fbs = std.io.fixedBufferStream(&buf);
303
304 //stream write
305 try writeStream(fbs.writer(), value);
306 const w1_pos = fbs.pos;
307 testing.expect(w1_pos > 0);
308
309 //stream read
310 fbs.pos = 0;
311 const sr = try readStream(T, fbs.reader());
312 testing.expect(fbs.pos == w1_pos);
313 testing.expect(sr == value);
314
315 //bigger type stream read
316 fbs.pos = 0;
317 const bsr = try readStream(B, fbs.reader());
318 testing.expect(fbs.pos == w1_pos);
319 testing.expect(bsr == value);
320
321 //mem write
322 const w2_pos = try writeMem(&buf, value);
323 testing.expect(w2_pos == w1_pos);
324
325 //mem read
326 var buf_ref: []u8 = buf[0..];
327 const mr = try readMem(T, &buf_ref.ptr);
328 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);
329 testing.expect(mr == value);
330
331 //bigger type mem read
332 buf_ref = buf[0..];
333 const bmr = try readMem(T, &buf_ref.ptr);
334 testing.expect(@ptrToInt(buf_ref.ptr) - @ptrToInt(&buf) == w2_pos);
335 testing.expect(bmr == value);
336}
337
338test "serialize unsigned LEB128" {
339 const max_bits = 18;
340
341 comptime var t = 0;
342 inline while (t <= max_bits) : (t += 1) {
343 const T = std.meta.Int(false, t);
344 const min = std.math.minInt(T);
345 const max = std.math.maxInt(T);
346 var i = @as(std.meta.Int(false, T.bit_count + 1), min);
347
348 while (i <= max) : (i += 1) try test_write_leb128(@intCast(T, i));
349 }
350}
351
352test "serialize signed LEB128" {
353 //explicitly test i0 because starting `t` at 0
354 // will break the while loop
355 try test_write_leb128(@as(i0, 0));
356
357 const max_bits = 18;
358
359 comptime var t = 1;
360 inline while (t <= max_bits) : (t += 1) {
361 const T = std.meta.Int(true, t);
362 const min = std.math.minInt(T);
363 const max = std.math.maxInt(T);
364 var i = @as(std.meta.Int(true, T.bit_count + 1), min);
365
366 while (i <= max) : (i += 1) try test_write_leb128(@intCast(T, i));
367 }
256}368}