Commit 7293e012d7956b892380517e914108ffadc6941b | zig

author	Andrew Kelley <andrew@ziglang.org> 2019-02-15 18:05:50-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2019-02-15 18:05:50-05:00
log	7293e012d7956b892380517e914108ffadc6941b
tree	420049c2174484e2c536d2e36b5b1bb906b921b3
parent	567c9b688effdb64e3995df09af4b45105515c2c
signature	Commit is signed but in an unrecognized format.

breaking: fix @sizeOf to be alloc size rather than store size

 * Fixes breaches of the guarantee that `@sizeOf(T) >= @alignOf(T)`
 * Fixes std.mem.secureZero for integers where this guarantee previously
   was breached
 * Fixes std.mem.Allocator for integers where this guarantee previously
   was breached

Closes #1851
Closes #1864

8 files changed, 112 insertions(+), 51 deletions(-)

doc
- langref.html.in
src
std
- io.zig
- mem.zig
test
- stage1
  - behavior.zig
  - behavior
    bugs
    1851.zig

doc/langref.html.in+6-1

...	...	@@ -6299,10 +6299,15 @@ pub const FloatMode = enum {
6299	6299	<pre>{#syntax#}@sizeOf(comptime T: type) comptime_int{#endsyntax#}</pre>
6300	6300	<p>
6301	6301	This function returns the number of bytes it takes to store {#syntax#}T{#endsyntax#} in memory.
	6302	The result is a target-specific compile time constant.
6302	6303	</p>
6303	6304	<p>
6304		The result is a target-specific compile time constant.
	6305	This size may contain padding bytes. If there were two consecutive T in memory, this would be the offset
	6306	in bytes between element at index 0 and the element at index 1. For {#link\|integer\|Integers#},
	6307	consider whether you want to use {#syntax#}@sizeOf(T){#endsyntax#} or
	6308	{#syntax#}@typeInfo(T).Int.bits{#endsyntax#}.
6305	6309	</p>
	6310	{#see_also\|@typeInfo#}
6306	6311	{#header_close#}
6307	6312
6308	6313	{#header_open\|@sliceToBytes#}

src/analyze.cpp+32-5

...	...	@@ -356,6 +356,28 @@ uint64_t type_size(CodeGen g, ZigType type_entry) {
356	356	}
357	357	}
358	358
	359	return LLVMABISizeOfType(g->target_data_ref, type_entry->type_ref);
	360	}
	361
	362	uint64_t type_size_store(CodeGen g, ZigType type_entry) {
	363	assert(type_is_complete(type_entry));
	364
	365	if (!type_has_bits(type_entry))
	366	return 0;
	367
	368	if (type_entry->id == ZigTypeIdStruct && type_entry->data.structure.layout == ContainerLayoutPacked) {
	369	uint64_t size_in_bits = type_size_bits(g, type_entry);
	370	return (size_in_bits + 7) / 8;
	371	} else if (type_entry->id == ZigTypeIdArray) {
	372	ZigType *child_type = type_entry->data.array.child_type;
	373	if (child_type->id == ZigTypeIdStruct &&
	374	child_type->data.structure.layout == ContainerLayoutPacked)
	375	{
	376	uint64_t size_in_bits = type_size_bits(g, type_entry);
	377	return (size_in_bits + 7) / 8;
	378	}
	379	}
	380
359	381	return LLVMStoreSizeOfType(g->target_data_ref, type_entry->type_ref);
360	382	}
361	383
...	...	@@ -6230,14 +6252,19 @@ void render_const_value(CodeGen g, Buf buf, ConstExprValue *const_val) {
6230	6252	case ZigTypeIdStruct:
6231	6253	{
6232	6254	if (is_slice(type_entry)) {
6233		ConstPtrValue *ptr = &const_val->data.x_struct.fields[slice_ptr_index].data.x_ptr;
6234		assert(ptr->special == ConstPtrSpecialBaseArray);
6235		ConstExprValue *array = ptr->data.base_array.array_val;
6236		size_t start = ptr->data.base_array.elem_index;
6237
6238	6255	ConstExprValue *len_val = &const_val->data.x_struct.fields[slice_len_index];
6239	6256	size_t len = bigint_as_unsigned(&len_val->data.x_bigint);
6240	6257
	6258	ConstExprValue *ptr_val = &const_val->data.x_struct.fields[slice_ptr_index];
	6259	if (ptr_val->special == ConstValSpecialUndef) {
	6260	assert(len == 0);
	6261	buf_appendf(buf, "((%s)(undefined))[0..0]", buf_ptr(&type_entry->name));
	6262	return;
	6263	}
	6264	assert(ptr_val->data.x_ptr.special == ConstPtrSpecialBaseArray);
	6265	ConstExprValue *array = ptr_val->data.x_ptr.data.base_array.array_val;
	6266	size_t start = ptr_val->data.x_ptr.data.base_array.elem_index;
	6267
6241	6268	render_const_val_array(g, buf, &type_entry->name, array, start, len);
6242	6269	} else {
6243	6270	buf_appendf(buf, "(struct %s constant)", buf_ptr(&type_entry->name));

src/analyze.hpp+1

...	...	@@ -19,6 +19,7 @@ ZigType get_pointer_to_type(CodeGen g, ZigType *child_type, bool is_const);
19	19	ZigType get_pointer_to_type_extra(CodeGen g, ZigType *child_type, bool is_const,
20	20	bool is_volatile, PtrLen ptr_len, uint32_t byte_alignment, uint32_t bit_offset, uint32_t unaligned_bit_count);
21	21	uint64_t type_size(CodeGen g, ZigType type_entry);
	22	uint64_t type_size_store(CodeGen g, ZigType type_entry);
22	23	uint64_t type_size_bits(CodeGen g, ZigType type_entry);
23	24	ZigType get_int_type(CodeGen g, bool is_signed, uint32_t size_in_bits);
24	25	ZigType get_vector_type(CodeGen g, uint32_t len, ZigType *elem_type);

src/ir.cpp+20-8

...	...	@@ -14331,15 +14331,15 @@ static Error ir_read_const_ptr(IrAnalyze ira, CodeGen codegen, AstNode *source
14331	14331	if ((err = type_resolve(codegen, out_val->type, ResolveStatusSizeKnown)))
14332	14332	return ErrorSemanticAnalyzeFail;
14333	14333
14334		size_t src_size = type_size(codegen, pointee->type);
14335		size_t dst_size = type_size(codegen, out_val->type);
14336
14337		if (src_size == dst_size && types_have_same_zig_comptime_repr(pointee->type, out_val->type)) {
14338		copy_const_val(out_val, pointee, ptr_val->data.x_ptr.mut == ConstPtrMutComptimeConst);
14339		return ErrorNone;
14340		}
	14334	// We don't need to read the padding bytes, so we look at type_size_store bytes
	14335	size_t src_size = type_size_store(codegen, pointee->type);
	14336	size_t dst_size = type_size_store(codegen, out_val->type);
14341	14337
14342	14338	if (dst_size <= src_size) {
	14339	if (types_have_same_zig_comptime_repr(pointee->type, out_val->type)) {
	14340	copy_const_val(out_val, pointee, ptr_val->data.x_ptr.mut == ConstPtrMutComptimeConst);
	14341	return ErrorNone;
	14342	}
14343	14343	Buf buf = BUF_INIT;
14344	14344	buf_resize(&buf, src_size);
14345	14345	buf_write_value_bytes(codegen, (uint8_t*)buf_ptr(&buf), pointee);
...	...	@@ -15798,6 +15798,8 @@ static IrInstruction ir_analyze_instruction_typeof(IrAnalyze ira, IrInstructio
15798	15798	static IrInstruction ir_analyze_instruction_to_ptr_type(IrAnalyze ira,
15799	15799	IrInstructionToPtrType *to_ptr_type_instruction)
15800	15800	{
	15801	Error err;
	15802
15801	15803	IrInstruction *value = to_ptr_type_instruction->value->child;
15802	15804	ZigType *type_entry = value->value.type;
15803	15805	if (type_is_invalid(type_entry))
...	...	@@ -15813,7 +15815,17 @@ static IrInstruction ir_analyze_instruction_to_ptr_type(IrAnalyze ira,
15813	15815	ptr_type = get_pointer_to_type(ira->codegen,
15814	15816	type_entry->data.pointer.child_type->data.array.child_type, type_entry->data.pointer.is_const);
15815	15817	} else if (is_slice(type_entry)) {
15816		ptr_type = adjust_ptr_len(ira->codegen, type_entry->data.structure.fields[0].type_entry, PtrLenSingle);
	15818	ZigType *slice_ptr_type = type_entry->data.structure.fields[0].type_entry;
	15819	ptr_type = adjust_ptr_len(ira->codegen, slice_ptr_type, PtrLenSingle);
	15820	// If the pointer is over-aligned, we may have to reduce it based on the alignment of the element type.
	15821	if (slice_ptr_type->data.pointer.explicit_alignment != 0) {
	15822	ZigType *elem_type = slice_ptr_type->data.pointer.child_type;
	15823	if ((err = type_resolve(ira->codegen, elem_type, ResolveStatusAlignmentKnown)))
	15824	return ira->codegen->invalid_instruction;
	15825	uint32_t elem_align = get_abi_alignment(ira->codegen, elem_type);
	15826	uint32_t reduced_align = min(elem_align, slice_ptr_type->data.pointer.explicit_alignment);
	15827	ptr_type = adjust_ptr_align(ira->codegen, ptr_type, reduced_align);
	15828	}
15817	15829	} else if (type_entry->id == ZigTypeIdArgTuple) {
15818	15830	ConstExprValue *arg_tuple_val = ir_resolve_const(ira, value, UndefBad);
15819	15831	if (!arg_tuple_val)

std/io.zig+5-8

...	...	@@ -935,8 +935,6 @@ pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type {
935	935	};
936	936	}
937	937
938
939
940	938	pub const BufferedAtomicFile = struct {
941	939	atomic_file: os.AtomicFile,
942	940	file_stream: os.File.OutStream,
...	...	@@ -978,7 +976,6 @@ pub const BufferedAtomicFile = struct {
978	976	}
979	977	};
980	978
981
982	979	pub fn readLine(buf: *std.Buffer) ![]u8 {
983	980	var stdin = try getStdIn();
984	981	var stdin_stream = stdin.inStream();
...	...	@@ -1073,13 +1070,13 @@ pub fn Deserializer(comptime endian: builtin.Endian, is_packed: bool, comptime E
1073	1070	else => in_stream,
1074	1071	} };
1075	1072	}
1076
	1073
1077	1074	pub fn alignToByte(self: *Self) void {
1078		if(!is_packed) return;
	1075	if (!is_packed) return;
1079	1076	self.in_stream.alignToByte();
1080	1077	}
1081	1078
1082		//@BUG: inferred error issue. See: #1386
	1079	//@BUG: inferred error issue. See: #1386
1083	1080	fn deserializeInt(self: *Self, comptime T: type) (Error \|\| error{EndOfStream})!T {
1084	1081	comptime assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T));
1085	1082
...	...	@@ -1088,7 +1085,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, is_packed: bool, comptime E
1088	1085
1089	1086	const U = @IntType(false, t_bit_count);
1090	1087	const Log2U = math.Log2Int(U);
1091		const int_size = @sizeOf(U);
	1088	const int_size = (U.bit_count + 7) / 8;
1092	1089
1093	1090	if (is_packed) {
1094	1091	const result = try self.in_stream.readBitsNoEof(U, t_bit_count);
...	...	@@ -1301,7 +1298,7 @@ pub fn Serializer(comptime endian: builtin.Endian, comptime is_packed: bool, com
1301	1298
1302	1299	const U = @IntType(false, t_bit_count);
1303	1300	const Log2U = math.Log2Int(U);
1304		const int_size = @sizeOf(U);
	1301	const int_size = (U.bit_count + 7) / 8;
1305	1302
1306	1303	const u_value = @bitCast(U, value);
1307	1304

std/mem.zig+20-29

...	...	@@ -423,8 +423,7 @@ pub fn readVarInt(comptime ReturnType: type, bytes: []const u8, endian: builtin.
423	423	/// This function cannot fail and cannot cause undefined behavior.
424	424	/// Assumes the endianness of memory is native. This means the function can
425	425	/// simply pointer cast memory.
426		pub fn readIntNative(comptime T: type, bytes: *const [@sizeOf(T)]u8) T {
427		comptime assert(T.bit_count % 8 == 0);
	426	pub fn readIntNative(comptime T: type, bytes: *const [@divExact(T.bit_count, 8)]u8) T {
428	427	return @ptrCast(align(1) const T, bytes).;
429	428	}
430	429
...	...	@@ -432,7 +431,7 @@ pub fn readIntNative(comptime T: type, bytes: *const [@sizeOf(T)]u8) T {
432	431	/// The bit count of T must be evenly divisible by 8.
433	432	/// This function cannot fail and cannot cause undefined behavior.
434	433	/// Assumes the endianness of memory is foreign, so it must byte-swap.
435		pub fn readIntForeign(comptime T: type, bytes: *const [@sizeOf(T)]u8) T {
	434	pub fn readIntForeign(comptime T: type, bytes: *const [@divExact(T.bit_count, 8)]u8) T {
436	435	return @bswap(T, readIntNative(T, bytes));
437	436	}
438	437
...	...	@@ -446,22 +445,20 @@ pub const readIntBig = switch (builtin.endian) {
446	445	builtin.Endian.Big => readIntNative,
447	446	};
448	447
449		/// Asserts that bytes.len >= @sizeOf(T). Reads the integer starting from index 0
	448	/// Asserts that bytes.len >= T.bit_count / 8. Reads the integer starting from index 0
450	449	/// and ignores extra bytes.
451		/// Note that @sizeOf(u24) is 3.
452	450	/// The bit count of T must be evenly divisible by 8.
453	451	/// Assumes the endianness of memory is native. This means the function can
454	452	/// simply pointer cast memory.
455	453	pub fn readIntSliceNative(comptime T: type, bytes: []const u8) T {
456		assert(@sizeOf(u24) == 3);
457		assert(bytes.len >= @sizeOf(T));
	454	const n = @divExact(T.bit_count, 8);
	455	assert(bytes.len >= n);
458	456	// TODO https://github.com/ziglang/zig/issues/863
459		return readIntNative(T, @ptrCast(*const [@sizeOf(T)]u8, bytes.ptr));
	457	return readIntNative(T, @ptrCast(*const [n]u8, bytes.ptr));
460	458	}
461	459
462		/// Asserts that bytes.len >= @sizeOf(T). Reads the integer starting from index 0
	460	/// Asserts that bytes.len >= T.bit_count / 8. Reads the integer starting from index 0
463	461	/// and ignores extra bytes.
464		/// Note that @sizeOf(u24) is 3.
465	462	/// The bit count of T must be evenly divisible by 8.
466	463	/// Assumes the endianness of memory is foreign, so it must byte-swap.
467	464	pub fn readIntSliceForeign(comptime T: type, bytes: []const u8) T {
...	...	@@ -481,7 +478,7 @@ pub const readIntSliceBig = switch (builtin.endian) {
481	478	/// Reads an integer from memory with bit count specified by T.
482	479	/// The bit count of T must be evenly divisible by 8.
483	480	/// This function cannot fail and cannot cause undefined behavior.
484		pub fn readInt(comptime T: type, bytes: *const [@sizeOf(T)]u8, endian: builtin.Endian) T {
	481	pub fn readInt(comptime T: type, bytes: *const [@divExact(T.bit_count, 8)]u8, endian: builtin.Endian) T {
485	482	if (endian == builtin.endian) {
486	483	return readIntNative(T, bytes);
487	484	} else {
...	...	@@ -489,15 +486,14 @@ pub fn readInt(comptime T: type, bytes: *const [@sizeOf(T)]u8, endian: builtin.E
489	486	}
490	487	}
491	488
492		/// Asserts that bytes.len >= @sizeOf(T). Reads the integer starting from index 0
	489	/// Asserts that bytes.len >= T.bit_count / 8. Reads the integer starting from index 0
493	490	/// and ignores extra bytes.
494		/// Note that @sizeOf(u24) is 3.
495	491	/// The bit count of T must be evenly divisible by 8.
496	492	pub fn readIntSlice(comptime T: type, bytes: []const u8, endian: builtin.Endian) T {
497		assert(@sizeOf(u24) == 3);
498		assert(bytes.len >= @sizeOf(T));
	493	const n = @divExact(T.bit_count, 8);
	494	assert(bytes.len >= n);
499	495	// TODO https://github.com/ziglang/zig/issues/863
500		return readInt(T, @ptrCast(*const [@sizeOf(T)]u8, bytes.ptr), endian);
	496	return readInt(T, @ptrCast(*const [n]u8, bytes.ptr), endian);
501	497	}
502	498
503	499	test "comptime read/write int" {
...	...	@@ -540,7 +536,7 @@ test "readIntBig and readIntLittle" {
540	536	/// accepts any integer bit width.
541	537	/// This function stores in native endian, which means it is implemented as a simple
542	538	/// memory store.
543		pub fn writeIntNative(comptime T: type, buf: *[@sizeOf(T)]u8, value: T) void {
	539	pub fn writeIntNative(comptime T: type, buf: *[(T.bit_count + 7) / 8]u8, value: T) void {
544	540	@ptrCast(align(1) T, buf). = value;
545	541	}
546	542
...	...	@@ -548,7 +544,7 @@ pub fn writeIntNative(comptime T: type, buf: *[@sizeOf(T)]u8, value: T) void {
548	544	/// This function always succeeds, has defined behavior for all inputs, but
549	545	/// the integer bit width must be divisible by 8.
550	546	/// This function stores in foreign endian, which means it does a @bswap first.
551		pub fn writeIntForeign(comptime T: type, buf: *[@sizeOf(T)]u8, value: T) void {
	547	pub fn writeIntForeign(comptime T: type, buf: *[@divExact(T.bit_count, 8)]u8, value: T) void {
552	548	writeIntNative(T, buf, @bswap(T, value));
553	549	}
554	550
...	...	@@ -565,8 +561,7 @@ pub const writeIntBig = switch (builtin.endian) {
565	561	/// Writes an integer to memory, storing it in twos-complement.
566	562	/// This function always succeeds, has defined behavior for all inputs, but
567	563	/// the integer bit width must be divisible by 8.
568		pub fn writeInt(comptime T: type, buffer: *[@sizeOf(T)]u8, value: T, endian: builtin.Endian) void {
569		comptime assert(T.bit_count % 8 == 0);
	564	pub fn writeInt(comptime T: type, buffer: *[@divExact(T.bit_count, 8)]u8, value: T, endian: builtin.Endian) void {
570	565	if (endian == builtin.endian) {
571	566	return writeIntNative(T, buffer, value);
572	567	} else {
...	...	@@ -575,15 +570,13 @@ pub fn writeInt(comptime T: type, buffer: *[@sizeOf(T)]u8, value: T, endian: bui
575	570	}
576	571
577	572	/// Writes a twos-complement little-endian integer to memory.
578		/// Asserts that buf.len >= @sizeOf(T). Note that @sizeOf(u24) is 3.
	573	/// Asserts that buf.len >= T.bit_count / 8.
579	574	/// The bit count of T must be divisible by 8.
580	575	/// Any extra bytes in buffer after writing the integer are set to zero. To
581	576	/// avoid the branch to check for extra buffer bytes, use writeIntLittle
582	577	/// instead.
583	578	pub fn writeIntSliceLittle(comptime T: type, buffer: []u8, value: T) void {
584		comptime assert(@sizeOf(u24) == 3);
585		comptime assert(T.bit_count % 8 == 0);
586		assert(buffer.len >= @sizeOf(T));
	579	assert(buffer.len >= @divExact(T.bit_count, 8));
587	580
588	581	// TODO I want to call writeIntLittle here but comptime eval facilities aren't good enough
589	582	const uint = @IntType(false, T.bit_count);
...	...	@@ -595,14 +588,12 @@ pub fn writeIntSliceLittle(comptime T: type, buffer: []u8, value: T) void {
595	588	}
596	589
597	590	/// Writes a twos-complement big-endian integer to memory.
598		/// Asserts that buffer.len >= @sizeOf(T). Note that @sizeOf(u24) is 3.
	591	/// Asserts that buffer.len >= T.bit_count / 8.
599	592	/// The bit count of T must be divisible by 8.
600	593	/// Any extra bytes in buffer before writing the integer are set to zero. To
601	594	/// avoid the branch to check for extra buffer bytes, use writeIntBig instead.
602	595	pub fn writeIntSliceBig(comptime T: type, buffer: []u8, value: T) void {
603		comptime assert(@sizeOf(u24) == 3);
604		comptime assert(T.bit_count % 8 == 0);
605		assert(buffer.len >= @sizeOf(T));
	596	assert(buffer.len >= @divExact(T.bit_count, 8));
606	597
607	598	// TODO I want to call writeIntBig here but comptime eval facilities aren't good enough
608	599	const uint = @IntType(false, T.bit_count);
...	...	@@ -626,7 +617,7 @@ pub const writeIntSliceForeign = switch (builtin.endian) {
626	617	};
627	618
628	619	/// Writes a twos-complement integer to memory, with the specified endianness.
629		/// Asserts that buf.len >= @sizeOf(T). Note that @sizeOf(u24) is 3.
	620	/// Asserts that buf.len >= T.bit_count / 8.
630	621	/// The bit count of T must be evenly divisible by 8.
631	622	/// Any extra bytes in buffer not part of the integer are set to zero, with
632	623	/// respect to endianness. To avoid the branch to check for extra buffer bytes,

test/stage1/behavior.zig+1

...	...	@@ -17,6 +17,7 @@ comptime {
17	17	_ = @import("behavior/bugs/1421.zig");
18	18	_ = @import("behavior/bugs/1442.zig");
19	19	_ = @import("behavior/bugs/1486.zig");
	20	_ = @import("behavior/bugs/1851.zig");
20	21	_ = @import("behavior/bugs/394.zig");
21	22	_ = @import("behavior/bugs/655.zig");
22	23	_ = @import("behavior/bugs/656.zig");

test/stage1/behavior/bugs/1851.zig created+27

...	...	@@ -0,0 +1,27 @@
	1	const std = @import("std");
	2	const expect = std.testing.expect;
	3
	4	test "allocation and looping over 3-byte integer" {
	5	expect(@sizeOf(u24) == 4);
	6	expect(@sizeOf([1]u24) == 4);
	7	expect(@alignOf(u24) == 4);
	8	expect(@alignOf([1]u24) == 4);
	9	var buffer: [100]u8 = undefined;
	10	const a = &std.heap.FixedBufferAllocator.init(&buffer).allocator;
	11
	12	var x = a.alloc(u24, 2) catch unreachable;
	13	expect(x.len == 2);
	14	x[0] = 0xFFFFFF;
	15	x[1] = 0xFFFFFF;
	16
	17	const bytes = @sliceToBytes(x);
	18	expect(@typeOf(bytes) == []align(4) u8);
	19	expect(bytes.len == 8);
	20
	21	for (bytes) \|*b\| {
	22	b.* = 0x00;
	23	}
	24
	25	expect(x[0] == 0x00);
	26	expect(x[1] == 0x00);
	27	}