Commit 60f0acd9b9c2bced47ba1c214460f34b73738f95 | zig

src/Module.zig+61-6

...	...	@@ -70,6 +70,17 @@ import_table: std.StringArrayHashMapUnmanaged(*File) = .{},
70	70	/// Keys are fully resolved file paths. This table owns the keys and values.
71	71	embed_table: std.StringHashMapUnmanaged(*EmbedFile) = .{},
72	72
	73	/// This is a temporary addition to stage2 in order to match stage1 behavior,
	74	/// however the end-game once the lang spec is settled will be to use a global
	75	/// InternPool for comptime memoized objects, making this behavior consistent across all types,
	76	/// not only string literals. Or, we might decide to not guarantee string literals
	77	/// to have equal comptime pointers, in which case this field can be deleted (perhaps
	78	/// the commit that introduced it can simply be reverted).
	79	/// This table uses an optional index so that when a Decl is destroyed, the string literal
	80	/// is still reclaimable by a future Decl.
	81	string_literal_table: std.HashMapUnmanaged(StringLiteralContext.Key, Decl.OptionalIndex, StringLiteralContext, std.hash_map.default_max_load_percentage) = .{},
	82	string_literal_bytes: std.ArrayListUnmanaged(u8) = .{},
	83
73	84	/// The set of all the generic function instantiations. This is used so that when a generic
74	85	/// function is called twice with the same comptime parameter arguments, both calls dispatch
75	86	/// to the same function.
...	...	@@ -157,6 +168,39 @@ decls_free_list: std.ArrayListUnmanaged(Decl.Index) = .{},
157	168
158	169	global_assembly: std.AutoHashMapUnmanaged(Decl.Index, []u8) = .{},
159	170
	171	pub const StringLiteralContext = struct {
	172	bytes: *std.ArrayListUnmanaged(u8),
	173
	174	pub const Key = struct {
	175	index: u32,
	176	len: u32,
	177	};
	178
	179	pub fn eql(self: @This(), a: Key, b: Key) bool {
	180	_ = self;
	181	return a.index == b.index and a.len == b.len;
	182	}
	183
	184	pub fn hash(self: @This(), x: Key) u64 {
	185	const x_slice = self.bytes.items[x.index..][0..x.len];
	186	return std.hash_map.hashString(x_slice);
	187	}
	188	};
	189
	190	pub const StringLiteralAdapter = struct {
	191	bytes: *std.ArrayListUnmanaged(u8),
	192
	193	pub fn eql(self: @This(), a_slice: []const u8, b: StringLiteralContext.Key) bool {
	194	const b_slice = self.bytes.items[b.index..][0..b.len];
	195	return mem.eql(u8, a_slice, b_slice);
	196	}
	197
	198	pub fn hash(self: @This(), adapted_key: []const u8) u64 {
	199	_ = self;
	200	return std.hash_map.hashString(adapted_key);
	201	}
	202	};
	203
160	204	const MonomorphedFuncsSet = std.HashMapUnmanaged(
161	205	*Fn,
162	206	void,
...	...	@@ -507,7 +551,8 @@ pub const Decl = struct {
507	551	decl.name = undefined;
508	552	}
509	553
510		pub fn clearValues(decl: *Decl, gpa: Allocator) void {
	554	pub fn clearValues(decl: Decl, mod: Module) void {
	555	const gpa = mod.gpa;
511	556	if (decl.getExternFn()) \|extern_fn\| {
512	557	extern_fn.deinit(gpa);
513	558	gpa.destroy(extern_fn);
...	...	@@ -521,6 +566,13 @@ pub const Decl = struct {
521	566	gpa.destroy(variable);
522	567	}
523	568	if (decl.value_arena) \|arena_state\| {
	569	if (decl.owns_tv) {
	570	if (decl.val.castTag(.str_lit)) \|str_lit\| {
	571	mod.string_literal_table.getPtrContext(str_lit.data, .{
	572	.bytes = &mod.string_literal_bytes,
	573	}).?.* = .none;
	574	}
	575	}
524	576	arena_state.promote(gpa).deinit();
525	577	decl.value_arena = null;
526	578	decl.has_tv = false;
...	...	@@ -2839,6 +2891,9 @@ pub fn deinit(mod: *Module) void {
2839	2891	mod.decls_free_list.deinit(gpa);
2840	2892	mod.allocated_decls.deinit(gpa);
2841	2893	mod.global_assembly.deinit(gpa);
	2894
	2895	mod.string_literal_table.deinit(gpa);
	2896	mod.string_literal_bytes.deinit(gpa);
2842	2897	}
2843	2898
2844	2899	pub fn destroyDecl(mod: *Module, decl_index: Decl.Index) void {
...	...	@@ -2857,7 +2912,7 @@ pub fn destroyDecl(mod: *Module, decl_index: Decl.Index) void {
2857	2912	if (decl.getInnerNamespace()) \|namespace\| {
2858	2913	namespace.destroyDecls(mod);
2859	2914	}
2860		decl.clearValues(gpa);
	2915	decl.clearValues(mod);
2861	2916	}
2862	2917	decl.dependants.deinit(gpa);
2863	2918	decl.dependencies.deinit(gpa);
...	...	@@ -4034,7 +4089,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
4034	4089	if (decl.getFunction()) \|prev_func\| {
4035	4090	prev_is_inline = prev_func.state == .inline_only;
4036	4091	}
4037		decl.clearValues(gpa);
	4092	decl.clearValues(mod);
4038	4093	}
4039	4094
4040	4095	decl.ty = try decl_tv.ty.copy(decl_arena_allocator);
...	...	@@ -4080,7 +4135,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
4080	4135	var type_changed = true;
4081	4136	if (decl.has_tv) {
4082	4137	type_changed = !decl.ty.eql(decl_tv.ty, mod);
4083		decl.clearValues(gpa);
	4138	decl.clearValues(mod);
4084	4139	}
4085	4140
4086	4141	decl.owns_tv = false;
...	...	@@ -4694,7 +4749,7 @@ pub fn clearDecl(
4694	4749	if (decl.getInnerNamespace()) \|namespace\| {
4695	4750	try namespace.deleteAllDecls(mod, outdated_decls);
4696	4751	}
4697		decl.clearValues(gpa);
	4752	decl.clearValues(mod);
4698	4753	}
4699	4754
4700	4755	if (decl.deletion_flag) {
...	...	@@ -5623,7 +5678,7 @@ pub fn populateTestFunctions(mod: *Module) !void {
5623	5678
5624	5679	// Since we are replacing the Decl's value we must perform cleanup on the
5625	5680	// previous value.
5626		decl.clearValues(gpa);
	5681	decl.clearValues(mod);
5627	5682	decl.ty = new_ty;
5628	5683	decl.val = new_val;
5629	5684	decl.has_tv = true;

src/Sema.zig+79-13

...	...	@@ -3842,20 +3842,44 @@ fn zirStr(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Ins
3842	3842	fn addStrLit(sema: Sema, block: Block, zir_bytes: []const u8) CompileError!Air.Inst.Ref {
3843	3843	// `zir_bytes` references memory inside the ZIR module, which can get deallocated
3844	3844	// after semantic analysis is complete, for example in the case of the initialization
3845		// expression of a variable declaration. We need the memory to be in the new
3846		// anonymous Decl's arena.
3847		var anon_decl = try block.startAnonDecl(LazySrcLoc.unneeded);
3848		defer anon_decl.deinit();
	3845	// expression of a variable declaration.
	3846	const mod = sema.mod;
	3847	const gpa = sema.gpa;
	3848	const string_bytes = &mod.string_literal_bytes;
	3849	const StringLiteralAdapter = Module.StringLiteralAdapter;
	3850	const StringLiteralContext = Module.StringLiteralContext;
	3851	try string_bytes.ensureUnusedCapacity(gpa, zir_bytes.len);
	3852	const gop = try mod.string_literal_table.getOrPutContextAdapted(gpa, zir_bytes, StringLiteralAdapter{
	3853	.bytes = string_bytes,
	3854	}, StringLiteralContext{
	3855	.bytes = string_bytes,
	3856	});
	3857	if (!gop.found_existing) {
	3858	gop.key_ptr.* = .{
	3859	.index = @intCast(u32, string_bytes.items.len),
	3860	.len = @intCast(u32, zir_bytes.len),
	3861	};
	3862	string_bytes.appendSliceAssumeCapacity(zir_bytes);
	3863	gop.value_ptr.* = .none;
	3864	}
	3865	const decl_index = gop.value_ptr.unwrap() orelse di: {
	3866	var anon_decl = try block.startAnonDecl(LazySrcLoc.unneeded);
	3867	defer anon_decl.deinit();
3849	3868
3850		const bytes = try anon_decl.arena().dupeZ(u8, zir_bytes);
	3869	const decl_index = try anon_decl.finish(
	3870	try Type.Tag.array_u8_sentinel_0.create(anon_decl.arena(), gop.key_ptr.len),
	3871	try Value.Tag.str_lit.create(anon_decl.arena(), gop.key_ptr.*),
	3872	0, // default alignment
	3873	);
3851	3874
3852		const new_decl = try anon_decl.finish(
3853		try Type.Tag.array_u8_sentinel_0.create(anon_decl.arena(), bytes.len),
3854		try Value.Tag.bytes.create(anon_decl.arena(), bytes[0 .. bytes.len + 1]),
3855		0, // default alignment
3856		);
	3875	// Needed so that `Decl.clearValues` will additionally set the corresponding
	3876	// string literal table value back to `Decl.OptionalIndex.none`.
	3877	mod.declPtr(decl_index).owns_tv = true;
3857	3878
3858		return sema.analyzeDeclRef(new_decl);
	3879	gop.value_ptr.* = decl_index.toOptional();
	3880	break :di decl_index;
	3881	};
	3882	return sema.analyzeDeclRef(decl_index);
3859	3883	}
3860	3884
3861	3885	fn zirInt(sema: Sema, block: Block, inst: Zir.Inst.Index) CompileError!Air.Inst.Ref {
...	...	@@ -19762,6 +19786,35 @@ fn beginComptimePtrMutation(
19762	19786	.ty = elem_ty,
19763	19787	};
19764	19788	},
	19789	.str_lit => {
	19790	// An array is memory-optimized to store a slice of bytes, but we are about
	19791	// to modify an individual field and the representation has to change.
	19792	// If we wanted to avoid this, there would need to be special detection
	19793	// elsewhere to identify when writing a value to an array element that is stored
	19794	// using the `str_lit` tag, and handle it without making a call to this function.
	19795	const arena = parent.beginArena(sema.mod);
	19796	defer parent.finishArena(sema.mod);
	19797
	19798	const str_lit = parent.val.castTag(.str_lit).?.data;
	19799	const dest_len = parent.ty.arrayLenIncludingSentinel();
	19800	const bytes = sema.mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
	19801	const elems = try arena.alloc(Value, @intCast(usize, dest_len));
	19802	for (bytes) \|byte, i\| {
	19803	elems[i] = try Value.Tag.int_u64.create(arena, byte);
	19804	}
	19805	if (parent.ty.sentinel()) \|sent_val\| {
	19806	assert(elems.len == bytes.len + 1);
	19807	elems[bytes.len] = sent_val;
	19808	}
	19809
	19810	parent.val.* = try Value.Tag.aggregate.create(arena, elems);
	19811
	19812	return ComptimePtrMutationKit{
	19813	.decl_ref_mut = parent.decl_ref_mut,
	19814	.val = &elems[elem_ptr.index],
	19815	.ty = elem_ty,
	19816	};
	19817	},
19765	19818	.repeated => {
19766	19819	// An array is memory-optimized to store only a single element value, and
19767	19820	// that value is understood to be the same for the entire length of the array.
...	...	@@ -20097,10 +20150,23 @@ fn beginComptimePtrLoad(
20097	20150	}
20098	20151	}
20099	20152
20100		deref.pointee = if (elem_ptr.index < check_len) TypedValue{
	20153	if (elem_ptr.index >= check_len) {
	20154	deref.pointee = null;
	20155	break :blk deref;
	20156	}
	20157	if (elem_ptr.index == check_len - 1) {
	20158	if (array_tv.ty.sentinel()) \|sent\| {
	20159	deref.pointee = TypedValue{
	20160	.ty = elem_ty,
	20161	.val = sent,
	20162	};
	20163	break :blk deref;
	20164	}
	20165	}
	20166	deref.pointee = TypedValue{
20101	20167	.ty = elem_ty,
20102	20168	.val = try array_tv.val.elemValue(sema.mod, sema.arena, elem_ptr.index),
20103		} else null;
	20169	};
20104	20170	break :blk deref;
20105	20171	},
20106	20172

src/TypedValue.zig+5

...	...	@@ -295,6 +295,11 @@ pub fn print(
295	295	return writer.print(".{s}", .{ty.enumFieldName(val.castTag(.enum_field_index).?.data)});
296	296	},
297	297	.bytes => return writer.print("\"{}\"", .{std.zig.fmtEscapes(val.castTag(.bytes).?.data)}),
	298	.str_lit => {
	299	const str_lit = val.castTag(.str_lit).?.data;
	300	const bytes = mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
	301	return writer.print("\"{}\"", .{std.zig.fmtEscapes(bytes)});
	302	},
298	303	.repeated => {
299	304	if (level == 0) {
300	305	return writer.writeAll(".{ ... }");

src/codegen.zig+14-2

...	...	@@ -203,11 +203,23 @@ pub fn generateSymbol(
203	203	},
204	204	.Array => switch (typed_value.val.tag()) {
205	205	.bytes => {
206		const payload = typed_value.val.castTag(.bytes).?;
	206	const bytes = typed_value.val.castTag(.bytes).?.data;
207	207	const len = @intCast(usize, typed_value.ty.arrayLenIncludingSentinel());
208	208	// The bytes payload already includes the sentinel, if any
209	209	try code.ensureUnusedCapacity(len);
210		code.appendSliceAssumeCapacity(payload.data[0..len]);
	210	code.appendSliceAssumeCapacity(bytes[0..len]);
	211	return Result{ .appended = {} };
	212	},
	213	.str_lit => {
	214	const str_lit = typed_value.val.castTag(.str_lit).?.data;
	215	const mod = bin_file.options.module.?;
	216	const bytes = mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
	217	try code.ensureUnusedCapacity(bytes.len + 1);
	218	code.appendSliceAssumeCapacity(bytes);
	219	if (typed_value.ty.sentinel()) \|sent_val\| {
	220	const byte = @intCast(u8, sent_val.toUnsignedInt(target));
	221	code.appendAssumeCapacity(byte);
	222	}
211	223	return Result{ .appended = {} };
212	224	},
213	225	.aggregate => {

src/codegen/llvm.zig+31-1

...	...	@@ -2936,9 +2936,39 @@ pub const DeclGen = struct {
2936	2936	return dg.context.constString(
2937	2937	bytes.ptr,
2938	2938	@intCast(c_uint, tv.ty.arrayLenIncludingSentinel()),
2939		.True, // don't null terminate. bytes has the sentinel, if any.
	2939	.True, // Don't null terminate. Bytes has the sentinel, if any.
2940	2940	);
2941	2941	},
	2942	.str_lit => {
	2943	const str_lit = tv.val.castTag(.str_lit).?.data;
	2944	const bytes = dg.module.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
	2945	if (tv.ty.sentinel()) \|sent_val\| {
	2946	const byte = @intCast(u8, sent_val.toUnsignedInt(target));
	2947	if (byte == 0 and bytes.len > 0) {
	2948	return dg.context.constString(
	2949	bytes.ptr,
	2950	@intCast(c_uint, bytes.len),
	2951	.False, // Yes, null terminate.
	2952	);
	2953	}
	2954	var array = std.ArrayList(u8).init(dg.gpa);
	2955	defer array.deinit();
	2956	try array.ensureUnusedCapacity(bytes.len + 1);
	2957	array.appendSliceAssumeCapacity(bytes);
	2958	array.appendAssumeCapacity(byte);
	2959	return dg.context.constString(
	2960	array.items.ptr,
	2961	@intCast(c_uint, array.items.len),
	2962	.True, // Don't null terminate.
	2963	);
	2964	} else {
	2965	return dg.context.constString(
	2966	bytes.ptr,
	2967	@intCast(c_uint, bytes.len),
	2968	.True, // Don't null terminate. `bytes` has the sentinel, if any.
	2969	);
	2970	}
	2971	},
2942	2972	.aggregate => {
2943	2973	const elem_vals = tv.val.castTag(.aggregate).?.data;
2944	2974	const elem_ty = tv.ty.elemType();

src/value.zig+41

...	...	@@ -126,6 +126,8 @@ pub const Value = extern union {
126	126	field_ptr,
127	127	/// A slice of u8 whose memory is managed externally.
128	128	bytes,
	129	/// Similar to bytes however it stores an index relative to `Module.string_literal_bytes`.
	130	str_lit,
129	131	/// This value is repeated some number of times. The amount of times to repeat
130	132	/// is stored externally.
131	133	repeated,
...	...	@@ -285,6 +287,7 @@ pub const Value = extern union {
285	287	.enum_literal,
286	288	=> Payload.Bytes,
287	289
	290	.str_lit => Payload.StrLit,
288	291	.slice => Payload.Slice,
289	292
290	293	.enum_field_index => Payload.U32,
...	...	@@ -538,6 +541,7 @@ pub const Value = extern union {
538	541	};
539	542	return Value{ .ptr_otherwise = &new_payload.base };
540	543	},
	544	.str_lit => return self.copyPayloadShallow(arena, Payload.StrLit),
541	545	.repeated,
542	546	.eu_payload,
543	547	.opt_payload,
...	...	@@ -764,6 +768,12 @@ pub const Value = extern union {
764	768	.enum_literal => return out_stream.print(".{}", .{std.zig.fmtId(val.castTag(.enum_literal).?.data)}),
765	769	.enum_field_index => return out_stream.print("(enum field {d})", .{val.castTag(.enum_field_index).?.data}),
766	770	.bytes => return out_stream.print("\"{}\"", .{std.zig.fmtEscapes(val.castTag(.bytes).?.data)}),
	771	.str_lit => {
	772	const str_lit = val.castTag(.str_lit).?.data;
	773	return out_stream.print("(.str_lit index={d} len={d})", .{
	774	str_lit.index, str_lit.len,
	775	});
	776	},
767	777	.repeated => {
768	778	try out_stream.writeAll("(repeated) ");
769	779	val = val.castTag(.repeated).?.data;
...	...	@@ -824,6 +834,11 @@ pub const Value = extern union {
824	834	const adjusted_bytes = bytes[0..adjusted_len];
825	835	return allocator.dupe(u8, adjusted_bytes);
826	836	},
	837	.str_lit => {
	838	const str_lit = val.castTag(.str_lit).?.data;
	839	const bytes = mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
	840	return allocator.dupe(u8, bytes);
	841	},
827	842	.enum_literal => return allocator.dupe(u8, val.castTag(.enum_literal).?.data),
828	843	.repeated => {
829	844	const byte = @intCast(u8, val.castTag(.repeated).?.data.toUnsignedInt(target));
...	...	@@ -2537,6 +2552,20 @@ pub const Value = extern union {
2537	2552	return initPayload(&buffer.base);
2538	2553	}
2539	2554	},
	2555	.str_lit => {
	2556	const str_lit = val.castTag(.str_lit).?.data;
	2557	const bytes = mod.string_literal_bytes.items[str_lit.index..][0..str_lit.len];
	2558	const byte = bytes[index];
	2559	if (arena) \|a\| {
	2560	return Tag.int_u64.create(a, byte);
	2561	} else {
	2562	buffer.* = .{
	2563	.base = .{ .tag = .int_u64 },
	2564	.data = byte,
	2565	};
	2566	return initPayload(&buffer.base);
	2567	}
	2568	},
2540	2569
2541	2570	// No matter the index; all the elements are the same!
2542	2571	.repeated => return val.castTag(.repeated).?.data,
...	...	@@ -2570,6 +2599,13 @@ pub const Value = extern union {
2570	2599	return switch (val.tag()) {
2571	2600	.empty_array_sentinel => if (start == 0 and end == 1) val else Value.initTag(.empty_array),
2572	2601	.bytes => Tag.bytes.create(arena, val.castTag(.bytes).?.data[start..end]),
	2602	.str_lit => {
	2603	const str_lit = val.castTag(.str_lit).?.data;
	2604	return Tag.str_lit.create(arena, .{
	2605	.index = @intCast(u32, str_lit.index + start),
	2606	.len = @intCast(u32, end - start),
	2607	});
	2608	},
2573	2609	.aggregate => Tag.aggregate.create(arena, val.castTag(.aggregate).?.data[start..end]),
2574	2610	.slice => sliceArray(val.castTag(.slice).?.data.ptr, mod, arena, start, end),
2575	2611
...	...	@@ -4721,6 +4757,11 @@ pub const Value = extern union {
4721	4757	data: []const u8,
4722	4758	};
4723	4759
	4760	pub const StrLit = struct {
	4761	base: Payload,
	4762	data: Module.StringLiteralContext.Key,
	4763	};
	4764
4724	4765	pub const Aggregate = struct {
4725	4766	base: Payload,
4726	4767	/// Field values. The types are according to the struct or array type.

test/behavior/eval.zig+11-2

...	...	@@ -643,9 +643,18 @@ fn assertEqualPtrs(ptr1: const u8, ptr2: const u8) !void {
643	643	try expect(ptr1 == ptr2);
644	644	}
645	645
	646	// This one is still up for debate in the language specification.
	647	// Application code should not rely on this behavior until it is solidified.
	648	// Currently, stage1 has special case code to make this pass for string literals
	649	// but it does not work if the values are constructed with comptime code, or if
	650	// arrays of non-u8 elements are used instead.
	651	// The official language specification might not make this guarantee. However, if
	652	// it does make this guarantee, it will make it consistently for all types, not
	653	// only string literals. This is why stage2 currently has a string table for
	654	// string literals, to match stage1 and pass this test, however the end-game once
	655	// the lang spec issue is settled would be to use a global InternPool for comptime
	656	// memoized objects, making this behavior consistent across all types.
646	657	test "string literal used as comptime slice is memoized" {
647		if (builtin.zig_backend != .stage1) return error.SkipZigTest; // TODO
648
649	658	const a = "link";
650	659	const b = "link";
651	660	comptime try expect(TypeWithCompTimeSlice(a).Node == TypeWithCompTimeSlice(b).Node);

test/cases/llvm/pointer_with_different_address_spaces.zig+2-2

...	...	@@ -1,12 +1,12 @@
1	1	fn entry(a: addrspace(.gs) i32) addrspace(.fs) i32 {
2	2	return a;
3	3	}
4		pub fn main() void {
	4	export fn entry2() void {
5	5	_ = entry;
6	6	}
7	7
8	8	// error
9		// output_mode=Exe
	9	// output_mode=Obj
10	10	// backend=stage2,llvm
11	11	// target=x86_64-linux,x86_64-macos
12	12	//

author	Andrew Kelley <andrew@ziglang.org> 2022-05-24 15:32:12-04:00
committer	GitHub <noreply@github.com> 2022-05-24 15:32:12-04:00
log	60f0acd9b9c2bced47ba1c214460f34b73738f95
tree	7cc1ba371eab54fb84d5f56a67c3dab0d035bf7a
parent	8171972cbb477672ee1a99d953df4aaecb744a0c
parent	953e2778d4402cf85b99061ef9bdb89aa3e5f2db
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #11706 from ziglang/string-literal-interning