authorgravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-03-18 16:02:30+01:00
committergravatar for luuk@degram.devLuuk de Gram <luuk@degram.dev> 2023-03-18 20:13:30+01:00
log4e0d7154b1a701b906f3d9c5401dc0109253971f
tree36a7c5fa940450ad36698121a8723f7733cc59ea
parent09d6938df9246bac20e84f5512743e96bccdfa3d
signaturelock-open Commit is signed but in an unrecognized format.

wasm-linker: implement __wasm_init_memory & flag

Implements the __wasm_init_memory and __wasm_init_memory_flag synthetic function and symbol. The former will initialize all passive segments during runtime. For the bss section we will fill it with zeroes, whereas the other segments will simply be initialized only. The latter stores the offset into the linear data section, after all heap memory that is part of the Wasm module. Any memory initialized at runtime starts from this offset.

1 files changed, 203 insertions(+), 0 deletions(-)

src/link/Wasm.zig+203
...@@ -192,6 +192,14 @@ pub const Segment = struct {...@@ -192,6 +192,14 @@ pub const Segment = struct {
192 pub fn isPassive(segment: Segment) bool {192 pub fn isPassive(segment: Segment) bool {
193 return segment.flags & @enumToInt(Flag.WASM_DATA_SEGMENT_IS_PASSIVE) != 0;193 return segment.flags & @enumToInt(Flag.WASM_DATA_SEGMENT_IS_PASSIVE) != 0;
194 }194 }
195
196 /// For a given segment, determines if it needs passive initialization
197 fn needsPassiveInitialization(segment: Segment, import_mem: bool, name: []const u8) bool {
198 if (import_mem and !std.mem.eql(u8, name, ".bss")) {
199 return true;
200 }
201 return segment.isPassive();
202 }
195};203};
196204
197pub const Export = struct {205pub const Export = struct {
...@@ -824,6 +832,178 @@ fn resolveSymbolsInArchives(wasm: *Wasm) !void {...@@ -824,6 +832,178 @@ fn resolveSymbolsInArchives(wasm: *Wasm) !void {
824 }832 }
825}833}
826834
835fn setupInitMemoryFunction(wasm: *Wasm) !void {
836 // Passive segments are used to avoid memory being reinitialized on each
837 // thread's instantiation. These passive segments are initialized and
838 // dropped in __wasm_init_memory, which is registered as the start function
839 // We also initialize bss segments (using memory.fill) as part of this
840 // function.
841 if (!wasm.hasPassiveInitializationSegments()) {
842 return;
843 }
844
845 const flag_address: u32 = if (wasm.base.options.shared_memory) address: {
846 // when we have passive initialization segments and shared memory
847 // `setupMemory` will create this symbol and set its virtual address.
848 const loc = wasm.findGlobalSymbol("__wasm_init_memory_flag").?;
849 break :address loc.getSymbol(wasm).virtual_address;
850 } else 0;
851
852 var function_body = std.ArrayList(u8).init(wasm.base.allocator);
853 defer function_body.deinit();
854 const writer = function_body.writer();
855
856 // we have 0 locals
857 try leb.writeULEB128(writer, @as(u32, 0));
858
859 if (wasm.base.options.shared_memory) {
860 // destination blocks
861 // based on values we jump to corresponding label
862 try writer.writeByte(std.wasm.opcode(.block)); // $drop
863 try writer.writeByte(std.wasm.block_empty); // block type
864
865 try writer.writeByte(std.wasm.opcode(.block)); // $wait
866 try writer.writeByte(std.wasm.block_empty); // block type
867
868 try writer.writeByte(std.wasm.opcode(.block)); // $init
869 try writer.writeByte(std.wasm.block_empty); // block type
870
871 // atomically check
872 try writer.writeByte(std.wasm.opcode(.i32_const));
873 try leb.writeULEB128(writer, flag_address);
874 try writer.writeByte(std.wasm.opcode(.i32_const));
875 try leb.writeULEB128(writer, @as(u32, 0));
876 try writer.writeByte(std.wasm.opcode(.i32_const));
877 try leb.writeULEB128(writer, @as(u32, 1));
878 try writer.writeByte(std.wasm.opcode(.atomics_prefix));
879 try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.i32_atomic_rmw_cmpxchg));
880 try leb.writeULEB128(writer, @as(u32, 2)); // alignment
881 try leb.writeULEB128(writer, @as(u32, 0)); // offset
882
883 // based on the value from the atomic check, jump to the label.
884 try writer.writeByte(std.wasm.opcode(.br_table));
885 try leb.writeULEB128(writer, @as(u32, 2)); // length of the table (we have 3 blocks but because of the mandatory default the length is 2).
886 try leb.writeULEB128(writer, @as(u32, 0)); // $init
887 try leb.writeULEB128(writer, @as(u32, 1)); // $wait
888 try leb.writeULEB128(writer, @as(u32, 2)); // $drop
889 try writer.writeByte(std.wasm.opcode(.end));
890 }
891
892 var it = wasm.data_segments.iterator();
893 var segment_index: u32 = 0;
894 while (it.next()) |entry| : (segment_index += 1) {
895 const segment: Segment = wasm.segments.items[entry.value_ptr.*];
896 if (segment.needsPassiveInitialization(wasm.base.options.import_memory, entry.key_ptr.*)) {
897 // For passive BSS segments we can simple issue a memory.fill(0).
898 // For non-BSS segments we do a memory.init. Both these
899 // instructions take as their first argument the destination
900 // address.
901 try writer.writeByte(std.wasm.opcode(.i32_const));
902 try leb.writeULEB128(writer, segment.offset);
903
904 if (wasm.base.options.shared_memory and std.mem.eql(u8, entry.key_ptr.*, ".tdata")) {
905 // When we initialize the TLS segment we also set the `__tls_base`
906 // global. This allows the runtime to use this static copy of the
907 // TLS data for the first/main thread.
908 try writer.writeByte(std.wasm.opcode(.i32_const));
909 try leb.writeULEB128(writer, segment.offset);
910 try writer.writeByte(std.wasm.opcode(.global_set));
911 const loc = wasm.findGlobalSymbol("__tls_base").?;
912 try leb.writeULEB128(writer, loc.getSymbol(wasm).index);
913 }
914
915 try writer.writeByte(std.wasm.opcode(.i32_const));
916 try leb.writeULEB128(writer, @as(u32, 0));
917 try writer.writeByte(std.wasm.opcode(.i32_const));
918 try leb.writeULEB128(writer, segment.size);
919 try writer.writeByte(std.wasm.opcode(.misc_prefix));
920 if (std.mem.eql(u8, entry.key_ptr.*, ".bss")) {
921 // fill bss segment with zeroes
922 try leb.writeULEB128(writer, std.wasm.miscOpcode(.memory_fill));
923 } else {
924 // initialize the segment
925 try leb.writeULEB128(writer, std.wasm.miscOpcode(.memory_init));
926 try leb.writeULEB128(writer, segment_index);
927 }
928 try writer.writeByte(0); // memory index immediate
929 }
930 }
931
932 if (wasm.base.options.shared_memory) {
933 // we set the init memory flag to value '2'
934 try writer.writeByte(std.wasm.opcode(.i32_const));
935 try leb.writeULEB128(writer, flag_address);
936 try writer.writeByte(std.wasm.opcode(.i32_const));
937 try leb.writeULEB128(writer, @as(u32, 2));
938 try writer.writeByte(std.wasm.opcode(.atomics_prefix));
939 try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.i32_atomic_store));
940 try leb.writeULEB128(writer, @as(u32, 2)); // alignment
941 try leb.writeULEB128(writer, @as(u32, 0)); // offset
942
943 // notify any waiters for segment initialization completion
944 try writer.writeByte(std.wasm.opcode(.i32_const));
945 try leb.writeULEB128(writer, flag_address);
946 try writer.writeByte(std.wasm.opcode(.i32_const));
947 try leb.writeILEB128(writer, @as(i32, -1)); // number of waiters
948 try writer.writeByte(std.wasm.opcode(.atomics_prefix));
949 try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.memory_atomic_notify));
950 try leb.writeULEB128(writer, @as(u32, 2)); // alignment
951 try leb.writeULEB128(writer, @as(u32, 0)); // offset
952 try writer.writeByte(std.wasm.opcode(.drop));
953
954 // branch and drop segments
955 try writer.writeByte(std.wasm.opcode(.br));
956 try leb.writeULEB128(writer, @as(u32, 1));
957
958 // wait for thread to initialize memory segments
959 try writer.writeByte(std.wasm.opcode(.end)); // end $wait
960 try writer.writeByte(std.wasm.opcode(.i32_const));
961 try leb.writeULEB128(writer, flag_address);
962 try writer.writeByte(std.wasm.opcode(.i32_const));
963 try leb.writeULEB128(writer, @as(u32, 1)); // expected flag value
964 try writer.writeByte(std.wasm.opcode(.i32_const));
965 try leb.writeILEB128(writer, @as(i32, -1)); // timeout
966 try writer.writeByte(std.wasm.opcode(.atomics_prefix));
967 try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.memory_atomic_wait32));
968 try leb.writeULEB128(writer, @as(u32, 2)); // alignment
969 try leb.writeULEB128(writer, @as(u32, 0)); // offset
970 try writer.writeByte(std.wasm.opcode(.drop));
971
972 try writer.writeByte(std.wasm.opcode(.end)); // end $drop
973 }
974
975 it.reset();
976 segment_index = 0;
977 while (it.next()) |entry| : (segment_index += 1) {
978 const name = entry.key_ptr.*;
979 const segment: Segment = wasm.segments.items[entry.value_ptr.*];
980 if (segment.needsPassiveInitialization(wasm.base.options.import_memory, name) and
981 !std.mem.eql(u8, name, ".bss"))
982 {
983 // The TLS region should not be dropped since its is needed
984 // during the initialization of each thread (__wasm_init_tls).
985 if (wasm.base.options.shared_memory and std.mem.eql(u8, name, ".tdata")) {
986 continue;
987 }
988
989 try writer.writeByte(std.wasm.opcode(.misc_prefix));
990 try leb.writeULEB128(writer, std.wasm.miscOpcode(.data_drop));
991 try leb.writeULEB128(writer, segment_index);
992 }
993 }
994
995 // End of the function body
996 try writer.writeByte(std.wasm.opcode(.end));
997
998 try wasm.createSyntheticFunction(
999 "__wasm_init_memory",
1000 std.wasm.Type{ .params = &.{}, .returns = &.{} },
1001 &function_body,
1002 );
1003}
1004
1005/// Constructs a synthetic function that performs runtime relocations for
1006/// TLS symbols. This function is called by `__wasm_init_tls`.
827fn setupTLSRelocationsFunction(wasm: *Wasm) !void {1007fn setupTLSRelocationsFunction(wasm: *Wasm) !void {
828 // When we have TLS GOT entries and shared memory is enabled,1008 // When we have TLS GOT entries and shared memory is enabled,
829 // we must perform runtime relocations or else we don't create the function.1009 // we must perform runtime relocations or else we don't create the function.
...@@ -2469,6 +2649,16 @@ fn setupMemory(wasm: *Wasm) !void {...@@ -2469,6 +2649,16 @@ fn setupMemory(wasm: *Wasm) !void {
2469 offset += segment.size;2649 offset += segment.size;
2470 }2650 }
24712651
2652 // create the memory init flag which is used by the init memory function
2653 if (wasm.base.options.shared_memory and wasm.hasPassiveInitializationSegments()) {
2654 // align to pointer size
2655 memory_ptr = mem.alignForwardGeneric(u64, memory_ptr, 4);
2656 const loc = try wasm.createSyntheticSymbol("__wasm_init_memory_flag", .data);
2657 const sym = loc.getSymbol(wasm);
2658 sym.virtual_address = @intCast(u32, memory_ptr);
2659 memory_ptr += 4;
2660 }
2661
2472 if (!place_stack_first and !is_obj) {2662 if (!place_stack_first and !is_obj) {
2473 memory_ptr = std.mem.alignForwardGeneric(u64, memory_ptr, stack_alignment);2663 memory_ptr = std.mem.alignForwardGeneric(u64, memory_ptr, stack_alignment);
2474 memory_ptr += stack_size;2664 memory_ptr += stack_size;
...@@ -3000,6 +3190,7 @@ fn linkWithZld(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) l...@@ -3000,6 +3190,7 @@ fn linkWithZld(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) l
3000 try wasm.mergeSections();3190 try wasm.mergeSections();
3001 try wasm.mergeTypes();3191 try wasm.mergeTypes();
3002 try wasm.initializeCallCtorsFunction();3192 try wasm.initializeCallCtorsFunction();
3193 try wasm.setupInitMemoryFunction();
3003 try wasm.setupTLSRelocationsFunction();3194 try wasm.setupTLSRelocationsFunction();
3004 try wasm.initializeTLSFunction();3195 try wasm.initializeTLSFunction();
3005 try wasm.setupExports();3196 try wasm.setupExports();
...@@ -3121,6 +3312,7 @@ pub fn flushModule(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod...@@ -3121,6 +3312,7 @@ pub fn flushModule(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod
3121 try wasm.mergeSections();3312 try wasm.mergeSections();
3122 try wasm.mergeTypes();3313 try wasm.mergeTypes();
3123 try wasm.initializeCallCtorsFunction();3314 try wasm.initializeCallCtorsFunction();
3315 try wasm.setupInitMemoryFunction();
3124 try wasm.setupTLSRelocationsFunction();3316 try wasm.setupTLSRelocationsFunction();
3125 try wasm.initializeTLSFunction();3317 try wasm.initializeTLSFunction();
3126 try wasm.setupExports();3318 try wasm.setupExports();
...@@ -4473,6 +4665,17 @@ fn emitDataRelocations(...@@ -4473,6 +4665,17 @@ fn emitDataRelocations(
4473 try writeCustomSectionHeader(binary_bytes.items, header_offset, size);4665 try writeCustomSectionHeader(binary_bytes.items, header_offset, size);
4474}4666}
44754667
4668fn hasPassiveInitializationSegments(wasm: *const Wasm) bool {
4669 var it = wasm.data_segments.iterator();
4670 while (it.next()) |entry| {
4671 const segment: Segment = wasm.segments.items[entry.value_ptr.*];
4672 if (segment.needsPassiveInitialization(wasm.base.options.import_memory, entry.key_ptr.*)) {
4673 return true;
4674 }
4675 }
4676 return false;
4677}
4678
4476pub fn getTypeIndex(wasm: *const Wasm, func_type: std.wasm.Type) ?u32 {4679pub fn getTypeIndex(wasm: *const Wasm, func_type: std.wasm.Type) ?u32 {
4477 var index: u32 = 0;4680 var index: u32 = 0;
4478 while (index < wasm.func_types.items.len) : (index += 1) {4681 while (index < wasm.func_types.items.len) : (index += 1) {