| ... | ... | @@ -125,6 +125,8 @@ exports: std.ArrayListUnmanaged(types.Export) = .{}, |
| 125 | 125 | /// List of initialization functions. These must be called in order of priority |
| 126 | 126 | /// by the (synthetic) __wasm_call_ctors function. |
| 127 | 127 | init_funcs: std.ArrayListUnmanaged(InitFuncLoc) = .{}, |
| 128 | /// Index to a function defining the entry of the wasm file |
| 129 | entry: ?u32 = null, |
| 128 | 130 | |
| 129 | 131 | /// Indirect function table, used to call function pointers |
| 130 | 132 | /// When this is non-zero, we must emit a table entry, |
| ... | ... | @@ -409,7 +411,7 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option |
| 409 | 411 | }, |
| 410 | 412 | ); |
| 411 | 413 | } else { |
| 412 | | symbol.index = @as(u32, @intCast(wasm_bin.imported_globals_count + wasm_bin.wasm_globals.items.len)); |
| 414 | symbol.index = @intCast(wasm_bin.imported_globals_count + wasm_bin.wasm_globals.items.len); |
| 413 | 415 | symbol.setFlag(.WASM_SYM_VISIBILITY_HIDDEN); |
| 414 | 416 | const global = try wasm_bin.wasm_globals.addOne(allocator); |
| 415 | 417 | global.* = .{ |
| ... | ... | @@ -432,7 +434,7 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option |
| 432 | 434 | }; |
| 433 | 435 | if (options.output_mode == .Obj or options.import_table) { |
| 434 | 436 | symbol.setUndefined(true); |
| 435 | | symbol.index = @as(u32, @intCast(wasm_bin.imported_tables_count)); |
| 437 | symbol.index = @intCast(wasm_bin.imported_tables_count); |
| 436 | 438 | wasm_bin.imported_tables_count += 1; |
| 437 | 439 | try wasm_bin.imports.put(allocator, loc, .{ |
| 438 | 440 | .module_name = try wasm_bin.string_table.put(allocator, wasm_bin.host_name), |
| ... | ... | @@ -466,19 +468,34 @@ pub fn openPath(allocator: Allocator, sub_path: []const u8, options: link.Option |
| 466 | 468 | const loc = try wasm_bin.createSyntheticSymbol("__tls_base", .global); |
| 467 | 469 | const symbol = loc.getSymbol(wasm_bin); |
| 468 | 470 | symbol.setFlag(.WASM_SYM_VISIBILITY_HIDDEN); |
| 471 | symbol.index = @intCast(wasm_bin.imported_globals_count + wasm_bin.wasm_globals.items.len); |
| 472 | try wasm_bin.wasm_globals.append(wasm_bin.base.allocator, .{ |
| 473 | .global_type = .{ .valtype = .i32, .mutable = true }, |
| 474 | .init = .{ .i32_const = undefined }, |
| 475 | }); |
| 469 | 476 | } |
| 470 | 477 | { |
| 471 | 478 | const loc = try wasm_bin.createSyntheticSymbol("__tls_size", .global); |
| 472 | 479 | const symbol = loc.getSymbol(wasm_bin); |
| 473 | 480 | symbol.setFlag(.WASM_SYM_VISIBILITY_HIDDEN); |
| 481 | symbol.index = @intCast(wasm_bin.imported_globals_count + wasm_bin.wasm_globals.items.len); |
| 482 | try wasm_bin.wasm_globals.append(wasm_bin.base.allocator, .{ |
| 483 | .global_type = .{ .valtype = .i32, .mutable = false }, |
| 484 | .init = .{ .i32_const = undefined }, |
| 485 | }); |
| 474 | 486 | } |
| 475 | 487 | { |
| 476 | 488 | const loc = try wasm_bin.createSyntheticSymbol("__tls_align", .global); |
| 477 | 489 | const symbol = loc.getSymbol(wasm_bin); |
| 478 | 490 | symbol.setFlag(.WASM_SYM_VISIBILITY_HIDDEN); |
| 491 | symbol.index = @intCast(wasm_bin.imported_globals_count + wasm_bin.wasm_globals.items.len); |
| 492 | try wasm_bin.wasm_globals.append(wasm_bin.base.allocator, .{ |
| 493 | .global_type = .{ .valtype = .i32, .mutable = false }, |
| 494 | .init = .{ .i32_const = undefined }, |
| 495 | }); |
| 479 | 496 | } |
| 480 | 497 | { |
| 481 | | const loc = try wasm_bin.createSyntheticSymbol("__wasm_tls_init", .function); |
| 498 | const loc = try wasm_bin.createSyntheticSymbol("__wasm_init_tls", .function); |
| 482 | 499 | const symbol = loc.getSymbol(wasm_bin); |
| 483 | 500 | symbol.setFlag(.WASM_SYM_VISIBILITY_HIDDEN); |
| 484 | 501 | } |
| ... | ... | @@ -844,6 +861,12 @@ fn resolveSymbolsInArchives(wasm: *Wasm) !void { |
| 844 | 861 | } |
| 845 | 862 | } |
| 846 | 863 | |
| 864 | /// Writes an unsigned 32-bit integer as a LEB128-encoded 'i32.const' value. |
| 865 | fn writeI32Const(writer: anytype, val: u32) !void { |
| 866 | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 867 | try leb.writeILEB128(writer, @as(i32, @bitCast(val))); |
| 868 | } |
| 869 | |
| 847 | 870 | fn setupInitMemoryFunction(wasm: *Wasm) !void { |
| 848 | 871 | // Passive segments are used to avoid memory being reinitialized on each |
| 849 | 872 | // thread's instantiation. These passive segments are initialized and |
| ... | ... | @@ -881,12 +904,9 @@ fn setupInitMemoryFunction(wasm: *Wasm) !void { |
| 881 | 904 | try writer.writeByte(std.wasm.block_empty); // block type |
| 882 | 905 | |
| 883 | 906 | // atomically check |
| 884 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 885 | | try leb.writeULEB128(writer, flag_address); |
| 886 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 887 | | try leb.writeULEB128(writer, @as(u32, 0)); |
| 888 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 889 | | try leb.writeULEB128(writer, @as(u32, 1)); |
| 907 | try writeI32Const(writer, flag_address); |
| 908 | try writeI32Const(writer, 0); |
| 909 | try writeI32Const(writer, 1); |
| 890 | 910 | try writer.writeByte(std.wasm.opcode(.atomics_prefix)); |
| 891 | 911 | try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.i32_atomic_rmw_cmpxchg)); |
| 892 | 912 | try leb.writeULEB128(writer, @as(u32, 2)); // alignment |
| ... | ... | @@ -910,24 +930,20 @@ fn setupInitMemoryFunction(wasm: *Wasm) !void { |
| 910 | 930 | // For non-BSS segments we do a memory.init. Both these |
| 911 | 931 | // instructions take as their first argument the destination |
| 912 | 932 | // address. |
| 913 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 914 | | try leb.writeULEB128(writer, segment.offset); |
| 933 | try writeI32Const(writer, segment.offset); |
| 915 | 934 | |
| 916 | 935 | if (wasm.base.options.shared_memory and std.mem.eql(u8, entry.key_ptr.*, ".tdata")) { |
| 917 | 936 | // When we initialize the TLS segment we also set the `__tls_base` |
| 918 | 937 | // global. This allows the runtime to use this static copy of the |
| 919 | 938 | // TLS data for the first/main thread. |
| 920 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 921 | | try leb.writeULEB128(writer, segment.offset); |
| 939 | try writeI32Const(writer, segment.offset); |
| 922 | 940 | try writer.writeByte(std.wasm.opcode(.global_set)); |
| 923 | 941 | const loc = wasm.findGlobalSymbol("__tls_base").?; |
| 924 | 942 | try leb.writeULEB128(writer, loc.getSymbol(wasm).index); |
| 925 | 943 | } |
| 926 | 944 | |
| 927 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 928 | | try leb.writeULEB128(writer, @as(u32, 0)); |
| 929 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 930 | | try leb.writeULEB128(writer, segment.size); |
| 945 | try writeI32Const(writer, 0); |
| 946 | try writeI32Const(writer, segment.size); |
| 931 | 947 | try writer.writeByte(std.wasm.opcode(.misc_prefix)); |
| 932 | 948 | if (std.mem.eql(u8, entry.key_ptr.*, ".bss")) { |
| 933 | 949 | // fill bss segment with zeroes |
| ... | ... | @@ -943,18 +959,15 @@ fn setupInitMemoryFunction(wasm: *Wasm) !void { |
| 943 | 959 | |
| 944 | 960 | if (wasm.base.options.shared_memory) { |
| 945 | 961 | // we set the init memory flag to value '2' |
| 946 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 947 | | try leb.writeULEB128(writer, flag_address); |
| 948 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 949 | | try leb.writeULEB128(writer, @as(u32, 2)); |
| 962 | try writeI32Const(writer, flag_address); |
| 963 | try writeI32Const(writer, 2); |
| 950 | 964 | try writer.writeByte(std.wasm.opcode(.atomics_prefix)); |
| 951 | 965 | try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.i32_atomic_store)); |
| 952 | 966 | try leb.writeULEB128(writer, @as(u32, 2)); // alignment |
| 953 | 967 | try leb.writeULEB128(writer, @as(u32, 0)); // offset |
| 954 | 968 | |
| 955 | 969 | // notify any waiters for segment initialization completion |
| 956 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 957 | | try leb.writeULEB128(writer, flag_address); |
| 970 | try writeI32Const(writer, flag_address); |
| 958 | 971 | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 959 | 972 | try leb.writeILEB128(writer, @as(i32, -1)); // number of waiters |
| 960 | 973 | try writer.writeByte(std.wasm.opcode(.atomics_prefix)); |
| ... | ... | @@ -969,12 +982,10 @@ fn setupInitMemoryFunction(wasm: *Wasm) !void { |
| 969 | 982 | |
| 970 | 983 | // wait for thread to initialize memory segments |
| 971 | 984 | try writer.writeByte(std.wasm.opcode(.end)); // end $wait |
| 972 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 973 | | try leb.writeULEB128(writer, flag_address); |
| 974 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 975 | | try leb.writeULEB128(writer, @as(u32, 1)); // expected flag value |
| 976 | | try writer.writeByte(std.wasm.opcode(.i32_const)); |
| 977 | | try leb.writeILEB128(writer, @as(i32, -1)); // timeout |
| 985 | try writeI32Const(writer, flag_address); |
| 986 | try writeI32Const(writer, 1); // expected flag value |
| 987 | try writer.writeByte(std.wasm.opcode(.i64_const)); |
| 988 | try leb.writeILEB128(writer, @as(i64, -1)); // timeout |
| 978 | 989 | try writer.writeByte(std.wasm.opcode(.atomics_prefix)); |
| 979 | 990 | try leb.writeULEB128(writer, std.wasm.atomicsOpcode(.memory_atomic_wait32)); |
| 980 | 991 | try leb.writeULEB128(writer, @as(u32, 2)); // alignment |
| ... | ... | @@ -2154,7 +2165,14 @@ fn allocateVirtualAddresses(wasm: *Wasm) void { |
| 2154 | 2165 | const segment_name = segment_info[symbol.index].outputName(merge_segment); |
| 2155 | 2166 | const segment_index = wasm.data_segments.get(segment_name).?; |
| 2156 | 2167 | const segment = wasm.segments.items[segment_index]; |
| 2157 | | symbol.virtual_address = atom.offset + segment.offset; |
| 2168 | |
| 2169 | // TLS symbols have their virtual address set relative to their own TLS segment, |
| 2170 | // rather than the entire Data section. |
| 2171 | if (symbol.hasFlag(.WASM_SYM_TLS)) { |
| 2172 | symbol.virtual_address = atom.offset; |
| 2173 | } else { |
| 2174 | symbol.virtual_address = atom.offset + segment.offset; |
| 2175 | } |
| 2158 | 2176 | } |
| 2159 | 2177 | } |
| 2160 | 2178 | |
| ... | ... | @@ -2168,7 +2186,7 @@ fn sortDataSegments(wasm: *Wasm) !void { |
| 2168 | 2186 | |
| 2169 | 2187 | const SortContext = struct { |
| 2170 | 2188 | fn sort(_: void, lhs: []const u8, rhs: []const u8) bool { |
| 2171 | | return order(lhs) <= order(rhs); |
| 2189 | return order(lhs) < order(rhs); |
| 2172 | 2190 | } |
| 2173 | 2191 | |
| 2174 | 2192 | fn order(name: []const u8) u8 { |
| ... | ... | @@ -2399,6 +2417,13 @@ pub fn createFunction( |
| 2399 | 2417 | return loc.index; |
| 2400 | 2418 | } |
| 2401 | 2419 | |
| 2420 | /// If required, sets the function index in the `start` section. |
| 2421 | fn setupStartSection(wasm: *Wasm) !void { |
| 2422 | if (wasm.findGlobalSymbol("__wasm_init_memory")) |loc| { |
| 2423 | wasm.entry = loc.getSymbol(wasm).index; |
| 2424 | } |
| 2425 | } |
| 2426 | |
| 2402 | 2427 | fn initializeTLSFunction(wasm: *Wasm) !void { |
| 2403 | 2428 | if (!wasm.base.options.shared_memory) return; |
| 2404 | 2429 | |
| ... | ... | @@ -2420,7 +2445,7 @@ fn initializeTLSFunction(wasm: *Wasm) !void { |
| 2420 | 2445 | try leb.writeULEB128(writer, param_local); |
| 2421 | 2446 | |
| 2422 | 2447 | const tls_base_loc = wasm.findGlobalSymbol("__tls_base").?; |
| 2423 | | try writer.writeByte(std.wasm.opcode(.global_get)); |
| 2448 | try writer.writeByte(std.wasm.opcode(.global_set)); |
| 2424 | 2449 | try leb.writeULEB128(writer, tls_base_loc.getSymbol(wasm).index); |
| 2425 | 2450 | |
| 2426 | 2451 | // load stack values for the bulk-memory operation |
| ... | ... | @@ -2748,27 +2773,18 @@ fn setupMemory(wasm: *Wasm) !void { |
| 2748 | 2773 | if (mem.eql(u8, entry.key_ptr.*, ".tdata")) { |
| 2749 | 2774 | if (wasm.findGlobalSymbol("__tls_size")) |loc| { |
| 2750 | 2775 | const sym = loc.getSymbol(wasm); |
| 2751 | | sym.index = @as(u32, @intCast(wasm.wasm_globals.items.len)) + wasm.imported_globals_count; |
| 2752 | | try wasm.wasm_globals.append(wasm.base.allocator, .{ |
| 2753 | | .global_type = .{ .valtype = .i32, .mutable = false }, |
| 2754 | | .init = .{ .i32_const = @as(i32, @intCast(segment.size)) }, |
| 2755 | | }); |
| 2776 | wasm.wasm_globals.items[sym.index - wasm.imported_globals_count].init.i32_const = @intCast(segment.size); |
| 2756 | 2777 | } |
| 2757 | 2778 | if (wasm.findGlobalSymbol("__tls_align")) |loc| { |
| 2758 | 2779 | const sym = loc.getSymbol(wasm); |
| 2759 | | sym.index = @as(u32, @intCast(wasm.wasm_globals.items.len)) + wasm.imported_globals_count; |
| 2760 | | try wasm.wasm_globals.append(wasm.base.allocator, .{ |
| 2761 | | .global_type = .{ .valtype = .i32, .mutable = false }, |
| 2762 | | .init = .{ .i32_const = @as(i32, @intCast(segment.alignment)) }, |
| 2763 | | }); |
| 2780 | wasm.wasm_globals.items[sym.index - wasm.imported_globals_count].init.i32_const = @intCast(segment.alignment); |
| 2764 | 2781 | } |
| 2765 | 2782 | if (wasm.findGlobalSymbol("__tls_base")) |loc| { |
| 2766 | 2783 | const sym = loc.getSymbol(wasm); |
| 2767 | | sym.index = @as(u32, @intCast(wasm.wasm_globals.items.len)) + wasm.imported_globals_count; |
| 2768 | | try wasm.wasm_globals.append(wasm.base.allocator, .{ |
| 2769 | | .global_type = .{ .valtype = .i32, .mutable = wasm.base.options.shared_memory }, |
| 2770 | | .init = .{ .i32_const = if (wasm.base.options.shared_memory) @as(u32, 0) else @as(i32, @intCast(memory_ptr)) }, |
| 2771 | | }); |
| 2784 | wasm.wasm_globals.items[sym.index - wasm.imported_globals_count].init.i32_const = if (wasm.base.options.shared_memory) |
| 2785 | @as(i32, 0) |
| 2786 | else |
| 2787 | @as(i32, @intCast(memory_ptr)); |
| 2772 | 2788 | } |
| 2773 | 2789 | } |
| 2774 | 2790 | |
| ... | ... | @@ -3323,6 +3339,7 @@ fn linkWithZld(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) l |
| 3323 | 3339 | try wasm.setupInitMemoryFunction(); |
| 3324 | 3340 | try wasm.setupTLSRelocationsFunction(); |
| 3325 | 3341 | try wasm.initializeTLSFunction(); |
| 3342 | try wasm.setupStartSection(); |
| 3326 | 3343 | try wasm.setupExports(); |
| 3327 | 3344 | try wasm.writeToFile(enabled_features, emit_features_count, arena); |
| 3328 | 3345 | |
| ... | ... | @@ -3460,6 +3477,7 @@ pub fn flushModule(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Nod |
| 3460 | 3477 | try wasm.setupInitMemoryFunction(); |
| 3461 | 3478 | try wasm.setupTLSRelocationsFunction(); |
| 3462 | 3479 | try wasm.initializeTLSFunction(); |
| 3480 | try wasm.setupStartSection(); |
| 3463 | 3481 | try wasm.setupExports(); |
| 3464 | 3482 | try wasm.writeToFile(enabled_features, emit_features_count, arena); |
| 3465 | 3483 | } |
| ... | ... | @@ -3520,6 +3538,7 @@ fn writeToFile( |
| 3520 | 3538 | |
| 3521 | 3539 | // Import section |
| 3522 | 3540 | const import_memory = wasm.base.options.import_memory or is_obj; |
| 3541 | const export_memory = wasm.base.options.export_memory; |
| 3523 | 3542 | if (wasm.imports.count() != 0 or import_memory) { |
| 3524 | 3543 | const header_offset = try reserveVecSectionHeader(&binary_bytes); |
| 3525 | 3544 | |
| ... | ... | @@ -3622,7 +3641,7 @@ fn writeToFile( |
| 3622 | 3641 | } |
| 3623 | 3642 | |
| 3624 | 3643 | // Export section |
| 3625 | | if (wasm.exports.items.len != 0 or !import_memory) { |
| 3644 | if (wasm.exports.items.len != 0 or export_memory) { |
| 3626 | 3645 | const header_offset = try reserveVecSectionHeader(&binary_bytes); |
| 3627 | 3646 | |
| 3628 | 3647 | for (wasm.exports.items) |exp| { |
| ... | ... | @@ -3633,7 +3652,7 @@ fn writeToFile( |
| 3633 | 3652 | try leb.writeULEB128(binary_writer, exp.index); |
| 3634 | 3653 | } |
| 3635 | 3654 | |
| 3636 | | if (!import_memory) { |
| 3655 | if (export_memory) { |
| 3637 | 3656 | try leb.writeULEB128(binary_writer, @as(u32, @intCast("memory".len))); |
| 3638 | 3657 | try binary_writer.writeAll("memory"); |
| 3639 | 3658 | try binary_writer.writeByte(std.wasm.externalKind(.memory)); |
| ... | ... | @@ -3645,11 +3664,22 @@ fn writeToFile( |
| 3645 | 3664 | header_offset, |
| 3646 | 3665 | .@"export", |
| 3647 | 3666 | @as(u32, @intCast(binary_bytes.items.len - header_offset - header_size)), |
| 3648 | | @as(u32, @intCast(wasm.exports.items.len)) + @intFromBool(!import_memory), |
| 3667 | @as(u32, @intCast(wasm.exports.items.len)) + @intFromBool(export_memory), |
| 3649 | 3668 | ); |
| 3650 | 3669 | section_count += 1; |
| 3651 | 3670 | } |
| 3652 | 3671 | |
| 3672 | if (wasm.entry) |entry_index| { |
| 3673 | const header_offset = try reserveVecSectionHeader(&binary_bytes); |
| 3674 | try writeVecSectionHeader( |
| 3675 | binary_bytes.items, |
| 3676 | header_offset, |
| 3677 | .start, |
| 3678 | @intCast(binary_bytes.items.len - header_offset - header_size), |
| 3679 | entry_index, |
| 3680 | ); |
| 3681 | } |
| 3682 | |
| 3653 | 3683 | // element section (function table) |
| 3654 | 3684 | if (wasm.function_table.count() > 0) { |
| 3655 | 3685 | const header_offset = try reserveVecSectionHeader(&binary_bytes); |
| ... | ... | @@ -3683,7 +3713,7 @@ fn writeToFile( |
| 3683 | 3713 | } |
| 3684 | 3714 | |
| 3685 | 3715 | // When the shared-memory option is enabled, we *must* emit the 'data count' section. |
| 3686 | | const data_segments_count = wasm.data_segments.count() - @intFromBool(wasm.data_segments.contains(".bss") and import_memory); |
| 3716 | const data_segments_count = wasm.data_segments.count() - @intFromBool(wasm.data_segments.contains(".bss") and !import_memory); |
| 3687 | 3717 | if (data_segments_count != 0 and wasm.base.options.shared_memory) { |
| 3688 | 3718 | const header_offset = try reserveVecSectionHeader(&binary_bytes); |
| 3689 | 3719 | try writeVecSectionHeader( |