| ... | ... | @@ -764,7 +764,6 @@ pub const Object = struct { |
| 764 | 764 | builder: Builder, |
| 765 | 765 | |
| 766 | 766 | module: *Module, |
| 767 | | llvm_module: *llvm.Module, |
| 768 | 767 | di_builder: ?*llvm.DIBuilder, |
| 769 | 768 | /// One of these mappings: |
| 770 | 769 | /// - *Module.File => *DIFile |
| ... | ... | @@ -945,9 +944,8 @@ pub const Object = struct { |
| 945 | 944 | .gpa = gpa, |
| 946 | 945 | .builder = builder, |
| 947 | 946 | .module = options.module.?, |
| 948 | | .llvm_module = builder.llvm.module.?, |
| 949 | 947 | .di_map = .{}, |
| 950 | | .di_builder = builder.llvm.di_builder, |
| 948 | .di_builder = if (builder.useLibLlvm()) builder.llvm.di_builder else null, // TODO |
| 951 | 949 | .di_compile_unit = builder.llvm.di_compile_unit, |
| 952 | 950 | .target_machine = target_machine, |
| 953 | 951 | .target_data = target_data, |
| ... | ... | @@ -991,9 +989,8 @@ pub const Object = struct { |
| 991 | 989 | } |
| 992 | 990 | |
| 993 | 991 | fn genErrorNameTable(o: *Object) Allocator.Error!void { |
| 994 | | // If o.error_name_table is null, there was no instruction that actually referenced the error table. |
| 995 | | const error_name_table_ptr_global = o.error_name_table; |
| 996 | | if (error_name_table_ptr_global == .none) return; |
| 992 | // If o.error_name_table is null, then it was not referenced by any instructions. |
| 993 | if (o.error_name_table == .none) return; |
| 997 | 994 | |
| 998 | 995 | const mod = o.module; |
| 999 | 996 | |
| ... | ... | @@ -1003,72 +1000,42 @@ pub const Object = struct { |
| 1003 | 1000 | |
| 1004 | 1001 | // TODO: Address space |
| 1005 | 1002 | const slice_ty = Type.slice_const_u8_sentinel_0; |
| 1006 | | const slice_alignment = slice_ty.abiAlignment(mod); |
| 1007 | 1003 | const llvm_usize_ty = try o.lowerType(Type.usize); |
| 1008 | 1004 | const llvm_slice_ty = try o.lowerType(slice_ty); |
| 1009 | 1005 | const llvm_table_ty = try o.builder.arrayType(error_name_list.len, llvm_slice_ty); |
| 1010 | 1006 | |
| 1011 | 1007 | llvm_errors[0] = try o.builder.undefConst(llvm_slice_ty); |
| 1012 | | for (llvm_errors[1..], error_name_list[1..]) |*llvm_error, name_nts| { |
| 1013 | | const name = try o.builder.string(mod.intern_pool.stringToSlice(name_nts)); |
| 1014 | | const str_init = try o.builder.stringNullConst(name); |
| 1015 | | const str_ty = str_init.typeOf(&o.builder); |
| 1016 | | const str_llvm_global = o.llvm_module.addGlobal(str_ty.toLlvm(&o.builder), ""); |
| 1017 | | str_llvm_global.setInitializer(str_init.toLlvm(&o.builder)); |
| 1018 | | str_llvm_global.setLinkage(.Private); |
| 1019 | | str_llvm_global.setGlobalConstant(.True); |
| 1020 | | str_llvm_global.setUnnamedAddr(.True); |
| 1021 | | str_llvm_global.setAlignment(1); |
| 1022 | | |
| 1023 | | var str_global = Builder.Global{ |
| 1024 | | .linkage = .private, |
| 1025 | | .unnamed_addr = .unnamed_addr, |
| 1026 | | .type = str_ty, |
| 1027 | | .kind = .{ .variable = @enumFromInt(o.builder.variables.items.len) }, |
| 1028 | | }; |
| 1029 | | var str_variable = Builder.Variable{ |
| 1030 | | .global = @enumFromInt(o.builder.globals.count()), |
| 1031 | | .mutability = .constant, |
| 1032 | | .init = str_init, |
| 1033 | | .alignment = comptime Builder.Alignment.fromByteUnits(1), |
| 1034 | | }; |
| 1035 | | try o.builder.llvm.globals.append(o.gpa, str_llvm_global); |
| 1036 | | const global_index = try o.builder.addGlobal(.empty, str_global); |
| 1037 | | try o.builder.variables.append(o.gpa, str_variable); |
| 1008 | for (llvm_errors[1..], error_name_list[1..]) |*llvm_error, name| { |
| 1009 | const name_string = try o.builder.string(mod.intern_pool.stringToSlice(name)); |
| 1010 | const name_init = try o.builder.stringNullConst(name_string); |
| 1011 | const name_variable_index = |
| 1012 | try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); |
| 1013 | try name_variable_index.setInitializer(name_init, &o.builder); |
| 1014 | name_variable_index.setLinkage(.private, &o.builder); |
| 1015 | name_variable_index.setMutability(.constant, &o.builder); |
| 1016 | name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 1017 | name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); |
| 1038 | 1018 | |
| 1039 | 1019 | llvm_error.* = try o.builder.structConst(llvm_slice_ty, &.{ |
| 1040 | | global_index.toConst(), |
| 1041 | | try o.builder.intConst(llvm_usize_ty, name.slice(&o.builder).?.len), |
| 1020 | name_variable_index.toConst(&o.builder), |
| 1021 | try o.builder.intConst(llvm_usize_ty, name_string.slice(&o.builder).?.len), |
| 1042 | 1022 | }); |
| 1043 | 1023 | } |
| 1044 | 1024 | |
| 1045 | | const error_name_table_init = try o.builder.arrayConst(llvm_table_ty, llvm_errors); |
| 1046 | | const error_name_table_global = o.llvm_module.addGlobal(llvm_table_ty.toLlvm(&o.builder), ""); |
| 1047 | | error_name_table_global.setInitializer(error_name_table_init.toLlvm(&o.builder)); |
| 1048 | | error_name_table_global.setLinkage(.Private); |
| 1049 | | error_name_table_global.setGlobalConstant(.True); |
| 1050 | | error_name_table_global.setUnnamedAddr(.True); |
| 1051 | | error_name_table_global.setAlignment(slice_alignment); // TODO: Dont hardcode |
| 1052 | | |
| 1053 | | var global = Builder.Global{ |
| 1054 | | .linkage = .private, |
| 1055 | | .unnamed_addr = .unnamed_addr, |
| 1056 | | .type = llvm_table_ty, |
| 1057 | | .kind = .{ .variable = @enumFromInt(o.builder.variables.items.len) }, |
| 1058 | | }; |
| 1059 | | var variable = Builder.Variable{ |
| 1060 | | .global = @enumFromInt(o.builder.globals.count()), |
| 1061 | | .mutability = .constant, |
| 1062 | | .init = error_name_table_init, |
| 1063 | | .alignment = Builder.Alignment.fromByteUnits(slice_alignment), |
| 1064 | | }; |
| 1065 | | try o.builder.llvm.globals.append(o.gpa, error_name_table_global); |
| 1066 | | _ = try o.builder.addGlobal(.empty, global); |
| 1067 | | try o.builder.variables.append(o.gpa, variable); |
| 1025 | const table_variable_index = try o.builder.addVariable(.empty, llvm_table_ty, .default); |
| 1026 | try table_variable_index.setInitializer( |
| 1027 | try o.builder.arrayConst(llvm_table_ty, llvm_errors), |
| 1028 | &o.builder, |
| 1029 | ); |
| 1030 | table_variable_index.setLinkage(.private, &o.builder); |
| 1031 | table_variable_index.setMutability(.constant, &o.builder); |
| 1032 | table_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 1033 | table_variable_index.setAlignment( |
| 1034 | Builder.Alignment.fromByteUnits(slice_ty.abiAlignment(mod)), |
| 1035 | &o.builder, |
| 1036 | ); |
| 1068 | 1037 | |
| 1069 | | const error_name_table_ptr = error_name_table_global; |
| 1070 | | error_name_table_ptr_global.ptr(&o.builder).init = variable.global.toConst(); |
| 1071 | | error_name_table_ptr_global.toLlvm(&o.builder).setInitializer(error_name_table_ptr); |
| 1038 | try o.error_name_table.setInitializer(table_variable_index.toConst(&o.builder), &o.builder); |
| 1072 | 1039 | } |
| 1073 | 1040 | |
| 1074 | 1041 | fn genCmpLtErrorsLenFunction(o: *Object) !void { |
| ... | ... | @@ -1181,17 +1148,7 @@ pub const Object = struct { |
| 1181 | 1148 | } |
| 1182 | 1149 | } |
| 1183 | 1150 | |
| 1184 | | if (comp.verbose_llvm_bc) |path| { |
| 1185 | | const path_z = try comp.gpa.dupeZ(u8, path); |
| 1186 | | defer comp.gpa.free(path_z); |
| 1187 | | |
| 1188 | | const error_code = self.llvm_module.writeBitcodeToFile(path_z); |
| 1189 | | if (error_code != 0) { |
| 1190 | | log.err("dump LLVM module failed bc={s}: {d}", .{ |
| 1191 | | path, error_code, |
| 1192 | | }); |
| 1193 | | } |
| 1194 | | } |
| 1151 | if (comp.verbose_llvm_bc) |path| _ = try self.builder.writeBitcodeToFile(path); |
| 1195 | 1152 | |
| 1196 | 1153 | var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); |
| 1197 | 1154 | defer arena_allocator.deinit(); |
| ... | ... | @@ -1200,20 +1157,10 @@ pub const Object = struct { |
| 1200 | 1157 | const mod = comp.bin_file.options.module.?; |
| 1201 | 1158 | const cache_dir = mod.zig_cache_artifact_directory; |
| 1202 | 1159 | |
| 1203 | | if (std.debug.runtime_safety) { |
| 1204 | | var error_message: [*:0]const u8 = undefined; |
| 1205 | | // verifyModule always allocs the error_message even if there is no error |
| 1206 | | defer llvm.disposeMessage(error_message); |
| 1207 | | |
| 1208 | | if (self.llvm_module.verify(.ReturnStatus, &error_message).toBool()) { |
| 1209 | | std.debug.print("\n{s}\n", .{error_message}); |
| 1210 | | |
| 1211 | | if (try locPath(arena, comp.emit_llvm_ir, cache_dir)) |emit_llvm_ir_path| { |
| 1212 | | _ = self.llvm_module.printModuleToFile(emit_llvm_ir_path, &error_message); |
| 1213 | | } |
| 1214 | | |
| 1215 | | @panic("LLVM module verification failed"); |
| 1216 | | } |
| 1160 | if (std.debug.runtime_safety and !try self.builder.verify()) { |
| 1161 | if (try locPath(arena, comp.emit_llvm_ir, cache_dir)) |emit_llvm_ir_path| |
| 1162 | _ = self.builder.printToFileZ(emit_llvm_ir_path); |
| 1163 | @panic("LLVM module verification failed"); |
| 1217 | 1164 | } |
| 1218 | 1165 | |
| 1219 | 1166 | var emit_bin_path: ?[*:0]const u8 = if (comp.bin_file.options.emit) |emit| |
| ... | ... | @@ -1233,12 +1180,17 @@ pub const Object = struct { |
| 1233 | 1180 | emit_asm_msg, emit_bin_msg, emit_llvm_ir_msg, emit_llvm_bc_msg, |
| 1234 | 1181 | }); |
| 1235 | 1182 | |
| 1183 | if (!self.builder.useLibLlvm()) { |
| 1184 | log.err("emitting without libllvm not implemented", .{}); |
| 1185 | return error.FailedToEmit; |
| 1186 | } |
| 1187 | |
| 1236 | 1188 | // Unfortunately, LLVM shits the bed when we ask for both binary and assembly. |
| 1237 | 1189 | // So we call the entire pipeline multiple times if this is requested. |
| 1238 | 1190 | var error_message: [*:0]const u8 = undefined; |
| 1239 | 1191 | if (emit_asm_path != null and emit_bin_path != null) { |
| 1240 | 1192 | if (self.target_machine.emitToFile( |
| 1241 | | self.llvm_module, |
| 1193 | self.builder.llvm.module.?, |
| 1242 | 1194 | &error_message, |
| 1243 | 1195 | comp.bin_file.options.optimize_mode == .Debug, |
| 1244 | 1196 | comp.bin_file.options.optimize_mode == .ReleaseSmall, |
| ... | ... | @@ -1262,7 +1214,7 @@ pub const Object = struct { |
| 1262 | 1214 | } |
| 1263 | 1215 | |
| 1264 | 1216 | if (self.target_machine.emitToFile( |
| 1265 | | self.llvm_module, |
| 1217 | self.builder.llvm.module.?, |
| 1266 | 1218 | &error_message, |
| 1267 | 1219 | comp.bin_file.options.optimize_mode == .Debug, |
| 1268 | 1220 | comp.bin_file.options.optimize_mode == .ReleaseSmall, |
| ... | ... | @@ -1305,11 +1257,9 @@ pub const Object = struct { |
| 1305 | 1257 | .err_msg = null, |
| 1306 | 1258 | }; |
| 1307 | 1259 | |
| 1308 | | const function = try o.resolveLlvmFunction(decl_index); |
| 1309 | | const global = function.ptrConst(&o.builder).global; |
| 1310 | | const llvm_func = global.toLlvm(&o.builder); |
| 1260 | const function_index = try o.resolveLlvmFunction(decl_index); |
| 1311 | 1261 | |
| 1312 | | var attributes = try function.ptrConst(&o.builder).attributes.toWip(&o.builder); |
| 1262 | var attributes = try function_index.ptrConst(&o.builder).attributes.toWip(&o.builder); |
| 1313 | 1263 | defer attributes.deinit(&o.builder); |
| 1314 | 1264 | |
| 1315 | 1265 | if (func.analysis(ip).is_noinline) { |
| ... | ... | @@ -1354,17 +1304,14 @@ pub const Object = struct { |
| 1354 | 1304 | } }, &o.builder); |
| 1355 | 1305 | } |
| 1356 | 1306 | |
| 1357 | | if (ip.stringToSliceUnwrap(decl.@"linksection")) |section| { |
| 1358 | | function.ptr(&o.builder).section = try o.builder.string(section); |
| 1359 | | llvm_func.setSection(section); |
| 1360 | | } |
| 1307 | if (ip.stringToSliceUnwrap(decl.@"linksection")) |section| |
| 1308 | function_index.setSection(try o.builder.string(section), &o.builder); |
| 1361 | 1309 | |
| 1362 | 1310 | var deinit_wip = true; |
| 1363 | | var wip = try Builder.WipFunction.init(&o.builder, function); |
| 1311 | var wip = try Builder.WipFunction.init(&o.builder, function_index); |
| 1364 | 1312 | defer if (deinit_wip) wip.deinit(); |
| 1365 | 1313 | wip.cursor = .{ .block = try wip.block(0, "Entry") }; |
| 1366 | 1314 | |
| 1367 | | const builder = wip.llvm.builder; |
| 1368 | 1315 | var llvm_arg_i: u32 = 0; |
| 1369 | 1316 | |
| 1370 | 1317 | // This gets the LLVM values from the function and stores them in `dg.args`. |
| ... | ... | @@ -1566,7 +1513,7 @@ pub const Object = struct { |
| 1566 | 1513 | } |
| 1567 | 1514 | } |
| 1568 | 1515 | |
| 1569 | | function.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 1516 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 1570 | 1517 | |
| 1571 | 1518 | var di_file: ?*llvm.DIFile = null; |
| 1572 | 1519 | var di_scope: ?*llvm.DIScope = null; |
| ... | ... | @@ -1585,7 +1532,7 @@ pub const Object = struct { |
| 1585 | 1532 | const subprogram = dib.createFunction( |
| 1586 | 1533 | di_file.?.toScope(), |
| 1587 | 1534 | ip.stringToSlice(decl.name), |
| 1588 | | llvm_func.getValueName(), |
| 1535 | function_index.name(&o.builder).slice(&o.builder).?, |
| 1589 | 1536 | di_file.?, |
| 1590 | 1537 | line_number, |
| 1591 | 1538 | decl_di_ty, |
| ... | ... | @@ -1598,7 +1545,7 @@ pub const Object = struct { |
| 1598 | 1545 | ); |
| 1599 | 1546 | try o.di_map.put(gpa, decl, subprogram.toNode()); |
| 1600 | 1547 | |
| 1601 | | llvm_func.fnSetSubprogram(subprogram); |
| 1548 | function_index.toLlvm(&o.builder).fnSetSubprogram(subprogram); |
| 1602 | 1549 | |
| 1603 | 1550 | di_scope = subprogram.toScope(); |
| 1604 | 1551 | } |
| ... | ... | @@ -1609,7 +1556,6 @@ pub const Object = struct { |
| 1609 | 1556 | .liveness = liveness, |
| 1610 | 1557 | .dg = &dg, |
| 1611 | 1558 | .wip = wip, |
| 1612 | | .builder = builder, |
| 1613 | 1559 | .ret_ptr = ret_ptr, |
| 1614 | 1560 | .args = args.items, |
| 1615 | 1561 | .arg_index = 0, |
| ... | ... | @@ -1670,8 +1616,7 @@ pub const Object = struct { |
| 1670 | 1616 | const gpa = mod.gpa; |
| 1671 | 1617 | // If the module does not already have the function, we ignore this function call |
| 1672 | 1618 | // because we call `updateDeclExports` at the end of `updateFunc` and `updateDecl`. |
| 1673 | | const global = self.decl_map.get(decl_index) orelse return; |
| 1674 | | const llvm_global = global.toLlvm(&self.builder); |
| 1619 | const global_index = self.decl_map.get(decl_index) orelse return; |
| 1675 | 1620 | const decl = mod.declPtr(decl_index); |
| 1676 | 1621 | if (decl.isExtern(mod)) { |
| 1677 | 1622 | const decl_name = decl_name: { |
| ... | ... | @@ -1689,114 +1634,91 @@ pub const Object = struct { |
| 1689 | 1634 | }; |
| 1690 | 1635 | |
| 1691 | 1636 | if (self.builder.getGlobal(decl_name)) |other_global| { |
| 1692 | | if (other_global.toLlvm(&self.builder) != llvm_global) { |
| 1637 | if (other_global != global_index) { |
| 1693 | 1638 | try self.extern_collisions.put(gpa, decl_index, {}); |
| 1694 | 1639 | } |
| 1695 | 1640 | } |
| 1696 | 1641 | |
| 1697 | | try global.rename(decl_name, &self.builder); |
| 1698 | | global.ptr(&self.builder).unnamed_addr = .default; |
| 1699 | | llvm_global.setUnnamedAddr(.False); |
| 1700 | | global.ptr(&self.builder).linkage = .external; |
| 1701 | | llvm_global.setLinkage(.External); |
| 1702 | | if (mod.wantDllExports()) { |
| 1703 | | global.ptr(&self.builder).dll_storage_class = .default; |
| 1704 | | llvm_global.setDLLStorageClass(.Default); |
| 1705 | | } |
| 1642 | try global_index.rename(decl_name, &self.builder); |
| 1643 | global_index.setLinkage(.external, &self.builder); |
| 1644 | global_index.setUnnamedAddr(.default, &self.builder); |
| 1645 | if (mod.wantDllExports()) global_index.setDllStorageClass(.default, &self.builder); |
| 1706 | 1646 | if (self.di_map.get(decl)) |di_node| { |
| 1707 | 1647 | const decl_name_slice = decl_name.slice(&self.builder).?; |
| 1708 | 1648 | if (try decl.isFunction(mod)) { |
| 1709 | 1649 | const di_func: *llvm.DISubprogram = @ptrCast(di_node); |
| 1710 | | const linkage_name = llvm.MDString.get(self.builder.llvm.context, decl_name_slice.ptr, decl_name_slice.len); |
| 1650 | const linkage_name = llvm.MDString.get( |
| 1651 | self.builder.llvm.context, |
| 1652 | decl_name_slice.ptr, |
| 1653 | decl_name_slice.len, |
| 1654 | ); |
| 1711 | 1655 | di_func.replaceLinkageName(linkage_name); |
| 1712 | 1656 | } else { |
| 1713 | 1657 | const di_global: *llvm.DIGlobalVariable = @ptrCast(di_node); |
| 1714 | | const linkage_name = llvm.MDString.get(self.builder.llvm.context, decl_name_slice.ptr, decl_name_slice.len); |
| 1658 | const linkage_name = llvm.MDString.get( |
| 1659 | self.builder.llvm.context, |
| 1660 | decl_name_slice.ptr, |
| 1661 | decl_name_slice.len, |
| 1662 | ); |
| 1715 | 1663 | di_global.replaceLinkageName(linkage_name); |
| 1716 | 1664 | } |
| 1717 | 1665 | } |
| 1718 | 1666 | if (decl.val.getVariable(mod)) |decl_var| { |
| 1719 | | if (decl_var.is_threadlocal) { |
| 1720 | | global.ptrConst(&self.builder).kind.variable.ptr(&self.builder).thread_local = |
| 1721 | | .generaldynamic; |
| 1722 | | llvm_global.setThreadLocalMode(.GeneralDynamicTLSModel); |
| 1723 | | } else { |
| 1724 | | global.ptrConst(&self.builder).kind.variable.ptr(&self.builder).thread_local = |
| 1725 | | .default; |
| 1726 | | llvm_global.setThreadLocalMode(.NotThreadLocal); |
| 1727 | | } |
| 1728 | | if (decl_var.is_weak_linkage) { |
| 1729 | | global.ptr(&self.builder).linkage = .extern_weak; |
| 1730 | | llvm_global.setLinkage(.ExternalWeak); |
| 1731 | | } |
| 1667 | global_index.ptrConst(&self.builder).kind.variable.setThreadLocal( |
| 1668 | if (decl_var.is_threadlocal) .generaldynamic else .default, |
| 1669 | &self.builder, |
| 1670 | ); |
| 1671 | if (decl_var.is_weak_linkage) global_index.setLinkage(.extern_weak, &self.builder); |
| 1732 | 1672 | } |
| 1733 | | global.ptr(&self.builder).updateAttributes(); |
| 1734 | 1673 | } else if (exports.len != 0) { |
| 1735 | | const exp_name = try self.builder.string(mod.intern_pool.stringToSlice(exports[0].opts.name)); |
| 1736 | | try global.rename(exp_name, &self.builder); |
| 1737 | | global.ptr(&self.builder).unnamed_addr = .default; |
| 1738 | | llvm_global.setUnnamedAddr(.False); |
| 1739 | | if (mod.wantDllExports()) { |
| 1740 | | global.ptr(&self.builder).dll_storage_class = .dllexport; |
| 1741 | | llvm_global.setDLLStorageClass(.DLLExport); |
| 1742 | | } |
| 1674 | const main_exp_name = try self.builder.string( |
| 1675 | mod.intern_pool.stringToSlice(exports[0].opts.name), |
| 1676 | ); |
| 1677 | try global_index.rename(main_exp_name, &self.builder); |
| 1678 | global_index.setUnnamedAddr(.default, &self.builder); |
| 1679 | if (mod.wantDllExports()) global_index.setDllStorageClass(.dllexport, &self.builder); |
| 1743 | 1680 | if (self.di_map.get(decl)) |di_node| { |
| 1744 | | const exp_name_slice = exp_name.slice(&self.builder).?; |
| 1681 | const main_exp_name_slice = main_exp_name.slice(&self.builder).?; |
| 1745 | 1682 | if (try decl.isFunction(mod)) { |
| 1746 | 1683 | const di_func: *llvm.DISubprogram = @ptrCast(di_node); |
| 1747 | | const linkage_name = llvm.MDString.get(self.builder.llvm.context, exp_name_slice.ptr, exp_name_slice.len); |
| 1684 | const linkage_name = llvm.MDString.get( |
| 1685 | self.builder.llvm.context, |
| 1686 | main_exp_name_slice.ptr, |
| 1687 | main_exp_name_slice.len, |
| 1688 | ); |
| 1748 | 1689 | di_func.replaceLinkageName(linkage_name); |
| 1749 | 1690 | } else { |
| 1750 | 1691 | const di_global: *llvm.DIGlobalVariable = @ptrCast(di_node); |
| 1751 | | const linkage_name = llvm.MDString.get(self.builder.llvm.context, exp_name_slice.ptr, exp_name_slice.len); |
| 1692 | const linkage_name = llvm.MDString.get( |
| 1693 | self.builder.llvm.context, |
| 1694 | main_exp_name_slice.ptr, |
| 1695 | main_exp_name_slice.len, |
| 1696 | ); |
| 1752 | 1697 | di_global.replaceLinkageName(linkage_name); |
| 1753 | 1698 | } |
| 1754 | 1699 | } |
| 1755 | | switch (exports[0].opts.linkage) { |
| 1700 | global_index.setLinkage(switch (exports[0].opts.linkage) { |
| 1756 | 1701 | .Internal => unreachable, |
| 1757 | | .Strong => { |
| 1758 | | global.ptr(&self.builder).linkage = .external; |
| 1759 | | llvm_global.setLinkage(.External); |
| 1760 | | }, |
| 1761 | | .Weak => { |
| 1762 | | global.ptr(&self.builder).linkage = .weak_odr; |
| 1763 | | llvm_global.setLinkage(.WeakODR); |
| 1764 | | }, |
| 1765 | | .LinkOnce => { |
| 1766 | | global.ptr(&self.builder).linkage = .linkonce_odr; |
| 1767 | | llvm_global.setLinkage(.LinkOnceODR); |
| 1768 | | }, |
| 1769 | | } |
| 1770 | | switch (exports[0].opts.visibility) { |
| 1771 | | .default => { |
| 1772 | | global.ptr(&self.builder).visibility = .default; |
| 1773 | | llvm_global.setVisibility(.Default); |
| 1774 | | }, |
| 1775 | | .hidden => { |
| 1776 | | global.ptr(&self.builder).visibility = .hidden; |
| 1777 | | llvm_global.setVisibility(.Hidden); |
| 1778 | | }, |
| 1779 | | .protected => { |
| 1780 | | global.ptr(&self.builder).visibility = .protected; |
| 1781 | | llvm_global.setVisibility(.Protected); |
| 1782 | | }, |
| 1783 | | } |
| 1784 | | if (mod.intern_pool.stringToSliceUnwrap(exports[0].opts.section)) |section| { |
| 1785 | | switch (global.ptrConst(&self.builder).kind) { |
| 1786 | | inline .variable, .function => |impl_index| impl_index.ptr(&self.builder).section = |
| 1702 | .Strong => .external, |
| 1703 | .Weak => .weak_odr, |
| 1704 | .LinkOnce => .linkonce_odr, |
| 1705 | }, &self.builder); |
| 1706 | global_index.setVisibility(switch (exports[0].opts.visibility) { |
| 1707 | .default => .default, |
| 1708 | .hidden => .hidden, |
| 1709 | .protected => .protected, |
| 1710 | }, &self.builder); |
| 1711 | if (mod.intern_pool.stringToSliceUnwrap(exports[0].opts.section)) |section| |
| 1712 | switch (global_index.ptrConst(&self.builder).kind) { |
| 1713 | inline .variable, .function => |impl_index| impl_index.setSection( |
| 1787 | 1714 | try self.builder.string(section), |
| 1715 | &self.builder, |
| 1716 | ), |
| 1788 | 1717 | else => unreachable, |
| 1789 | | } |
| 1790 | | llvm_global.setSection(section); |
| 1791 | | } |
| 1792 | | if (decl.val.getVariable(mod)) |decl_var| { |
| 1793 | | if (decl_var.is_threadlocal) { |
| 1794 | | global.ptrConst(&self.builder).kind.variable.ptr(&self.builder).thread_local = |
| 1795 | | .generaldynamic; |
| 1796 | | llvm_global.setThreadLocalMode(.GeneralDynamicTLSModel); |
| 1797 | | } |
| 1798 | | } |
| 1799 | | global.ptr(&self.builder).updateAttributes(); |
| 1718 | }; |
| 1719 | if (decl.val.getVariable(mod)) |decl_var| if (decl_var.is_threadlocal) |
| 1720 | global_index.ptrConst(&self.builder).kind |
| 1721 | .variable.setThreadLocal(.generaldynamic, &self.builder); |
| 1800 | 1722 | |
| 1801 | 1723 | // If a Decl is exported more than one time (which is rare), |
| 1802 | 1724 | // we add aliases for all but the first export. |
| ... | ... | @@ -1805,49 +1727,47 @@ pub const Object = struct { |
| 1805 | 1727 | // Until then we iterate over existing aliases and make them point |
| 1806 | 1728 | // to the correct decl, or otherwise add a new alias. Old aliases are leaked. |
| 1807 | 1729 | for (exports[1..]) |exp| { |
| 1808 | | const exp_name_z = mod.intern_pool.stringToSlice(exp.opts.name); |
| 1809 | | |
| 1810 | | if (self.llvm_module.getNamedGlobalAlias(exp_name_z.ptr, exp_name_z.len)) |alias| { |
| 1811 | | alias.setAliasee(llvm_global); |
| 1812 | | } else { |
| 1813 | | _ = self.llvm_module.addAlias( |
| 1814 | | global.ptrConst(&self.builder).type.toLlvm(&self.builder), |
| 1815 | | 0, |
| 1816 | | llvm_global, |
| 1817 | | exp_name_z, |
| 1818 | | ); |
| 1730 | const exp_name = try self.builder.string(mod.intern_pool.stringToSlice(exp.opts.name)); |
| 1731 | if (self.builder.getGlobal(exp_name)) |global| { |
| 1732 | switch (global.ptrConst(&self.builder).kind) { |
| 1733 | .alias => |alias| { |
| 1734 | alias.setAliasee(global_index.toConst(), &self.builder); |
| 1735 | continue; |
| 1736 | }, |
| 1737 | .variable, .function => {}, |
| 1738 | else => unreachable, |
| 1739 | } |
| 1819 | 1740 | } |
| 1741 | _ = try self.builder.addAlias( |
| 1742 | exp_name, |
| 1743 | global_index.typeOf(&self.builder), |
| 1744 | .default, |
| 1745 | global_index.toConst(), |
| 1746 | ); |
| 1820 | 1747 | } |
| 1821 | 1748 | } else { |
| 1822 | | const fqn = try self.builder.string(mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod))); |
| 1823 | | try global.rename(fqn, &self.builder); |
| 1824 | | global.ptr(&self.builder).linkage = .internal; |
| 1825 | | llvm_global.setLinkage(.Internal); |
| 1826 | | if (mod.wantDllExports()) { |
| 1827 | | global.ptr(&self.builder).dll_storage_class = .default; |
| 1828 | | llvm_global.setDLLStorageClass(.Default); |
| 1829 | | } |
| 1830 | | global.ptr(&self.builder).unnamed_addr = .unnamed_addr; |
| 1831 | | llvm_global.setUnnamedAddr(.True); |
| 1749 | const fqn = try self.builder.string( |
| 1750 | mod.intern_pool.stringToSlice(try decl.getFullyQualifiedName(mod)), |
| 1751 | ); |
| 1752 | try global_index.rename(fqn, &self.builder); |
| 1753 | global_index.setLinkage(.internal, &self.builder); |
| 1754 | if (mod.wantDllExports()) global_index.setDllStorageClass(.default, &self.builder); |
| 1755 | global_index.setUnnamedAddr(.unnamed_addr, &self.builder); |
| 1832 | 1756 | if (decl.val.getVariable(mod)) |decl_var| { |
| 1833 | | const single_threaded = mod.comp.bin_file.options.single_threaded; |
| 1834 | | if (decl_var.is_threadlocal and !single_threaded) { |
| 1835 | | global.ptrConst(&self.builder).kind.variable.ptr(&self.builder).thread_local = |
| 1836 | | .generaldynamic; |
| 1837 | | llvm_global.setThreadLocalMode(.GeneralDynamicTLSModel); |
| 1838 | | } else { |
| 1839 | | global.ptrConst(&self.builder).kind.variable.ptr(&self.builder).thread_local = |
| 1840 | | .default; |
| 1841 | | llvm_global.setThreadLocalMode(.NotThreadLocal); |
| 1842 | | } |
| 1757 | global_index.ptrConst(&self.builder).kind.variable.setThreadLocal( |
| 1758 | if (decl_var.is_threadlocal and !mod.comp.bin_file.options.single_threaded) |
| 1759 | .generaldynamic |
| 1760 | else |
| 1761 | .default, |
| 1762 | &self.builder, |
| 1763 | ); |
| 1843 | 1764 | } |
| 1844 | | global.ptr(&self.builder).updateAttributes(); |
| 1845 | 1765 | } |
| 1846 | 1766 | } |
| 1847 | 1767 | |
| 1848 | 1768 | pub fn freeDecl(self: *Object, decl_index: Module.Decl.Index) void { |
| 1849 | 1769 | const global = self.decl_map.get(decl_index) orelse return; |
| 1850 | | global.toLlvm(&self.builder).deleteGlobal(); |
| 1770 | global.delete(&self.builder); |
| 1851 | 1771 | } |
| 1852 | 1772 | |
| 1853 | 1773 | fn getDIFile(o: *Object, gpa: Allocator, file: *const Module.File) !*llvm.DIFile { |
| ... | ... | @@ -2883,8 +2803,12 @@ pub const Object = struct { |
| 2883 | 2803 | /// If the llvm function does not exist, create it. |
| 2884 | 2804 | /// Note that this can be called before the function's semantic analysis has |
| 2885 | 2805 | /// completed, so if any attributes rely on that, they must be done in updateFunc, not here. |
| 2886 | | fn resolveLlvmFunction(o: *Object, decl_index: Module.Decl.Index) Allocator.Error!Builder.Function.Index { |
| 2806 | fn resolveLlvmFunction( |
| 2807 | o: *Object, |
| 2808 | decl_index: Module.Decl.Index, |
| 2809 | ) Allocator.Error!Builder.Function.Index { |
| 2887 | 2810 | const mod = o.module; |
| 2811 | const ip = &mod.intern_pool; |
| 2888 | 2812 | const gpa = o.gpa; |
| 2889 | 2813 | const decl = mod.declPtr(decl_index); |
| 2890 | 2814 | const zig_fn_type = decl.ty; |
| ... | ... | @@ -2896,31 +2820,20 @@ pub const Object = struct { |
| 2896 | 2820 | const target = mod.getTarget(); |
| 2897 | 2821 | const sret = firstParamSRet(fn_info, mod); |
| 2898 | 2822 | |
| 2899 | | const fn_type = try o.lowerType(zig_fn_type); |
| 2900 | | |
| 2901 | | const ip = &mod.intern_pool; |
| 2902 | | const fqn = try o.builder.string(ip.stringToSlice(try decl.getFullyQualifiedName(mod))); |
| 2903 | | |
| 2904 | | const llvm_addrspace = toLlvmAddressSpace(decl.@"addrspace", target); |
| 2905 | | const llvm_fn = o.llvm_module.addFunctionInAddressSpace(fqn.slice(&o.builder).?, fn_type.toLlvm(&o.builder), @intFromEnum(llvm_addrspace)); |
| 2906 | | |
| 2907 | | var global = Builder.Global{ |
| 2908 | | .type = fn_type, |
| 2909 | | .kind = .{ .function = @enumFromInt(o.builder.functions.items.len) }, |
| 2910 | | }; |
| 2911 | | var function = Builder.Function{ |
| 2912 | | .global = @enumFromInt(o.builder.globals.count()), |
| 2913 | | }; |
| 2823 | const function_index = try o.builder.addFunction( |
| 2824 | try o.lowerType(zig_fn_type), |
| 2825 | try o.builder.string(ip.stringToSlice(try decl.getFullyQualifiedName(mod))), |
| 2826 | toLlvmAddressSpace(decl.@"addrspace", target), |
| 2827 | ); |
| 2828 | gop.value_ptr.* = function_index.ptrConst(&o.builder).global; |
| 2914 | 2829 | |
| 2915 | 2830 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 2916 | 2831 | defer attributes.deinit(&o.builder); |
| 2917 | 2832 | |
| 2918 | 2833 | const is_extern = decl.isExtern(mod); |
| 2919 | 2834 | if (!is_extern) { |
| 2920 | | global.linkage = .internal; |
| 2921 | | llvm_fn.setLinkage(.Internal); |
| 2922 | | global.unnamed_addr = .unnamed_addr; |
| 2923 | | llvm_fn.setUnnamedAddr(.True); |
| 2835 | function_index.setLinkage(.internal, &o.builder); |
| 2836 | function_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 2924 | 2837 | } else { |
| 2925 | 2838 | if (target.isWasm()) { |
| 2926 | 2839 | try attributes.addFnAttr(.{ .string = .{ |
| ... | ... | @@ -2957,35 +2870,22 @@ pub const Object = struct { |
| 2957 | 2870 | } |
| 2958 | 2871 | |
| 2959 | 2872 | switch (fn_info.cc) { |
| 2960 | | .Unspecified, .Inline => { |
| 2961 | | function.call_conv = .fastcc; |
| 2962 | | llvm_fn.setFunctionCallConv(.Fast); |
| 2963 | | }, |
| 2964 | | .Naked => { |
| 2965 | | try attributes.addFnAttr(.naked, &o.builder); |
| 2966 | | }, |
| 2873 | .Unspecified, .Inline => function_index.setCallConv(.fastcc, &o.builder), |
| 2874 | .Naked => try attributes.addFnAttr(.naked, &o.builder), |
| 2967 | 2875 | .Async => { |
| 2968 | | function.call_conv = .fastcc; |
| 2969 | | llvm_fn.setFunctionCallConv(.Fast); |
| 2876 | function_index.setCallConv(.fastcc, &o.builder); |
| 2970 | 2877 | @panic("TODO: LLVM backend lower async function"); |
| 2971 | 2878 | }, |
| 2972 | | else => { |
| 2973 | | function.call_conv = toLlvmCallConv(fn_info.cc, target); |
| 2974 | | llvm_fn.setFunctionCallConv(@enumFromInt(@intFromEnum(function.call_conv))); |
| 2975 | | }, |
| 2879 | else => function_index.setCallConv(toLlvmCallConv(fn_info.cc, target), &o.builder), |
| 2976 | 2880 | } |
| 2977 | 2881 | |
| 2978 | | if (fn_info.alignment.toByteUnitsOptional()) |a| { |
| 2979 | | function.alignment = Builder.Alignment.fromByteUnits(a); |
| 2980 | | llvm_fn.setAlignment(@intCast(a)); |
| 2981 | | } |
| 2882 | if (fn_info.alignment.toByteUnitsOptional()) |alignment| |
| 2883 | function_index.setAlignment(Builder.Alignment.fromByteUnits(alignment), &o.builder); |
| 2982 | 2884 | |
| 2983 | 2885 | // Function attributes that are independent of analysis results of the function body. |
| 2984 | 2886 | try o.addCommonFnAttributes(&attributes); |
| 2985 | 2887 | |
| 2986 | | if (fn_info.return_type == .noreturn_type) { |
| 2987 | | try attributes.addFnAttr(.noreturn, &o.builder); |
| 2988 | | } |
| 2888 | if (fn_info.return_type == .noreturn_type) try attributes.addFnAttr(.noreturn, &o.builder); |
| 2989 | 2889 | |
| 2990 | 2890 | // Add parameter attributes. We handle only the case of extern functions (no body) |
| 2991 | 2891 | // because functions with bodies are handled in `updateFunc`. |
| ... | ... | @@ -3007,9 +2907,7 @@ pub const Object = struct { |
| 3007 | 2907 | Builder.Alignment.fromByteUnits(param_ty.toType().abiAlignment(mod)); |
| 3008 | 2908 | try o.addByRefParamAttrs(&attributes, it.llvm_index - 1, alignment, it.byval_attr, param_llvm_ty); |
| 3009 | 2909 | }, |
| 3010 | | .byref_mut => { |
| 3011 | | try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder); |
| 3012 | | }, |
| 2910 | .byref_mut => try attributes.addParamAttr(it.llvm_index - 1, .noundef, &o.builder), |
| 3013 | 2911 | // No attributes needed for these. |
| 3014 | 2912 | .no_bits, |
| 3015 | 2913 | .abi_sized_int, |
| ... | ... | @@ -3025,11 +2923,8 @@ pub const Object = struct { |
| 3025 | 2923 | }; |
| 3026 | 2924 | } |
| 3027 | 2925 | |
| 3028 | | try o.builder.llvm.globals.append(o.gpa, llvm_fn); |
| 3029 | | gop.value_ptr.* = try o.builder.addGlobal(fqn, global); |
| 3030 | | try o.builder.functions.append(o.gpa, function); |
| 3031 | | global.kind.function.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 3032 | | return global.kind.function; |
| 2926 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 2927 | return function_index; |
| 3033 | 2928 | } |
| 3034 | 2929 | |
| 3035 | 2930 | fn addCommonFnAttributes( |
| ... | ... | @@ -3093,76 +2988,50 @@ pub const Object = struct { |
| 3093 | 2988 | } |
| 3094 | 2989 | } |
| 3095 | 2990 | |
| 3096 | | fn resolveGlobalDecl(o: *Object, decl_index: Module.Decl.Index) Allocator.Error!Builder.Variable.Index { |
| 2991 | fn resolveGlobalDecl( |
| 2992 | o: *Object, |
| 2993 | decl_index: Module.Decl.Index, |
| 2994 | ) Allocator.Error!Builder.Variable.Index { |
| 3097 | 2995 | const gop = try o.decl_map.getOrPut(o.gpa, decl_index); |
| 3098 | 2996 | if (gop.found_existing) return gop.value_ptr.ptr(&o.builder).kind.variable; |
| 3099 | 2997 | errdefer assert(o.decl_map.remove(decl_index)); |
| 3100 | 2998 | |
| 3101 | 2999 | const mod = o.module; |
| 3102 | 3000 | const decl = mod.declPtr(decl_index); |
| 3103 | | const fqn = try o.builder.string(mod.intern_pool.stringToSlice( |
| 3104 | | try decl.getFullyQualifiedName(mod), |
| 3105 | | )); |
| 3106 | | |
| 3107 | | const target = mod.getTarget(); |
| 3108 | | |
| 3109 | | var global = Builder.Global{ |
| 3110 | | .addr_space = toLlvmGlobalAddressSpace(decl.@"addrspace", target), |
| 3111 | | .type = try o.lowerType(decl.ty), |
| 3112 | | .kind = .{ .variable = @enumFromInt(o.builder.variables.items.len) }, |
| 3113 | | }; |
| 3114 | | var variable = Builder.Variable{ |
| 3115 | | .global = @enumFromInt(o.builder.globals.count()), |
| 3116 | | }; |
| 3117 | | |
| 3118 | 3001 | const is_extern = decl.isExtern(mod); |
| 3119 | | const name = if (is_extern) |
| 3120 | | try o.builder.string(mod.intern_pool.stringToSlice(decl.name)) |
| 3121 | | else |
| 3122 | | fqn; |
| 3123 | | const llvm_global = o.llvm_module.addGlobalInAddressSpace( |
| 3124 | | global.type.toLlvm(&o.builder), |
| 3125 | | fqn.slice(&o.builder).?, |
| 3126 | | @intFromEnum(global.addr_space), |
| 3002 | |
| 3003 | const variable_index = try o.builder.addVariable( |
| 3004 | try o.builder.string(mod.intern_pool.stringToSlice( |
| 3005 | if (is_extern) decl.name else try decl.getFullyQualifiedName(mod), |
| 3006 | )), |
| 3007 | try o.lowerType(decl.ty), |
| 3008 | toLlvmGlobalAddressSpace(decl.@"addrspace", mod.getTarget()), |
| 3127 | 3009 | ); |
| 3010 | gop.value_ptr.* = variable_index.ptrConst(&o.builder).global; |
| 3128 | 3011 | |
| 3129 | 3012 | // This is needed for declarations created by `@extern`. |
| 3130 | 3013 | if (is_extern) { |
| 3131 | | global.unnamed_addr = .default; |
| 3132 | | llvm_global.setUnnamedAddr(.False); |
| 3133 | | global.linkage = .external; |
| 3134 | | llvm_global.setLinkage(.External); |
| 3014 | variable_index.setLinkage(.external, &o.builder); |
| 3015 | variable_index.setUnnamedAddr(.default, &o.builder); |
| 3135 | 3016 | if (decl.val.getVariable(mod)) |decl_var| { |
| 3136 | 3017 | const single_threaded = mod.comp.bin_file.options.single_threaded; |
| 3137 | | if (decl_var.is_threadlocal and !single_threaded) { |
| 3138 | | variable.thread_local = .generaldynamic; |
| 3139 | | llvm_global.setThreadLocalMode(.GeneralDynamicTLSModel); |
| 3140 | | } else { |
| 3141 | | variable.thread_local = .default; |
| 3142 | | llvm_global.setThreadLocalMode(.NotThreadLocal); |
| 3143 | | } |
| 3144 | | if (decl_var.is_weak_linkage) { |
| 3145 | | global.linkage = .extern_weak; |
| 3146 | | llvm_global.setLinkage(.ExternalWeak); |
| 3147 | | } |
| 3018 | variable_index.setThreadLocal( |
| 3019 | if (decl_var.is_threadlocal and !single_threaded) .generaldynamic else .default, |
| 3020 | &o.builder, |
| 3021 | ); |
| 3022 | if (decl_var.is_weak_linkage) variable_index.setLinkage(.extern_weak, &o.builder); |
| 3148 | 3023 | } |
| 3149 | 3024 | } else { |
| 3150 | | global.linkage = .internal; |
| 3151 | | llvm_global.setLinkage(.Internal); |
| 3152 | | global.unnamed_addr = .unnamed_addr; |
| 3153 | | llvm_global.setUnnamedAddr(.True); |
| 3025 | variable_index.setLinkage(.internal, &o.builder); |
| 3026 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 3154 | 3027 | } |
| 3155 | | |
| 3156 | | try o.builder.llvm.globals.append(o.gpa, llvm_global); |
| 3157 | | gop.value_ptr.* = try o.builder.addGlobal(name, global); |
| 3158 | | try o.builder.variables.append(o.gpa, variable); |
| 3159 | | return global.kind.variable; |
| 3028 | return variable_index; |
| 3160 | 3029 | } |
| 3161 | 3030 | |
| 3162 | 3031 | fn lowerType(o: *Object, t: Type) Allocator.Error!Builder.Type { |
| 3163 | 3032 | const ty = try o.lowerTypeInner(t); |
| 3164 | 3033 | const mod = o.module; |
| 3165 | | if (std.debug.runtime_safety and false) check: { |
| 3034 | if (std.debug.runtime_safety and o.builder.useLibLlvm() and false) check: { |
| 3166 | 3035 | const llvm_ty = ty.toLlvm(&o.builder); |
| 3167 | 3036 | if (t.zigTypeTag(mod) == .Opaque) break :check; |
| 3168 | 3037 | if (!t.hasRuntimeBits(mod)) break :check; |
| ... | ... | @@ -4533,65 +4402,22 @@ pub const DeclGen = struct { |
| 4533 | 4402 | if (decl.val.getExternFunc(mod)) |extern_func| { |
| 4534 | 4403 | _ = try o.resolveLlvmFunction(extern_func.decl); |
| 4535 | 4404 | } else { |
| 4536 | | const target = mod.getTarget(); |
| 4537 | | const variable = try o.resolveGlobalDecl(decl_index); |
| 4538 | | const global = variable.ptrConst(&o.builder).global; |
| 4539 | | var llvm_global = global.toLlvm(&o.builder); |
| 4540 | | variable.ptr(&o.builder).alignment = Builder.Alignment.fromByteUnits(decl.getAlignment(mod)); |
| 4541 | | llvm_global.setAlignment(decl.getAlignment(mod)); |
| 4542 | | if (mod.intern_pool.stringToSliceUnwrap(decl.@"linksection")) |section| { |
| 4543 | | variable.ptr(&o.builder).section = try o.builder.string(section); |
| 4544 | | llvm_global.setSection(section); |
| 4545 | | } |
| 4405 | const variable_index = try o.resolveGlobalDecl(decl_index); |
| 4406 | variable_index.setAlignment( |
| 4407 | Builder.Alignment.fromByteUnits(decl.getAlignment(mod)), |
| 4408 | &o.builder, |
| 4409 | ); |
| 4410 | if (mod.intern_pool.stringToSliceUnwrap(decl.@"linksection")) |section| |
| 4411 | variable_index.setSection(try o.builder.string(section), &o.builder); |
| 4546 | 4412 | assert(decl.has_tv); |
| 4547 | 4413 | const init_val = if (decl.val.getVariable(mod)) |decl_var| decl_var.init else init_val: { |
| 4548 | | variable.ptr(&o.builder).mutability = .constant; |
| 4549 | | llvm_global.setGlobalConstant(.True); |
| 4414 | variable_index.setMutability(.constant, &o.builder); |
| 4550 | 4415 | break :init_val decl.val.toIntern(); |
| 4551 | 4416 | }; |
| 4552 | | if (init_val != .none) { |
| 4553 | | const llvm_init = try o.lowerValue(init_val); |
| 4554 | | const llvm_init_ty = llvm_init.typeOf(&o.builder); |
| 4555 | | if (global.ptrConst(&o.builder).type == llvm_init_ty) { |
| 4556 | | llvm_global.setInitializer(llvm_init.toLlvm(&o.builder)); |
| 4557 | | } else { |
| 4558 | | // LLVM does not allow us to change the type of globals. So we must |
| 4559 | | // create a new global with the correct type, copy all its attributes, |
| 4560 | | // and then update all references to point to the new global, |
| 4561 | | // delete the original, and rename the new one to the old one's name. |
| 4562 | | // This is necessary because LLVM does not support const bitcasting |
| 4563 | | // a struct with padding bytes, which is needed to lower a const union value |
| 4564 | | // to LLVM, when a field other than the most-aligned is active. Instead, |
| 4565 | | // we must lower to an unnamed struct, and pointer cast at usage sites |
| 4566 | | // of the global. Such an unnamed struct is the cause of the global type |
| 4567 | | // mismatch, because we don't have the LLVM type until the *value* is created, |
| 4568 | | // whereas the global needs to be created based on the type alone, because |
| 4569 | | // lowering the value may reference the global as a pointer. |
| 4570 | | // Related: https://github.com/ziglang/zig/issues/13265 |
| 4571 | | const llvm_global_addrspace = toLlvmGlobalAddressSpace(decl.@"addrspace", target); |
| 4572 | | const new_global = o.llvm_module.addGlobalInAddressSpace( |
| 4573 | | llvm_init_ty.toLlvm(&o.builder), |
| 4574 | | "", |
| 4575 | | @intFromEnum(llvm_global_addrspace), |
| 4576 | | ); |
| 4577 | | new_global.setLinkage(llvm_global.getLinkage()); |
| 4578 | | new_global.setUnnamedAddr(llvm_global.getUnnamedAddress()); |
| 4579 | | new_global.setAlignment(llvm_global.getAlignment()); |
| 4580 | | if (mod.intern_pool.stringToSliceUnwrap(decl.@"linksection")) |section| |
| 4581 | | new_global.setSection(section); |
| 4582 | | new_global.setInitializer(llvm_init.toLlvm(&o.builder)); |
| 4583 | | // TODO: How should this work then the address space of a global changed? |
| 4584 | | llvm_global.replaceAllUsesWith(new_global); |
| 4585 | | new_global.takeName(llvm_global); |
| 4586 | | o.builder.llvm.globals.items[@intFromEnum(variable.ptrConst(&o.builder).global)] = |
| 4587 | | new_global; |
| 4588 | | llvm_global.deleteGlobal(); |
| 4589 | | llvm_global = new_global; |
| 4590 | | variable.ptr(&o.builder).mutability = .global; |
| 4591 | | global.ptr(&o.builder).type = llvm_init_ty; |
| 4592 | | } |
| 4593 | | variable.ptr(&o.builder).init = llvm_init; |
| 4594 | | } |
| 4417 | try variable_index.setInitializer(switch (init_val) { |
| 4418 | .none => .no_init, |
| 4419 | else => try o.lowerValue(init_val), |
| 4420 | }, &o.builder); |
| 4595 | 4421 | |
| 4596 | 4422 | if (o.di_builder) |dib| { |
| 4597 | 4423 | const di_file = try o.getDIFile(o.gpa, mod.namespacePtr(decl.src_namespace).file_scope); |
| ... | ... | @@ -4601,7 +4427,7 @@ pub const DeclGen = struct { |
| 4601 | 4427 | const di_global = dib.createGlobalVariableExpression( |
| 4602 | 4428 | di_file.toScope(), |
| 4603 | 4429 | mod.intern_pool.stringToSlice(decl.name), |
| 4604 | | llvm_global.getValueName(), |
| 4430 | variable_index.name(&o.builder).slice(&o.builder).?, |
| 4605 | 4431 | di_file, |
| 4606 | 4432 | line_number, |
| 4607 | 4433 | try o.lowerDebugType(decl.ty, .full), |
| ... | ... | @@ -4609,7 +4435,8 @@ pub const DeclGen = struct { |
| 4609 | 4435 | ); |
| 4610 | 4436 | |
| 4611 | 4437 | try o.di_map.put(o.gpa, dg.decl, di_global.getVariable().toNode()); |
| 4612 | | if (!is_internal_linkage or decl.isExtern(mod)) llvm_global.attachMetaData(di_global); |
| 4438 | if (!is_internal_linkage or decl.isExtern(mod)) |
| 4439 | variable_index.toLlvm(&o.builder).attachMetaData(di_global); |
| 4613 | 4440 | } |
| 4614 | 4441 | } |
| 4615 | 4442 | } |
| ... | ... | @@ -4621,7 +4448,6 @@ pub const FuncGen = struct { |
| 4621 | 4448 | air: Air, |
| 4622 | 4449 | liveness: Liveness, |
| 4623 | 4450 | wip: Builder.WipFunction, |
| 4624 | | builder: *llvm.Builder, |
| 4625 | 4451 | di_scope: ?*llvm.DIScope, |
| 4626 | 4452 | di_file: ?*llvm.DIFile, |
| 4627 | 4453 | base_line: u32, |
| ... | ... | @@ -4710,38 +4536,22 @@ pub const FuncGen = struct { |
| 4710 | 4536 | // We have an LLVM value but we need to create a global constant and |
| 4711 | 4537 | // set the value as its initializer, and then return a pointer to the global. |
| 4712 | 4538 | const target = mod.getTarget(); |
| 4713 | | const llvm_wanted_addrspace = toLlvmAddressSpace(.generic, target); |
| 4714 | | const llvm_actual_addrspace = toLlvmGlobalAddressSpace(.generic, target); |
| 4715 | | const llvm_ty = llvm_val.typeOf(&o.builder); |
| 4716 | | const llvm_alignment = tv.ty.abiAlignment(mod); |
| 4717 | | const llvm_global = o.llvm_module.addGlobalInAddressSpace(llvm_ty.toLlvm(&o.builder), "", @intFromEnum(llvm_actual_addrspace)); |
| 4718 | | llvm_global.setInitializer(llvm_val.toLlvm(&o.builder)); |
| 4719 | | llvm_global.setLinkage(.Private); |
| 4720 | | llvm_global.setGlobalConstant(.True); |
| 4721 | | llvm_global.setUnnamedAddr(.True); |
| 4722 | | llvm_global.setAlignment(llvm_alignment); |
| 4723 | | |
| 4724 | | var global = Builder.Global{ |
| 4725 | | .linkage = .private, |
| 4726 | | .unnamed_addr = .unnamed_addr, |
| 4727 | | .addr_space = llvm_actual_addrspace, |
| 4728 | | .type = llvm_ty, |
| 4729 | | .kind = .{ .variable = @enumFromInt(o.builder.variables.items.len) }, |
| 4730 | | }; |
| 4731 | | var variable = Builder.Variable{ |
| 4732 | | .global = @enumFromInt(o.builder.globals.count()), |
| 4733 | | .mutability = .constant, |
| 4734 | | .init = llvm_val, |
| 4735 | | .alignment = Builder.Alignment.fromByteUnits(llvm_alignment), |
| 4736 | | }; |
| 4737 | | try o.builder.llvm.globals.append(o.gpa, llvm_global); |
| 4738 | | const global_index = try o.builder.addGlobal(.empty, global); |
| 4739 | | try o.builder.variables.append(o.gpa, variable); |
| 4740 | | |
| 4539 | const variable_index = try o.builder.addVariable( |
| 4540 | .empty, |
| 4541 | llvm_val.typeOf(&o.builder), |
| 4542 | toLlvmGlobalAddressSpace(.generic, target), |
| 4543 | ); |
| 4544 | try variable_index.setInitializer(llvm_val, &o.builder); |
| 4545 | variable_index.setLinkage(.private, &o.builder); |
| 4546 | variable_index.setMutability(.constant, &o.builder); |
| 4547 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 4548 | variable_index.setAlignment(Builder.Alignment.fromByteUnits( |
| 4549 | tv.ty.abiAlignment(mod), |
| 4550 | ), &o.builder); |
| 4741 | 4551 | return o.builder.convConst( |
| 4742 | 4552 | .unneeded, |
| 4743 | | global_index.toConst(), |
| 4744 | | try o.builder.ptrType(llvm_wanted_addrspace), |
| 4553 | variable_index.toConst(&o.builder), |
| 4554 | try o.builder.ptrType(toLlvmAddressSpace(.generic, target)), |
| 4745 | 4555 | ); |
| 4746 | 4556 | } |
| 4747 | 4557 | |
| ... | ... | @@ -4768,18 +4578,18 @@ pub const FuncGen = struct { |
| 4768 | 4578 | |
| 4769 | 4579 | const val: Builder.Value = switch (air_tags[inst]) { |
| 4770 | 4580 | // zig fmt: off |
| 4771 | | .add => try self.airAdd(inst, false), |
| 4772 | | .add_optimized => try self.airAdd(inst, true), |
| 4581 | .add => try self.airAdd(inst, .normal), |
| 4582 | .add_optimized => try self.airAdd(inst, .fast), |
| 4773 | 4583 | .add_wrap => try self.airAddWrap(inst), |
| 4774 | 4584 | .add_sat => try self.airAddSat(inst), |
| 4775 | 4585 | |
| 4776 | | .sub => try self.airSub(inst, false), |
| 4777 | | .sub_optimized => try self.airSub(inst, true), |
| 4586 | .sub => try self.airSub(inst, .normal), |
| 4587 | .sub_optimized => try self.airSub(inst, .fast), |
| 4778 | 4588 | .sub_wrap => try self.airSubWrap(inst), |
| 4779 | 4589 | .sub_sat => try self.airSubSat(inst), |
| 4780 | 4590 | |
| 4781 | | .mul => try self.airMul(inst, false), |
| 4782 | | .mul_optimized => try self.airMul(inst, true), |
| 4591 | .mul => try self.airMul(inst, .normal), |
| 4592 | .mul_optimized => try self.airMul(inst, .fast), |
| 4783 | 4593 | .mul_wrap => try self.airMulWrap(inst), |
| 4784 | 4594 | .mul_sat => try self.airMulSat(inst), |
| 4785 | 4595 | |
| ... | ... | @@ -4787,12 +4597,12 @@ pub const FuncGen = struct { |
| 4787 | 4597 | .sub_safe => try self.airSafeArithmetic(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), |
| 4788 | 4598 | .mul_safe => try self.airSafeArithmetic(inst, .@"smul.with.overflow", .@"umul.with.overflow"), |
| 4789 | 4599 | |
| 4790 | | .div_float => try self.airDivFloat(inst, false), |
| 4791 | | .div_trunc => try self.airDivTrunc(inst, false), |
| 4792 | | .div_floor => try self.airDivFloor(inst, false), |
| 4793 | | .div_exact => try self.airDivExact(inst, false), |
| 4794 | | .rem => try self.airRem(inst, false), |
| 4795 | | .mod => try self.airMod(inst, false), |
| 4600 | .div_float => try self.airDivFloat(inst, .normal), |
| 4601 | .div_trunc => try self.airDivTrunc(inst, .normal), |
| 4602 | .div_floor => try self.airDivFloor(inst, .normal), |
| 4603 | .div_exact => try self.airDivExact(inst, .normal), |
| 4604 | .rem => try self.airRem(inst, .normal), |
| 4605 | .mod => try self.airMod(inst, .normal), |
| 4796 | 4606 | .ptr_add => try self.airPtrAdd(inst), |
| 4797 | 4607 | .ptr_sub => try self.airPtrSub(inst), |
| 4798 | 4608 | .shl => try self.airShl(inst), |
| ... | ... | @@ -4803,12 +4613,12 @@ pub const FuncGen = struct { |
| 4803 | 4613 | .slice => try self.airSlice(inst), |
| 4804 | 4614 | .mul_add => try self.airMulAdd(inst), |
| 4805 | 4615 | |
| 4806 | | .div_float_optimized => try self.airDivFloat(inst, true), |
| 4807 | | .div_trunc_optimized => try self.airDivTrunc(inst, true), |
| 4808 | | .div_floor_optimized => try self.airDivFloor(inst, true), |
| 4809 | | .div_exact_optimized => try self.airDivExact(inst, true), |
| 4810 | | .rem_optimized => try self.airRem(inst, true), |
| 4811 | | .mod_optimized => try self.airMod(inst, true), |
| 4616 | .div_float_optimized => try self.airDivFloat(inst, .fast), |
| 4617 | .div_trunc_optimized => try self.airDivTrunc(inst, .fast), |
| 4618 | .div_floor_optimized => try self.airDivFloor(inst, .fast), |
| 4619 | .div_exact_optimized => try self.airDivExact(inst, .fast), |
| 4620 | .rem_optimized => try self.airRem(inst, .fast), |
| 4621 | .mod_optimized => try self.airMod(inst, .fast), |
| 4812 | 4622 | |
| 4813 | 4623 | .add_with_overflow => try self.airOverflow(inst, .@"sadd.with.overflow", .@"uadd.with.overflow"), |
| 4814 | 4624 | .sub_with_overflow => try self.airOverflow(inst, .@"ssub.with.overflow", .@"usub.with.overflow"), |
| ... | ... | @@ -4836,25 +4646,25 @@ pub const FuncGen = struct { |
| 4836 | 4646 | .round => try self.airUnaryOp(inst, .round), |
| 4837 | 4647 | .trunc_float => try self.airUnaryOp(inst, .trunc), |
| 4838 | 4648 | |
| 4839 | | .neg => try self.airNeg(inst, false), |
| 4840 | | .neg_optimized => try self.airNeg(inst, true), |
| 4841 | | |
| 4842 | | .cmp_eq => try self.airCmp(inst, .eq, false), |
| 4843 | | .cmp_gt => try self.airCmp(inst, .gt, false), |
| 4844 | | .cmp_gte => try self.airCmp(inst, .gte, false), |
| 4845 | | .cmp_lt => try self.airCmp(inst, .lt, false), |
| 4846 | | .cmp_lte => try self.airCmp(inst, .lte, false), |
| 4847 | | .cmp_neq => try self.airCmp(inst, .neq, false), |
| 4848 | | |
| 4849 | | .cmp_eq_optimized => try self.airCmp(inst, .eq, true), |
| 4850 | | .cmp_gt_optimized => try self.airCmp(inst, .gt, true), |
| 4851 | | .cmp_gte_optimized => try self.airCmp(inst, .gte, true), |
| 4852 | | .cmp_lt_optimized => try self.airCmp(inst, .lt, true), |
| 4853 | | .cmp_lte_optimized => try self.airCmp(inst, .lte, true), |
| 4854 | | .cmp_neq_optimized => try self.airCmp(inst, .neq, true), |
| 4855 | | |
| 4856 | | .cmp_vector => try self.airCmpVector(inst, false), |
| 4857 | | .cmp_vector_optimized => try self.airCmpVector(inst, true), |
| 4649 | .neg => try self.airNeg(inst, .normal), |
| 4650 | .neg_optimized => try self.airNeg(inst, .fast), |
| 4651 | |
| 4652 | .cmp_eq => try self.airCmp(inst, .eq, .normal), |
| 4653 | .cmp_gt => try self.airCmp(inst, .gt, .normal), |
| 4654 | .cmp_gte => try self.airCmp(inst, .gte, .normal), |
| 4655 | .cmp_lt => try self.airCmp(inst, .lt, .normal), |
| 4656 | .cmp_lte => try self.airCmp(inst, .lte, .normal), |
| 4657 | .cmp_neq => try self.airCmp(inst, .neq, .normal), |
| 4658 | |
| 4659 | .cmp_eq_optimized => try self.airCmp(inst, .eq, .fast), |
| 4660 | .cmp_gt_optimized => try self.airCmp(inst, .gt, .fast), |
| 4661 | .cmp_gte_optimized => try self.airCmp(inst, .gte, .fast), |
| 4662 | .cmp_lt_optimized => try self.airCmp(inst, .lt, .fast), |
| 4663 | .cmp_lte_optimized => try self.airCmp(inst, .lte, .fast), |
| 4664 | .cmp_neq_optimized => try self.airCmp(inst, .neq, .fast), |
| 4665 | |
| 4666 | .cmp_vector => try self.airCmpVector(inst, .normal), |
| 4667 | .cmp_vector_optimized => try self.airCmpVector(inst, .fast), |
| 4858 | 4668 | .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), |
| 4859 | 4669 | |
| 4860 | 4670 | .is_non_null => try self.airIsNonNull(inst, false, .ne), |
| ... | ... | @@ -4906,8 +4716,8 @@ pub const FuncGen = struct { |
| 4906 | 4716 | .ptr_slice_ptr_ptr => try self.airPtrSliceFieldPtr(inst, 0), |
| 4907 | 4717 | .ptr_slice_len_ptr => try self.airPtrSliceFieldPtr(inst, 1), |
| 4908 | 4718 | |
| 4909 | | .int_from_float => try self.airIntFromFloat(inst, false), |
| 4910 | | .int_from_float_optimized => try self.airIntFromFloat(inst, true), |
| 4719 | .int_from_float => try self.airIntFromFloat(inst, .normal), |
| 4720 | .int_from_float_optimized => try self.airIntFromFloat(inst, .fast), |
| 4911 | 4721 | |
| 4912 | 4722 | .array_to_slice => try self.airArrayToSlice(inst), |
| 4913 | 4723 | .float_from_int => try self.airFloatFromInt(inst), |
| ... | ... | @@ -4939,8 +4749,8 @@ pub const FuncGen = struct { |
| 4939 | 4749 | .is_named_enum_value => try self.airIsNamedEnumValue(inst), |
| 4940 | 4750 | .error_set_has_value => try self.airErrorSetHasValue(inst), |
| 4941 | 4751 | |
| 4942 | | .reduce => try self.airReduce(inst, false), |
| 4943 | | .reduce_optimized => try self.airReduce(inst, true), |
| 4752 | .reduce => try self.airReduce(inst, .normal), |
| 4753 | .reduce_optimized => try self.airReduce(inst, .fast), |
| 4944 | 4754 | |
| 4945 | 4755 | .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), |
| 4946 | 4756 | .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), |
| ... | ... | @@ -5466,7 +5276,7 @@ pub const FuncGen = struct { |
| 5466 | 5276 | const result_alignment = Builder.Alignment.fromByteUnits(va_list_ty.abiAlignment(mod)); |
| 5467 | 5277 | const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); |
| 5468 | 5278 | |
| 5469 | | _ = try self.wip.callIntrinsic(.none, .va_copy, &.{}, &.{ dest_list, src_list }, ""); |
| 5279 | _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, ""); |
| 5470 | 5280 | return if (isByRef(va_list_ty, mod)) |
| 5471 | 5281 | dest_list |
| 5472 | 5282 | else |
| ... | ... | @@ -5477,7 +5287,7 @@ pub const FuncGen = struct { |
| 5477 | 5287 | const un_op = self.air.instructions.items(.data)[inst].un_op; |
| 5478 | 5288 | const src_list = try self.resolveInst(un_op); |
| 5479 | 5289 | |
| 5480 | | _ = try self.wip.callIntrinsic(.none, .va_end, &.{}, &.{src_list}, ""); |
| 5290 | _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{}, &.{src_list}, ""); |
| 5481 | 5291 | return .none; |
| 5482 | 5292 | } |
| 5483 | 5293 | |
| ... | ... | @@ -5490,27 +5300,28 @@ pub const FuncGen = struct { |
| 5490 | 5300 | const result_alignment = Builder.Alignment.fromByteUnits(va_list_ty.abiAlignment(mod)); |
| 5491 | 5301 | const dest_list = try self.buildAlloca(llvm_va_list_ty, result_alignment); |
| 5492 | 5302 | |
| 5493 | | _ = try self.wip.callIntrinsic(.none, .va_start, &.{}, &.{dest_list}, ""); |
| 5303 | _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, ""); |
| 5494 | 5304 | return if (isByRef(va_list_ty, mod)) |
| 5495 | 5305 | dest_list |
| 5496 | 5306 | else |
| 5497 | 5307 | try self.wip.load(.normal, llvm_va_list_ty, dest_list, result_alignment, ""); |
| 5498 | 5308 | } |
| 5499 | 5309 | |
| 5500 | | fn airCmp(self: *FuncGen, inst: Air.Inst.Index, op: math.CompareOperator, want_fast_math: bool) !Builder.Value { |
| 5501 | | self.builder.setFastMath(want_fast_math); |
| 5502 | | |
| 5310 | fn airCmp( |
| 5311 | self: *FuncGen, |
| 5312 | inst: Air.Inst.Index, |
| 5313 | op: math.CompareOperator, |
| 5314 | fast: Builder.FastMathKind, |
| 5315 | ) !Builder.Value { |
| 5503 | 5316 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| 5504 | 5317 | const lhs = try self.resolveInst(bin_op.lhs); |
| 5505 | 5318 | const rhs = try self.resolveInst(bin_op.rhs); |
| 5506 | 5319 | const operand_ty = self.typeOf(bin_op.lhs); |
| 5507 | 5320 | |
| 5508 | | return self.cmp(lhs, rhs, operand_ty, op); |
| 5321 | return self.cmp(fast, op, operand_ty, lhs, rhs); |
| 5509 | 5322 | } |
| 5510 | 5323 | |
| 5511 | | fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 5512 | | self.builder.setFastMath(want_fast_math); |
| 5513 | | |
| 5324 | fn airCmpVector(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 5514 | 5325 | const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; |
| 5515 | 5326 | const extra = self.air.extraData(Air.VectorCmp, ty_pl.payload).data; |
| 5516 | 5327 | |
| ... | ... | @@ -5519,7 +5330,7 @@ pub const FuncGen = struct { |
| 5519 | 5330 | const vec_ty = self.typeOf(extra.lhs); |
| 5520 | 5331 | const cmp_op = extra.compareOperator(); |
| 5521 | 5332 | |
| 5522 | | return self.cmp(lhs, rhs, vec_ty, cmp_op); |
| 5333 | return self.cmp(fast, cmp_op, vec_ty, lhs, rhs); |
| 5523 | 5334 | } |
| 5524 | 5335 | |
| 5525 | 5336 | fn airCmpLtErrorsLen(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| ... | ... | @@ -5540,10 +5351,11 @@ pub const FuncGen = struct { |
| 5540 | 5351 | |
| 5541 | 5352 | fn cmp( |
| 5542 | 5353 | self: *FuncGen, |
| 5354 | fast: Builder.FastMathKind, |
| 5355 | op: math.CompareOperator, |
| 5356 | operand_ty: Type, |
| 5543 | 5357 | lhs: Builder.Value, |
| 5544 | 5358 | rhs: Builder.Value, |
| 5545 | | operand_ty: Type, |
| 5546 | | op: math.CompareOperator, |
| 5547 | 5359 | ) Allocator.Error!Builder.Value { |
| 5548 | 5360 | const o = self.dg.object; |
| 5549 | 5361 | const mod = o.module; |
| ... | ... | @@ -5595,7 +5407,7 @@ pub const FuncGen = struct { |
| 5595 | 5407 | self.wip.cursor = .{ .block = both_pl_block }; |
| 5596 | 5408 | const lhs_payload = try self.optPayloadHandle(opt_llvm_ty, lhs, scalar_ty, true); |
| 5597 | 5409 | const rhs_payload = try self.optPayloadHandle(opt_llvm_ty, rhs, scalar_ty, true); |
| 5598 | | const payload_cmp = try self.cmp(lhs_payload, rhs_payload, payload_ty, op); |
| 5410 | const payload_cmp = try self.cmp(fast, op, payload_ty, lhs_payload, rhs_payload); |
| 5599 | 5411 | _ = try self.wip.br(end_block); |
| 5600 | 5412 | const both_pl_block_end = self.wip.cursor.block; |
| 5601 | 5413 | |
| ... | ... | @@ -5624,7 +5436,7 @@ pub const FuncGen = struct { |
| 5624 | 5436 | ); |
| 5625 | 5437 | return phi.toValue(); |
| 5626 | 5438 | }, |
| 5627 | | .Float => return self.buildFloatCmp(op, operand_ty, .{ lhs, rhs }), |
| 5439 | .Float => return self.buildFloatCmp(fast, op, operand_ty, .{ lhs, rhs }), |
| 5628 | 5440 | else => unreachable, |
| 5629 | 5441 | }; |
| 5630 | 5442 | const is_signed = int_ty.isSignedInt(mod); |
| ... | ... | @@ -5995,8 +5807,12 @@ pub const FuncGen = struct { |
| 5995 | 5807 | ); |
| 5996 | 5808 | } |
| 5997 | 5809 | |
| 5998 | | fn airIntFromFloat(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 5999 | | self.builder.setFastMath(want_fast_math); |
| 5810 | fn airIntFromFloat( |
| 5811 | self: *FuncGen, |
| 5812 | inst: Air.Inst.Index, |
| 5813 | fast: Builder.FastMathKind, |
| 5814 | ) !Builder.Value { |
| 5815 | _ = fast; |
| 6000 | 5816 | |
| 6001 | 5817 | const o = self.dg.object; |
| 6002 | 5818 | const mod = o.module; |
| ... | ... | @@ -6414,6 +6230,8 @@ pub const FuncGen = struct { |
| 6414 | 6230 | } |
| 6415 | 6231 | |
| 6416 | 6232 | fn airDbgStmt(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 6233 | if (!self.dg.object.builder.useLibLlvm()) return .none; |
| 6234 | |
| 6417 | 6235 | const di_scope = self.di_scope orelse return .none; |
| 6418 | 6236 | const dbg_stmt = self.air.instructions.items(.data)[inst].dbg_stmt; |
| 6419 | 6237 | self.prev_dbg_line = @intCast(self.base_line + dbg_stmt.line + 1); |
| ... | ... | @@ -6422,12 +6240,19 @@ pub const FuncGen = struct { |
| 6422 | 6240 | self.dbg_inlined.items[self.dbg_inlined.items.len - 1].loc |
| 6423 | 6241 | else |
| 6424 | 6242 | null; |
| 6425 | | self.builder.setCurrentDebugLocation(self.prev_dbg_line, self.prev_dbg_column, di_scope, inlined_at); |
| 6243 | self.wip.llvm.builder.setCurrentDebugLocation( |
| 6244 | self.prev_dbg_line, |
| 6245 | self.prev_dbg_column, |
| 6246 | di_scope, |
| 6247 | inlined_at, |
| 6248 | ); |
| 6426 | 6249 | return .none; |
| 6427 | 6250 | } |
| 6428 | 6251 | |
| 6429 | 6252 | fn airDbgInlineBegin(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 6430 | 6253 | const o = self.dg.object; |
| 6254 | if (!o.builder.useLibLlvm()) return .none; |
| 6255 | |
| 6431 | 6256 | const dib = o.di_builder orelse return .none; |
| 6432 | 6257 | const ty_fn = self.air.instructions.items(.data)[inst].ty_fn; |
| 6433 | 6258 | |
| ... | ... | @@ -6438,7 +6263,7 @@ pub const FuncGen = struct { |
| 6438 | 6263 | const di_file = try o.getDIFile(self.gpa, mod.namespacePtr(decl.src_namespace).file_scope); |
| 6439 | 6264 | self.di_file = di_file; |
| 6440 | 6265 | const line_number = decl.src_line + 1; |
| 6441 | | const cur_debug_location = self.builder.getCurrentDebugLocation2(); |
| 6266 | const cur_debug_location = self.wip.llvm.builder.getCurrentDebugLocation2(); |
| 6442 | 6267 | |
| 6443 | 6268 | try self.dbg_inlined.append(self.gpa, .{ |
| 6444 | 6269 | .loc = @ptrCast(cur_debug_location), |
| ... | ... | @@ -6486,6 +6311,8 @@ pub const FuncGen = struct { |
| 6486 | 6311 | |
| 6487 | 6312 | fn airDbgInlineEnd(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 6488 | 6313 | const o = self.dg.object; |
| 6314 | if (!o.builder.useLibLlvm()) return .none; |
| 6315 | |
| 6489 | 6316 | if (o.di_builder == null) return .none; |
| 6490 | 6317 | const ty_fn = self.air.instructions.items(.data)[inst].ty_fn; |
| 6491 | 6318 | |
| ... | ... | @@ -6501,6 +6328,8 @@ pub const FuncGen = struct { |
| 6501 | 6328 | |
| 6502 | 6329 | fn airDbgBlockBegin(self: *FuncGen) !Builder.Value { |
| 6503 | 6330 | const o = self.dg.object; |
| 6331 | if (!o.builder.useLibLlvm()) return .none; |
| 6332 | |
| 6504 | 6333 | const dib = o.di_builder orelse return .none; |
| 6505 | 6334 | const old_scope = self.di_scope.?; |
| 6506 | 6335 | try self.dbg_block_stack.append(self.gpa, old_scope); |
| ... | ... | @@ -6511,6 +6340,8 @@ pub const FuncGen = struct { |
| 6511 | 6340 | |
| 6512 | 6341 | fn airDbgBlockEnd(self: *FuncGen) !Builder.Value { |
| 6513 | 6342 | const o = self.dg.object; |
| 6343 | if (!o.builder.useLibLlvm()) return .none; |
| 6344 | |
| 6514 | 6345 | if (o.di_builder == null) return .none; |
| 6515 | 6346 | self.di_scope = self.dbg_block_stack.pop(); |
| 6516 | 6347 | return .none; |
| ... | ... | @@ -6518,6 +6349,8 @@ pub const FuncGen = struct { |
| 6518 | 6349 | |
| 6519 | 6350 | fn airDbgVarPtr(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 6520 | 6351 | const o = self.dg.object; |
| 6352 | if (!o.builder.useLibLlvm()) return .none; |
| 6353 | |
| 6521 | 6354 | const mod = o.module; |
| 6522 | 6355 | const dib = o.di_builder orelse return .none; |
| 6523 | 6356 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| ... | ... | @@ -6546,6 +6379,8 @@ pub const FuncGen = struct { |
| 6546 | 6379 | |
| 6547 | 6380 | fn airDbgVarVal(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 6548 | 6381 | const o = self.dg.object; |
| 6382 | if (!o.builder.useLibLlvm()) return .none; |
| 6383 | |
| 6549 | 6384 | const dib = o.di_builder orelse return .none; |
| 6550 | 6385 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 6551 | 6386 | const operand = try self.resolveInst(pl_op.operand); |
| ... | ... | @@ -7346,7 +7181,7 @@ pub const FuncGen = struct { |
| 7346 | 7181 | const o = self.dg.object; |
| 7347 | 7182 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 7348 | 7183 | const index = pl_op.payload; |
| 7349 | | return self.wip.callIntrinsic(.none, .@"wasm.memory.size", &.{.i32}, &.{ |
| 7184 | return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.size", &.{.i32}, &.{ |
| 7350 | 7185 | try o.builder.intValue(.i32, index), |
| 7351 | 7186 | }, ""); |
| 7352 | 7187 | } |
| ... | ... | @@ -7355,7 +7190,7 @@ pub const FuncGen = struct { |
| 7355 | 7190 | const o = self.dg.object; |
| 7356 | 7191 | const pl_op = self.air.instructions.items(.data)[inst].pl_op; |
| 7357 | 7192 | const index = pl_op.payload; |
| 7358 | | return self.wip.callIntrinsic(.none, .@"wasm.memory.grow", &.{.i32}, &.{ |
| 7193 | return self.wip.callIntrinsic(.normal, .none, .@"wasm.memory.grow", &.{.i32}, &.{ |
| 7359 | 7194 | try o.builder.intValue(.i32, index), try self.resolveInst(pl_op.operand), |
| 7360 | 7195 | }, ""); |
| 7361 | 7196 | } |
| ... | ... | @@ -7391,8 +7226,9 @@ pub const FuncGen = struct { |
| 7391 | 7226 | const inst_ty = self.typeOfIndex(inst); |
| 7392 | 7227 | const scalar_ty = inst_ty.scalarType(mod); |
| 7393 | 7228 | |
| 7394 | | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, inst_ty, 2, .{ lhs, rhs }); |
| 7229 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmin, .normal, inst_ty, 2, .{ lhs, rhs }); |
| 7395 | 7230 | return self.wip.callIntrinsic( |
| 7231 | .normal, |
| 7396 | 7232 | .none, |
| 7397 | 7233 | if (scalar_ty.isSignedInt(mod)) .smin else .umin, |
| 7398 | 7234 | &.{try o.lowerType(inst_ty)}, |
| ... | ... | @@ -7410,8 +7246,9 @@ pub const FuncGen = struct { |
| 7410 | 7246 | const inst_ty = self.typeOfIndex(inst); |
| 7411 | 7247 | const scalar_ty = inst_ty.scalarType(mod); |
| 7412 | 7248 | |
| 7413 | | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, inst_ty, 2, .{ lhs, rhs }); |
| 7249 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.fmax, .normal, inst_ty, 2, .{ lhs, rhs }); |
| 7414 | 7250 | return self.wip.callIntrinsic( |
| 7251 | .normal, |
| 7415 | 7252 | .none, |
| 7416 | 7253 | if (scalar_ty.isSignedInt(mod)) .smax else .umax, |
| 7417 | 7254 | &.{try o.lowerType(inst_ty)}, |
| ... | ... | @@ -7430,9 +7267,7 @@ pub const FuncGen = struct { |
| 7430 | 7267 | return self.wip.buildAggregate(try o.lowerType(inst_ty), &.{ ptr, len }, ""); |
| 7431 | 7268 | } |
| 7432 | 7269 | |
| 7433 | | fn airAdd(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7434 | | self.builder.setFastMath(want_fast_math); |
| 7435 | | |
| 7270 | fn airAdd(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7436 | 7271 | const o = self.dg.object; |
| 7437 | 7272 | const mod = o.module; |
| 7438 | 7273 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7441,7 +7276,7 @@ pub const FuncGen = struct { |
| 7441 | 7276 | const inst_ty = self.typeOfIndex(inst); |
| 7442 | 7277 | const scalar_ty = inst_ty.scalarType(mod); |
| 7443 | 7278 | |
| 7444 | | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, inst_ty, 2, .{ lhs, rhs }); |
| 7279 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.add, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7445 | 7280 | return self.wip.bin(if (scalar_ty.isSignedInt(mod)) .@"add nsw" else .@"add nuw", lhs, rhs, ""); |
| 7446 | 7281 | } |
| 7447 | 7282 | |
| ... | ... | @@ -7463,12 +7298,13 @@ pub const FuncGen = struct { |
| 7463 | 7298 | const intrinsic = if (scalar_ty.isSignedInt(mod)) signed_intrinsic else unsigned_intrinsic; |
| 7464 | 7299 | const llvm_inst_ty = try o.lowerType(inst_ty); |
| 7465 | 7300 | const results = |
| 7466 | | try fg.wip.callIntrinsic(.none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); |
| 7301 | try fg.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_inst_ty}, &.{ lhs, rhs }, ""); |
| 7467 | 7302 | |
| 7468 | 7303 | const overflow_bits = try fg.wip.extractValue(results, &.{1}, ""); |
| 7469 | 7304 | const overflow_bits_ty = overflow_bits.typeOfWip(&fg.wip); |
| 7470 | 7305 | const overflow_bit = if (overflow_bits_ty.isVector(&o.builder)) |
| 7471 | 7306 | try fg.wip.callIntrinsic( |
| 7307 | .normal, |
| 7472 | 7308 | .none, |
| 7473 | 7309 | .@"vector.reduce.or", |
| 7474 | 7310 | &.{overflow_bits_ty}, |
| ... | ... | @@ -7508,6 +7344,7 @@ pub const FuncGen = struct { |
| 7508 | 7344 | |
| 7509 | 7345 | if (scalar_ty.isAnyFloat()) return self.todo("saturating float add", .{}); |
| 7510 | 7346 | return self.wip.callIntrinsic( |
| 7347 | .normal, |
| 7511 | 7348 | .none, |
| 7512 | 7349 | if (scalar_ty.isSignedInt(mod)) .@"sadd.sat" else .@"uadd.sat", |
| 7513 | 7350 | &.{try o.lowerType(inst_ty)}, |
| ... | ... | @@ -7516,9 +7353,7 @@ pub const FuncGen = struct { |
| 7516 | 7353 | ); |
| 7517 | 7354 | } |
| 7518 | 7355 | |
| 7519 | | fn airSub(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7520 | | self.builder.setFastMath(want_fast_math); |
| 7521 | | |
| 7356 | fn airSub(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7522 | 7357 | const o = self.dg.object; |
| 7523 | 7358 | const mod = o.module; |
| 7524 | 7359 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7527,7 +7362,7 @@ pub const FuncGen = struct { |
| 7527 | 7362 | const inst_ty = self.typeOfIndex(inst); |
| 7528 | 7363 | const scalar_ty = inst_ty.scalarType(mod); |
| 7529 | 7364 | |
| 7530 | | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, inst_ty, 2, .{ lhs, rhs }); |
| 7365 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.sub, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7531 | 7366 | return self.wip.bin(if (scalar_ty.isSignedInt(mod)) .@"sub nsw" else .@"sub nuw", lhs, rhs, ""); |
| 7532 | 7367 | } |
| 7533 | 7368 | |
| ... | ... | @@ -7550,6 +7385,7 @@ pub const FuncGen = struct { |
| 7550 | 7385 | |
| 7551 | 7386 | if (scalar_ty.isAnyFloat()) return self.todo("saturating float sub", .{}); |
| 7552 | 7387 | return self.wip.callIntrinsic( |
| 7388 | .normal, |
| 7553 | 7389 | .none, |
| 7554 | 7390 | if (scalar_ty.isSignedInt(mod)) .@"ssub.sat" else .@"usub.sat", |
| 7555 | 7391 | &.{try o.lowerType(inst_ty)}, |
| ... | ... | @@ -7558,9 +7394,7 @@ pub const FuncGen = struct { |
| 7558 | 7394 | ); |
| 7559 | 7395 | } |
| 7560 | 7396 | |
| 7561 | | fn airMul(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7562 | | self.builder.setFastMath(want_fast_math); |
| 7563 | | |
| 7397 | fn airMul(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7564 | 7398 | const o = self.dg.object; |
| 7565 | 7399 | const mod = o.module; |
| 7566 | 7400 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7569,7 +7403,7 @@ pub const FuncGen = struct { |
| 7569 | 7403 | const inst_ty = self.typeOfIndex(inst); |
| 7570 | 7404 | const scalar_ty = inst_ty.scalarType(mod); |
| 7571 | 7405 | |
| 7572 | | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, inst_ty, 2, .{ lhs, rhs }); |
| 7406 | if (scalar_ty.isAnyFloat()) return self.buildFloatOp(.mul, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7573 | 7407 | return self.wip.bin(if (scalar_ty.isSignedInt(mod)) .@"mul nsw" else .@"mul nuw", lhs, rhs, ""); |
| 7574 | 7408 | } |
| 7575 | 7409 | |
| ... | ... | @@ -7592,6 +7426,7 @@ pub const FuncGen = struct { |
| 7592 | 7426 | |
| 7593 | 7427 | if (scalar_ty.isAnyFloat()) return self.todo("saturating float mul", .{}); |
| 7594 | 7428 | return self.wip.callIntrinsic( |
| 7429 | .normal, |
| 7595 | 7430 | .none, |
| 7596 | 7431 | if (scalar_ty.isSignedInt(mod)) .@"smul.fix.sat" else .@"umul.fix.sat", |
| 7597 | 7432 | &.{try o.lowerType(inst_ty)}, |
| ... | ... | @@ -7600,20 +7435,16 @@ pub const FuncGen = struct { |
| 7600 | 7435 | ); |
| 7601 | 7436 | } |
| 7602 | 7437 | |
| 7603 | | fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7604 | | self.builder.setFastMath(want_fast_math); |
| 7605 | | |
| 7438 | fn airDivFloat(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7606 | 7439 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| 7607 | 7440 | const lhs = try self.resolveInst(bin_op.lhs); |
| 7608 | 7441 | const rhs = try self.resolveInst(bin_op.rhs); |
| 7609 | 7442 | const inst_ty = self.typeOfIndex(inst); |
| 7610 | 7443 | |
| 7611 | | return self.buildFloatOp(.div, inst_ty, 2, .{ lhs, rhs }); |
| 7444 | return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7612 | 7445 | } |
| 7613 | 7446 | |
| 7614 | | fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7615 | | self.builder.setFastMath(want_fast_math); |
| 7616 | | |
| 7447 | fn airDivTrunc(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7617 | 7448 | const o = self.dg.object; |
| 7618 | 7449 | const mod = o.module; |
| 7619 | 7450 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7623,15 +7454,13 @@ pub const FuncGen = struct { |
| 7623 | 7454 | const scalar_ty = inst_ty.scalarType(mod); |
| 7624 | 7455 | |
| 7625 | 7456 | if (scalar_ty.isRuntimeFloat()) { |
| 7626 | | const result = try self.buildFloatOp(.div, inst_ty, 2, .{ lhs, rhs }); |
| 7627 | | return self.buildFloatOp(.trunc, inst_ty, 1, .{result}); |
| 7457 | const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7458 | return self.buildFloatOp(.trunc, fast, inst_ty, 1, .{result}); |
| 7628 | 7459 | } |
| 7629 | 7460 | return self.wip.bin(if (scalar_ty.isSignedInt(mod)) .sdiv else .udiv, lhs, rhs, ""); |
| 7630 | 7461 | } |
| 7631 | 7462 | |
| 7632 | | fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7633 | | self.builder.setFastMath(want_fast_math); |
| 7634 | | |
| 7463 | fn airDivFloor(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7635 | 7464 | const o = self.dg.object; |
| 7636 | 7465 | const mod = o.module; |
| 7637 | 7466 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7641,8 +7470,8 @@ pub const FuncGen = struct { |
| 7641 | 7470 | const scalar_ty = inst_ty.scalarType(mod); |
| 7642 | 7471 | |
| 7643 | 7472 | if (scalar_ty.isRuntimeFloat()) { |
| 7644 | | const result = try self.buildFloatOp(.div, inst_ty, 2, .{ lhs, rhs }); |
| 7645 | | return self.buildFloatOp(.floor, inst_ty, 1, .{result}); |
| 7473 | const result = try self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7474 | return self.buildFloatOp(.floor, fast, inst_ty, 1, .{result}); |
| 7646 | 7475 | } |
| 7647 | 7476 | if (scalar_ty.isSignedInt(mod)) { |
| 7648 | 7477 | const inst_llvm_ty = try o.lowerType(inst_ty); |
| ... | ... | @@ -7657,15 +7486,13 @@ pub const FuncGen = struct { |
| 7657 | 7486 | const div_sign_mask = try self.wip.bin(.ashr, div_sign, bit_size_minus_one, ""); |
| 7658 | 7487 | const zero = try o.builder.zeroInitValue(inst_llvm_ty); |
| 7659 | 7488 | const rem_nonzero = try self.wip.icmp(.ne, rem, zero, ""); |
| 7660 | | const correction = try self.wip.select(rem_nonzero, div_sign_mask, zero, ""); |
| 7489 | const correction = try self.wip.select(.normal, rem_nonzero, div_sign_mask, zero, ""); |
| 7661 | 7490 | return self.wip.bin(.@"add nsw", div, correction, ""); |
| 7662 | 7491 | } |
| 7663 | 7492 | return self.wip.bin(.udiv, lhs, rhs, ""); |
| 7664 | 7493 | } |
| 7665 | 7494 | |
| 7666 | | fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7667 | | self.builder.setFastMath(want_fast_math); |
| 7668 | | |
| 7495 | fn airDivExact(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7669 | 7496 | const o = self.dg.object; |
| 7670 | 7497 | const mod = o.module; |
| 7671 | 7498 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7674,16 +7501,16 @@ pub const FuncGen = struct { |
| 7674 | 7501 | const inst_ty = self.typeOfIndex(inst); |
| 7675 | 7502 | const scalar_ty = inst_ty.scalarType(mod); |
| 7676 | 7503 | |
| 7677 | | if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, inst_ty, 2, .{ lhs, rhs }); |
| 7678 | | return self.wip.bin(if (scalar_ty.isSignedInt(mod)) |
| 7679 | | .@"sdiv exact" |
| 7680 | | else |
| 7681 | | .@"udiv exact", lhs, rhs, ""); |
| 7504 | if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.div, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7505 | return self.wip.bin( |
| 7506 | if (scalar_ty.isSignedInt(mod)) .@"sdiv exact" else .@"udiv exact", |
| 7507 | lhs, |
| 7508 | rhs, |
| 7509 | "", |
| 7510 | ); |
| 7682 | 7511 | } |
| 7683 | 7512 | |
| 7684 | | fn airRem(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7685 | | self.builder.setFastMath(want_fast_math); |
| 7686 | | |
| 7513 | fn airRem(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7687 | 7514 | const o = self.dg.object; |
| 7688 | 7515 | const mod = o.module; |
| 7689 | 7516 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7692,16 +7519,15 @@ pub const FuncGen = struct { |
| 7692 | 7519 | const inst_ty = self.typeOfIndex(inst); |
| 7693 | 7520 | const scalar_ty = inst_ty.scalarType(mod); |
| 7694 | 7521 | |
| 7695 | | if (scalar_ty.isRuntimeFloat()) return self.buildFloatOp(.fmod, inst_ty, 2, .{ lhs, rhs }); |
| 7522 | if (scalar_ty.isRuntimeFloat()) |
| 7523 | return self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7696 | 7524 | return self.wip.bin(if (scalar_ty.isSignedInt(mod)) |
| 7697 | 7525 | .srem |
| 7698 | 7526 | else |
| 7699 | 7527 | .urem, lhs, rhs, ""); |
| 7700 | 7528 | } |
| 7701 | 7529 | |
| 7702 | | fn airMod(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 7703 | | self.builder.setFastMath(want_fast_math); |
| 7704 | | |
| 7530 | fn airMod(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 7705 | 7531 | const o = self.dg.object; |
| 7706 | 7532 | const mod = o.module; |
| 7707 | 7533 | const bin_op = self.air.instructions.items(.data)[inst].bin_op; |
| ... | ... | @@ -7712,12 +7538,12 @@ pub const FuncGen = struct { |
| 7712 | 7538 | const scalar_ty = inst_ty.scalarType(mod); |
| 7713 | 7539 | |
| 7714 | 7540 | if (scalar_ty.isRuntimeFloat()) { |
| 7715 | | const a = try self.buildFloatOp(.fmod, inst_ty, 2, .{ lhs, rhs }); |
| 7716 | | const b = try self.buildFloatOp(.add, inst_ty, 2, .{ a, rhs }); |
| 7717 | | const c = try self.buildFloatOp(.fmod, inst_ty, 2, .{ b, rhs }); |
| 7541 | const a = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ lhs, rhs }); |
| 7542 | const b = try self.buildFloatOp(.add, fast, inst_ty, 2, .{ a, rhs }); |
| 7543 | const c = try self.buildFloatOp(.fmod, fast, inst_ty, 2, .{ b, rhs }); |
| 7718 | 7544 | const zero = try o.builder.zeroInitValue(inst_llvm_ty); |
| 7719 | | const ltz = try self.buildFloatCmp(.lt, inst_ty, .{ lhs, zero }); |
| 7720 | | return self.wip.select(ltz, c, a, ""); |
| 7545 | const ltz = try self.buildFloatCmp(fast, .lt, inst_ty, .{ lhs, zero }); |
| 7546 | return self.wip.select(fast, ltz, c, a, ""); |
| 7721 | 7547 | } |
| 7722 | 7548 | if (scalar_ty.isSignedInt(mod)) { |
| 7723 | 7549 | const bit_size_minus_one = try o.builder.splatValue(inst_llvm_ty, try o.builder.intConst( |
| ... | ... | @@ -7731,7 +7557,7 @@ pub const FuncGen = struct { |
| 7731 | 7557 | const rhs_masked = try self.wip.bin(.@"and", rhs, div_sign_mask, ""); |
| 7732 | 7558 | const zero = try o.builder.zeroInitValue(inst_llvm_ty); |
| 7733 | 7559 | const rem_nonzero = try self.wip.icmp(.ne, rem, zero, ""); |
| 7734 | | const correction = try self.wip.select(rem_nonzero, rhs_masked, zero, ""); |
| 7560 | const correction = try self.wip.select(.normal, rem_nonzero, rhs_masked, zero, ""); |
| 7735 | 7561 | return self.wip.bin(.@"add nsw", rem, correction, ""); |
| 7736 | 7562 | } |
| 7737 | 7563 | return self.wip.bin(.urem, lhs, rhs, ""); |
| ... | ... | @@ -7804,7 +7630,7 @@ pub const FuncGen = struct { |
| 7804 | 7630 | const llvm_inst_ty = try o.lowerType(inst_ty); |
| 7805 | 7631 | const llvm_lhs_ty = try o.lowerType(lhs_ty); |
| 7806 | 7632 | const results = |
| 7807 | | try self.wip.callIntrinsic(.none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); |
| 7633 | try self.wip.callIntrinsic(.normal, .none, intrinsic, &.{llvm_lhs_ty}, &.{ lhs, rhs }, ""); |
| 7808 | 7634 | |
| 7809 | 7635 | const result_val = try self.wip.extractValue(results, &.{0}, ""); |
| 7810 | 7636 | const overflow_bit = try self.wip.extractValue(results, &.{1}, ""); |
| ... | ... | @@ -7879,13 +7705,18 @@ pub const FuncGen = struct { |
| 7879 | 7705 | .function => |function| function, |
| 7880 | 7706 | else => unreachable, |
| 7881 | 7707 | }; |
| 7882 | | return o.builder.addFunction(try o.builder.fnType(return_type, param_types, .normal), fn_name); |
| 7708 | return o.builder.addFunction( |
| 7709 | try o.builder.fnType(return_type, param_types, .normal), |
| 7710 | fn_name, |
| 7711 | toLlvmAddressSpace(.generic, o.module.getTarget()), |
| 7712 | ); |
| 7883 | 7713 | } |
| 7884 | 7714 | |
| 7885 | 7715 | /// Creates a floating point comparison by lowering to the appropriate |
| 7886 | 7716 | /// hardware instruction or softfloat routine for the target |
| 7887 | 7717 | fn buildFloatCmp( |
| 7888 | 7718 | self: *FuncGen, |
| 7719 | fast: Builder.FastMathKind, |
| 7889 | 7720 | pred: math.CompareOperator, |
| 7890 | 7721 | ty: Type, |
| 7891 | 7722 | params: [2]Builder.Value, |
| ... | ... | @@ -7905,7 +7736,7 @@ pub const FuncGen = struct { |
| 7905 | 7736 | .gt => .ogt, |
| 7906 | 7737 | .gte => .oge, |
| 7907 | 7738 | }; |
| 7908 | | return self.wip.fcmp(cond, params[0], params[1], ""); |
| 7739 | return self.wip.fcmp(fast, cond, params[0], params[1], ""); |
| 7909 | 7740 | } |
| 7910 | 7741 | |
| 7911 | 7742 | const float_bits = scalar_ty.floatBits(target); |
| ... | ... | @@ -7996,6 +7827,7 @@ pub const FuncGen = struct { |
| 7996 | 7827 | fn buildFloatOp( |
| 7997 | 7828 | self: *FuncGen, |
| 7998 | 7829 | comptime op: FloatOp, |
| 7830 | fast: Builder.FastMathKind, |
| 7999 | 7831 | ty: Type, |
| 8000 | 7832 | comptime params_len: usize, |
| 8001 | 7833 | params: [params_len]Builder.Value, |
| ... | ... | @@ -8009,13 +7841,23 @@ pub const FuncGen = struct { |
| 8009 | 7841 | if (op != .tan and intrinsicsAllowed(scalar_ty, target)) switch (op) { |
| 8010 | 7842 | // Some operations are dedicated LLVM instructions, not available as intrinsics |
| 8011 | 7843 | .neg => return self.wip.un(.fneg, params[0], ""), |
| 8012 | | .add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (op) { |
| 8013 | | .add => .fadd, |
| 8014 | | .sub => .fsub, |
| 8015 | | .mul => .fmul, |
| 8016 | | .div => .fdiv, |
| 8017 | | .fmod => .frem, |
| 8018 | | else => unreachable, |
| 7844 | .add, .sub, .mul, .div, .fmod => return self.wip.bin(switch (fast) { |
| 7845 | .normal => switch (op) { |
| 7846 | .add => .fadd, |
| 7847 | .sub => .fsub, |
| 7848 | .mul => .fmul, |
| 7849 | .div => .fdiv, |
| 7850 | .fmod => .frem, |
| 7851 | else => unreachable, |
| 7852 | }, |
| 7853 | .fast => switch (op) { |
| 7854 | .add => .@"fadd fast", |
| 7855 | .sub => .@"fsub fast", |
| 7856 | .mul => .@"fmul fast", |
| 7857 | .div => .@"fdiv fast", |
| 7858 | .fmod => .@"frem fast", |
| 7859 | else => unreachable, |
| 7860 | }, |
| 8019 | 7861 | }, params[0], params[1], ""), |
| 8020 | 7862 | .fmax, |
| 8021 | 7863 | .fmin, |
| ... | ... | @@ -8033,7 +7875,7 @@ pub const FuncGen = struct { |
| 8033 | 7875 | .sqrt, |
| 8034 | 7876 | .trunc, |
| 8035 | 7877 | .fma, |
| 8036 | | => return self.wip.callIntrinsic(.none, switch (op) { |
| 7878 | => return self.wip.callIntrinsic(fast, .none, switch (op) { |
| 8037 | 7879 | .fmax => .maxnum, |
| 8038 | 7880 | .fmin => .minnum, |
| 8039 | 7881 | .ceil => .ceil, |
| ... | ... | @@ -8108,7 +7950,7 @@ pub const FuncGen = struct { |
| 8108 | 7950 | } |
| 8109 | 7951 | |
| 8110 | 7952 | return self.wip.call( |
| 8111 | | .normal, |
| 7953 | fast.toCallKind(), |
| 8112 | 7954 | .ccc, |
| 8113 | 7955 | .none, |
| 8114 | 7956 | libc_fn.typeOf(&o.builder), |
| ... | ... | @@ -8127,7 +7969,7 @@ pub const FuncGen = struct { |
| 8127 | 7969 | const addend = try self.resolveInst(pl_op.operand); |
| 8128 | 7970 | |
| 8129 | 7971 | const ty = self.typeOfIndex(inst); |
| 8130 | | return self.buildFloatOp(.fma, ty, 3, .{ mulend1, mulend2, addend }); |
| 7972 | return self.buildFloatOp(.fma, .normal, ty, 3, .{ mulend1, mulend2, addend }); |
| 8131 | 7973 | } |
| 8132 | 7974 | |
| 8133 | 7975 | fn airShlWithOverflow(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| ... | ... | @@ -8248,6 +8090,7 @@ pub const FuncGen = struct { |
| 8248 | 8090 | const llvm_lhs_ty = try o.lowerType(lhs_ty); |
| 8249 | 8091 | const llvm_lhs_scalar_ty = llvm_lhs_ty.scalarType(&o.builder); |
| 8250 | 8092 | const result = try self.wip.callIntrinsic( |
| 8093 | .normal, |
| 8251 | 8094 | .none, |
| 8252 | 8095 | if (lhs_scalar_ty.isSignedInt(mod)) .@"sshl.sat" else .@"ushl.sat", |
| 8253 | 8096 | &.{llvm_lhs_ty}, |
| ... | ... | @@ -8269,7 +8112,7 @@ pub const FuncGen = struct { |
| 8269 | 8112 | try o.builder.intConst(llvm_lhs_scalar_ty, -1), |
| 8270 | 8113 | ); |
| 8271 | 8114 | const in_range = try self.wip.icmp(.ult, rhs, bits, ""); |
| 8272 | | return self.wip.select(in_range, result, lhs_max, ""); |
| 8115 | return self.wip.select(.normal, in_range, result, lhs_max, ""); |
| 8273 | 8116 | } |
| 8274 | 8117 | |
| 8275 | 8118 | fn airShr(self: *FuncGen, inst: Air.Inst.Index, is_exact: bool) !Builder.Value { |
| ... | ... | @@ -8682,14 +8525,14 @@ pub const FuncGen = struct { |
| 8682 | 8525 | |
| 8683 | 8526 | fn airTrap(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 8684 | 8527 | _ = inst; |
| 8685 | | _ = try self.wip.callIntrinsic(.none, .trap, &.{}, &.{}, ""); |
| 8528 | _ = try self.wip.callIntrinsic(.normal, .none, .trap, &.{}, &.{}, ""); |
| 8686 | 8529 | _ = try self.wip.@"unreachable"(); |
| 8687 | 8530 | return .none; |
| 8688 | 8531 | } |
| 8689 | 8532 | |
| 8690 | 8533 | fn airBreakpoint(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 8691 | 8534 | _ = inst; |
| 8692 | | _ = try self.wip.callIntrinsic(.none, .debugtrap, &.{}, &.{}, ""); |
| 8535 | _ = try self.wip.callIntrinsic(.normal, .none, .debugtrap, &.{}, &.{}, ""); |
| 8693 | 8536 | return .none; |
| 8694 | 8537 | } |
| 8695 | 8538 | |
| ... | ... | @@ -8701,7 +8544,7 @@ pub const FuncGen = struct { |
| 8701 | 8544 | // https://github.com/ziglang/zig/issues/11946 |
| 8702 | 8545 | return o.builder.intValue(llvm_usize, 0); |
| 8703 | 8546 | } |
| 8704 | | const result = try self.wip.callIntrinsic(.none, .returnaddress, &.{}, &.{ |
| 8547 | const result = try self.wip.callIntrinsic(.normal, .none, .returnaddress, &.{}, &.{ |
| 8705 | 8548 | try o.builder.intValue(.i32, 0), |
| 8706 | 8549 | }, ""); |
| 8707 | 8550 | return self.wip.cast(.ptrtoint, result, llvm_usize, ""); |
| ... | ... | @@ -8710,7 +8553,7 @@ pub const FuncGen = struct { |
| 8710 | 8553 | fn airFrameAddress(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| 8711 | 8554 | _ = inst; |
| 8712 | 8555 | const o = self.dg.object; |
| 8713 | | const result = try self.wip.callIntrinsic(.none, .frameaddress, &.{.ptr}, &.{ |
| 8556 | const result = try self.wip.callIntrinsic(.normal, .none, .frameaddress, &.{.ptr}, &.{ |
| 8714 | 8557 | try o.builder.intValue(.i32, 0), |
| 8715 | 8558 | }, ""); |
| 8716 | 8559 | return self.wip.cast(.ptrtoint, result, try o.lowerType(Type.usize), ""); |
| ... | ... | @@ -8768,7 +8611,7 @@ pub const FuncGen = struct { |
| 8768 | 8611 | |
| 8769 | 8612 | if (optional_ty.optionalReprIsPayload(mod)) { |
| 8770 | 8613 | const zero = try o.builder.zeroInitValue(payload.typeOfWip(&self.wip)); |
| 8771 | | return self.wip.select(success_bit, zero, payload, ""); |
| 8614 | return self.wip.select(.normal, success_bit, zero, payload, ""); |
| 8772 | 8615 | } |
| 8773 | 8616 | |
| 8774 | 8617 | comptime assert(optional_layout_version == 3); |
| ... | ... | @@ -9053,8 +8896,8 @@ pub const FuncGen = struct { |
| 9053 | 8896 | access_kind: Builder.MemoryAccessKind, |
| 9054 | 8897 | ) !void { |
| 9055 | 8898 | const o = self.dg.object; |
| 9056 | | const llvm_usize_ty = try o.lowerType(Type.usize); |
| 9057 | | const cond = try self.cmp(len, try o.builder.intValue(llvm_usize_ty, 0), Type.usize, .neq); |
| 8899 | const usize_zero = try o.builder.intValue(try o.lowerType(Type.usize), 0); |
| 8900 | const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero); |
| 9058 | 8901 | const memset_block = try self.wip.block(1, "MemsetTrapSkip"); |
| 9059 | 8902 | const end_block = try self.wip.block(2, "MemsetTrapEnd"); |
| 9060 | 8903 | _ = try self.wip.brCond(cond, memset_block, end_block); |
| ... | ... | @@ -9087,8 +8930,8 @@ pub const FuncGen = struct { |
| 9087 | 8930 | std.Target.wasm.featureSetHas(o.target.cpu.features, .bulk_memory) and |
| 9088 | 8931 | dest_ptr_ty.isSlice(mod)) |
| 9089 | 8932 | { |
| 9090 | | const zero_usize = try o.builder.intValue(try o.lowerType(Type.usize), 0); |
| 9091 | | const cond = try self.cmp(len, zero_usize, Type.usize, .neq); |
| 8933 | const usize_zero = try o.builder.intValue(try o.lowerType(Type.usize), 0); |
| 8934 | const cond = try self.cmp(.normal, .neq, Type.usize, len, usize_zero); |
| 9092 | 8935 | const memcpy_block = try self.wip.block(1, "MemcpyTrapSkip"); |
| 9093 | 8936 | const end_block = try self.wip.block(2, "MemcpyTrapEnd"); |
| 9094 | 8937 | _ = try self.wip.brCond(cond, memcpy_block, end_block); |
| ... | ... | @@ -9166,17 +9009,15 @@ pub const FuncGen = struct { |
| 9166 | 9009 | const operand = try self.resolveInst(un_op); |
| 9167 | 9010 | const operand_ty = self.typeOf(un_op); |
| 9168 | 9011 | |
| 9169 | | return self.buildFloatOp(op, operand_ty, 1, .{operand}); |
| 9012 | return self.buildFloatOp(op, .normal, operand_ty, 1, .{operand}); |
| 9170 | 9013 | } |
| 9171 | 9014 | |
| 9172 | | fn airNeg(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 9173 | | self.builder.setFastMath(want_fast_math); |
| 9174 | | |
| 9015 | fn airNeg(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 9175 | 9016 | const un_op = self.air.instructions.items(.data)[inst].un_op; |
| 9176 | 9017 | const operand = try self.resolveInst(un_op); |
| 9177 | 9018 | const operand_ty = self.typeOf(un_op); |
| 9178 | 9019 | |
| 9179 | | return self.buildFloatOp(.neg, operand_ty, 1, .{operand}); |
| 9020 | return self.buildFloatOp(.neg, fast, operand_ty, 1, .{operand}); |
| 9180 | 9021 | } |
| 9181 | 9022 | |
| 9182 | 9023 | fn airClzCtz(self: *FuncGen, inst: Air.Inst.Index, intrinsic: Builder.Intrinsic) !Builder.Value { |
| ... | ... | @@ -9187,6 +9028,7 @@ pub const FuncGen = struct { |
| 9187 | 9028 | const operand = try self.resolveInst(ty_op.operand); |
| 9188 | 9029 | |
| 9189 | 9030 | const result = try self.wip.callIntrinsic( |
| 9031 | .normal, |
| 9190 | 9032 | .none, |
| 9191 | 9033 | intrinsic, |
| 9192 | 9034 | &.{try o.lowerType(operand_ty)}, |
| ... | ... | @@ -9204,6 +9046,7 @@ pub const FuncGen = struct { |
| 9204 | 9046 | const operand = try self.resolveInst(ty_op.operand); |
| 9205 | 9047 | |
| 9206 | 9048 | const result = try self.wip.callIntrinsic( |
| 9049 | .normal, |
| 9207 | 9050 | .none, |
| 9208 | 9051 | intrinsic, |
| 9209 | 9052 | &.{try o.lowerType(operand_ty)}, |
| ... | ... | @@ -9242,7 +9085,8 @@ pub const FuncGen = struct { |
| 9242 | 9085 | bits = bits + 8; |
| 9243 | 9086 | } |
| 9244 | 9087 | |
| 9245 | | const result = try self.wip.callIntrinsic(.none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); |
| 9088 | const result = |
| 9089 | try self.wip.callIntrinsic(.normal, .none, .bswap, &.{llvm_operand_ty}, &.{operand}, ""); |
| 9246 | 9090 | return self.wip.conv(.unsigned, result, try o.lowerType(inst_ty), ""); |
| 9247 | 9091 | } |
| 9248 | 9092 | |
| ... | ... | @@ -9309,20 +9153,18 @@ pub const FuncGen = struct { |
| 9309 | 9153 | const function_index = try o.builder.addFunction( |
| 9310 | 9154 | try o.builder.fnType(.i1, &.{try o.lowerType(enum_type.tag_ty.toType())}, .normal), |
| 9311 | 9155 | try o.builder.fmt("__zig_is_named_enum_value_{}", .{fqn.fmt(&mod.intern_pool)}), |
| 9156 | toLlvmAddressSpace(.generic, mod.getTarget()), |
| 9312 | 9157 | ); |
| 9313 | 9158 | |
| 9314 | 9159 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 9315 | 9160 | defer attributes.deinit(&o.builder); |
| 9316 | 9161 | try o.addCommonFnAttributes(&attributes); |
| 9317 | | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 9318 | 9162 | |
| 9319 | | function_index.ptrConst(&o.builder).global.ptr(&o.builder).linkage = .internal; |
| 9320 | | function_index.ptr(&o.builder).call_conv = .fastcc; |
| 9163 | function_index.setLinkage(.internal, &o.builder); |
| 9164 | function_index.setCallConv(.fastcc, &o.builder); |
| 9165 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 9321 | 9166 | gop.value_ptr.* = function_index; |
| 9322 | 9167 | |
| 9323 | | function_index.toLlvm(&o.builder).setLinkage(.Internal); |
| 9324 | | function_index.toLlvm(&o.builder).setFunctionCallConv(.Fast); |
| 9325 | | |
| 9326 | 9168 | var wip = try Builder.WipFunction.init(&o.builder, function_index); |
| 9327 | 9169 | defer wip.deinit(); |
| 9328 | 9170 | wip.cursor = .{ .block = try wip.block(0, "Entry") }; |
| ... | ... | @@ -9383,20 +9225,18 @@ pub const FuncGen = struct { |
| 9383 | 9225 | const function_index = try o.builder.addFunction( |
| 9384 | 9226 | try o.builder.fnType(ret_ty, &.{try o.lowerType(enum_type.tag_ty.toType())}, .normal), |
| 9385 | 9227 | try o.builder.fmt("__zig_tag_name_{}", .{fqn.fmt(&mod.intern_pool)}), |
| 9228 | toLlvmAddressSpace(.generic, mod.getTarget()), |
| 9386 | 9229 | ); |
| 9387 | 9230 | |
| 9388 | 9231 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 9389 | 9232 | defer attributes.deinit(&o.builder); |
| 9390 | 9233 | try o.addCommonFnAttributes(&attributes); |
| 9391 | | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 9392 | 9234 | |
| 9393 | | function_index.ptrConst(&o.builder).global.ptr(&o.builder).linkage = .internal; |
| 9394 | | function_index.ptr(&o.builder).call_conv = .fastcc; |
| 9235 | function_index.setLinkage(.internal, &o.builder); |
| 9236 | function_index.setCallConv(.fastcc, &o.builder); |
| 9237 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 9395 | 9238 | gop.value_ptr.* = function_index.ptrConst(&o.builder).global; |
| 9396 | 9239 | |
| 9397 | | function_index.toLlvm(&o.builder).setLinkage(.Internal); |
| 9398 | | function_index.toLlvm(&o.builder).setFunctionCallConv(.Fast); |
| 9399 | | |
| 9400 | 9240 | var wip = try Builder.WipFunction.init(&o.builder, function_index); |
| 9401 | 9241 | defer wip.deinit(); |
| 9402 | 9242 | wip.cursor = .{ .block = try wip.block(0, "Entry") }; |
| ... | ... | @@ -9407,36 +9247,20 @@ pub const FuncGen = struct { |
| 9407 | 9247 | try wip.@"switch"(tag_int_value, bad_value_block, @intCast(enum_type.names.len)); |
| 9408 | 9248 | defer wip_switch.finish(&wip); |
| 9409 | 9249 | |
| 9410 | | for (enum_type.names, 0..) |name_ip, field_index| { |
| 9411 | | const name = try o.builder.string(mod.intern_pool.stringToSlice(name_ip)); |
| 9412 | | const str_init = try o.builder.stringNullConst(name); |
| 9413 | | const str_ty = str_init.typeOf(&o.builder); |
| 9414 | | const str_llvm_global = o.llvm_module.addGlobal(str_ty.toLlvm(&o.builder), ""); |
| 9415 | | str_llvm_global.setInitializer(str_init.toLlvm(&o.builder)); |
| 9416 | | str_llvm_global.setLinkage(.Private); |
| 9417 | | str_llvm_global.setGlobalConstant(.True); |
| 9418 | | str_llvm_global.setUnnamedAddr(.True); |
| 9419 | | str_llvm_global.setAlignment(1); |
| 9420 | | |
| 9421 | | var str_global = Builder.Global{ |
| 9422 | | .linkage = .private, |
| 9423 | | .unnamed_addr = .unnamed_addr, |
| 9424 | | .type = str_ty, |
| 9425 | | .kind = .{ .variable = @enumFromInt(o.builder.variables.items.len) }, |
| 9426 | | }; |
| 9427 | | var str_variable = Builder.Variable{ |
| 9428 | | .global = @enumFromInt(o.builder.globals.count()), |
| 9429 | | .mutability = .constant, |
| 9430 | | .init = str_init, |
| 9431 | | .alignment = comptime Builder.Alignment.fromByteUnits(1), |
| 9432 | | }; |
| 9433 | | try o.builder.llvm.globals.append(o.gpa, str_llvm_global); |
| 9434 | | const global_index = try o.builder.addGlobal(.empty, str_global); |
| 9435 | | try o.builder.variables.append(o.gpa, str_variable); |
| 9436 | | |
| 9437 | | const slice_val = try o.builder.structValue(ret_ty, &.{ |
| 9438 | | global_index.toConst(), |
| 9439 | | try o.builder.intConst(usize_ty, name.slice(&o.builder).?.len), |
| 9250 | for (enum_type.names, 0..) |name, field_index| { |
| 9251 | const name_string = try o.builder.string(mod.intern_pool.stringToSlice(name)); |
| 9252 | const name_init = try o.builder.stringNullConst(name_string); |
| 9253 | const name_variable_index = |
| 9254 | try o.builder.addVariable(.empty, name_init.typeOf(&o.builder), .default); |
| 9255 | try name_variable_index.setInitializer(name_init, &o.builder); |
| 9256 | name_variable_index.setLinkage(.private, &o.builder); |
| 9257 | name_variable_index.setMutability(.constant, &o.builder); |
| 9258 | name_variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 9259 | name_variable_index.setAlignment(comptime Builder.Alignment.fromByteUnits(1), &o.builder); |
| 9260 | |
| 9261 | const name_val = try o.builder.structValue(ret_ty, &.{ |
| 9262 | name_variable_index.toConst(&o.builder), |
| 9263 | try o.builder.intConst(usize_ty, name_string.slice(&o.builder).?.len), |
| 9440 | 9264 | }); |
| 9441 | 9265 | |
| 9442 | 9266 | const return_block = try wip.block(1, "Name"); |
| ... | ... | @@ -9446,7 +9270,7 @@ pub const FuncGen = struct { |
| 9446 | 9270 | try wip_switch.addCase(this_tag_int_value, return_block, &wip); |
| 9447 | 9271 | |
| 9448 | 9272 | wip.cursor = .{ .block = return_block }; |
| 9449 | | _ = try wip.ret(slice_val); |
| 9273 | _ = try wip.ret(name_val); |
| 9450 | 9274 | } |
| 9451 | 9275 | |
| 9452 | 9276 | wip.cursor = .{ .block = bad_value_block }; |
| ... | ... | @@ -9465,19 +9289,16 @@ pub const FuncGen = struct { |
| 9465 | 9289 | const function_index = try o.builder.addFunction( |
| 9466 | 9290 | try o.builder.fnType(.i1, &.{Builder.Type.err_int}, .normal), |
| 9467 | 9291 | name, |
| 9292 | toLlvmAddressSpace(.generic, o.module.getTarget()), |
| 9468 | 9293 | ); |
| 9469 | 9294 | |
| 9470 | 9295 | var attributes: Builder.FunctionAttributes.Wip = .{}; |
| 9471 | 9296 | defer attributes.deinit(&o.builder); |
| 9472 | 9297 | try o.addCommonFnAttributes(&attributes); |
| 9473 | | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 9474 | | |
| 9475 | | function_index.ptrConst(&o.builder).global.ptr(&o.builder).linkage = .internal; |
| 9476 | | function_index.ptr(&o.builder).call_conv = .fastcc; |
| 9477 | | |
| 9478 | | function_index.toLlvm(&o.builder).setLinkage(.Internal); |
| 9479 | | function_index.toLlvm(&o.builder).setFunctionCallConv(.Fast); |
| 9480 | 9298 | |
| 9299 | function_index.setLinkage(.internal, &o.builder); |
| 9300 | function_index.setCallConv(.fastcc, &o.builder); |
| 9301 | function_index.setAttributes(try attributes.finish(&o.builder), &o.builder); |
| 9481 | 9302 | return function_index; |
| 9482 | 9303 | } |
| 9483 | 9304 | |
| ... | ... | @@ -9511,7 +9332,7 @@ pub const FuncGen = struct { |
| 9511 | 9332 | const a = try self.resolveInst(extra.lhs); |
| 9512 | 9333 | const b = try self.resolveInst(extra.rhs); |
| 9513 | 9334 | |
| 9514 | | return self.wip.select(pred, a, b, ""); |
| 9335 | return self.wip.select(.normal, pred, a, b, ""); |
| 9515 | 9336 | } |
| 9516 | 9337 | |
| 9517 | 9338 | fn airShuffle(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { |
| ... | ... | @@ -9623,8 +9444,7 @@ pub const FuncGen = struct { |
| 9623 | 9444 | return self.wip.load(.normal, llvm_result_ty, accum_ptr, .default, ""); |
| 9624 | 9445 | } |
| 9625 | 9446 | |
| 9626 | | fn airReduce(self: *FuncGen, inst: Air.Inst.Index, want_fast_math: bool) !Builder.Value { |
| 9627 | | self.builder.setFastMath(want_fast_math); |
| 9447 | fn airReduce(self: *FuncGen, inst: Air.Inst.Index, fast: Builder.FastMathKind) !Builder.Value { |
| 9628 | 9448 | const o = self.dg.object; |
| 9629 | 9449 | const mod = o.module; |
| 9630 | 9450 | const target = mod.getTarget(); |
| ... | ... | @@ -9637,14 +9457,14 @@ pub const FuncGen = struct { |
| 9637 | 9457 | const llvm_scalar_ty = try o.lowerType(scalar_ty); |
| 9638 | 9458 | |
| 9639 | 9459 | switch (reduce.operation) { |
| 9640 | | .And, .Or, .Xor => return self.wip.callIntrinsic(.none, switch (reduce.operation) { |
| 9460 | .And, .Or, .Xor => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { |
| 9641 | 9461 | .And => .@"vector.reduce.and", |
| 9642 | 9462 | .Or => .@"vector.reduce.or", |
| 9643 | 9463 | .Xor => .@"vector.reduce.xor", |
| 9644 | 9464 | else => unreachable, |
| 9645 | 9465 | }, &.{llvm_operand_ty}, &.{operand}, ""), |
| 9646 | 9466 | .Min, .Max => switch (scalar_ty.zigTypeTag(mod)) { |
| 9647 | | .Int => return self.wip.callIntrinsic(.none, switch (reduce.operation) { |
| 9467 | .Int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { |
| 9648 | 9468 | .Min => if (scalar_ty.isSignedInt(mod)) |
| 9649 | 9469 | .@"vector.reduce.smin" |
| 9650 | 9470 | else |
| ... | ... | @@ -9656,7 +9476,7 @@ pub const FuncGen = struct { |
| 9656 | 9476 | else => unreachable, |
| 9657 | 9477 | }, &.{llvm_operand_ty}, &.{operand}, ""), |
| 9658 | 9478 | .Float => if (intrinsicsAllowed(scalar_ty, target)) |
| 9659 | | return self.wip.callIntrinsic(.none, switch (reduce.operation) { |
| 9479 | return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { |
| 9660 | 9480 | .Min => .@"vector.reduce.fmin", |
| 9661 | 9481 | .Max => .@"vector.reduce.fmax", |
| 9662 | 9482 | else => unreachable, |
| ... | ... | @@ -9664,13 +9484,13 @@ pub const FuncGen = struct { |
| 9664 | 9484 | else => unreachable, |
| 9665 | 9485 | }, |
| 9666 | 9486 | .Add, .Mul => switch (scalar_ty.zigTypeTag(mod)) { |
| 9667 | | .Int => return self.wip.callIntrinsic(.none, switch (reduce.operation) { |
| 9487 | .Int => return self.wip.callIntrinsic(.normal, .none, switch (reduce.operation) { |
| 9668 | 9488 | .Add => .@"vector.reduce.add", |
| 9669 | 9489 | .Mul => .@"vector.reduce.mul", |
| 9670 | 9490 | else => unreachable, |
| 9671 | 9491 | }, &.{llvm_operand_ty}, &.{operand}, ""), |
| 9672 | 9492 | .Float => if (intrinsicsAllowed(scalar_ty, target)) |
| 9673 | | return self.wip.callIntrinsic(.none, switch (reduce.operation) { |
| 9493 | return self.wip.callIntrinsic(fast, .none, switch (reduce.operation) { |
| 9674 | 9494 | .Add => .@"vector.reduce.fadd", |
| 9675 | 9495 | .Mul => .@"vector.reduce.fmul", |
| 9676 | 9496 | else => unreachable, |
| ... | ... | @@ -10021,7 +9841,7 @@ pub const FuncGen = struct { |
| 10021 | 9841 | .data => {}, |
| 10022 | 9842 | } |
| 10023 | 9843 | |
| 10024 | | _ = try self.wip.callIntrinsic(.none, .prefetch, &.{.ptr}, &.{ |
| 9844 | _ = try self.wip.callIntrinsic(.normal, .none, .prefetch, &.{.ptr}, &.{ |
| 10025 | 9845 | try self.resolveInst(prefetch.ptr), |
| 10026 | 9846 | try o.builder.intValue(.i32, prefetch.rw), |
| 10027 | 9847 | try o.builder.intValue(.i32, prefetch.locality), |
| ... | ... | @@ -10045,7 +9865,7 @@ pub const FuncGen = struct { |
| 10045 | 9865 | default: u32, |
| 10046 | 9866 | comptime basename: []const u8, |
| 10047 | 9867 | ) !Builder.Value { |
| 10048 | | return self.wip.callIntrinsic(.none, switch (dimension) { |
| 9868 | return self.wip.callIntrinsic(.normal, .none, switch (dimension) { |
| 10049 | 9869 | 0 => @field(Builder.Intrinsic, basename ++ ".x"), |
| 10050 | 9870 | 1 => @field(Builder.Intrinsic, basename ++ ".y"), |
| 10051 | 9871 | 2 => @field(Builder.Intrinsic, basename ++ ".z"), |
| ... | ... | @@ -10074,7 +9894,8 @@ pub const FuncGen = struct { |
| 10074 | 9894 | |
| 10075 | 9895 | // Fetch the dispatch pointer, which points to this structure: |
| 10076 | 9896 | // https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/adae6c61e10d371f7cbc3d0e94ae2c070cab18a4/src/inc/hsa.h#L2913 |
| 10077 | | const dispatch_ptr = try self.wip.callIntrinsic(.none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, ""); |
| 9897 | const dispatch_ptr = |
| 9898 | try self.wip.callIntrinsic(.normal, .none, .@"amdgcn.dispatch.ptr", &.{}, &.{}, ""); |
| 10078 | 9899 | |
| 10079 | 9900 | // Load the work_group_* member from the struct as u16. |
| 10080 | 9901 | // Just treat the dispatch pointer as an array of u16 to keep things simple. |
| ... | ... | @@ -10097,40 +9918,24 @@ pub const FuncGen = struct { |
| 10097 | 9918 | |
| 10098 | 9919 | fn getErrorNameTable(self: *FuncGen) Allocator.Error!Builder.Variable.Index { |
| 10099 | 9920 | const o = self.dg.object; |
| 9921 | const mod = o.module; |
| 9922 | |
| 10100 | 9923 | const table = o.error_name_table; |
| 10101 | 9924 | if (table != .none) return table; |
| 10102 | 9925 | |
| 10103 | | const mod = o.module; |
| 10104 | | const slice_ty = Type.slice_const_u8_sentinel_0; |
| 10105 | | const slice_alignment = slice_ty.abiAlignment(mod); |
| 10106 | | const undef_init = try o.builder.undefConst(.ptr); // TODO: Address space |
| 10107 | | |
| 10108 | | const name = try o.builder.string("__zig_err_name_table"); |
| 10109 | | const error_name_table_global = o.llvm_module.addGlobal(Builder.Type.ptr.toLlvm(&o.builder), name.slice(&o.builder).?); |
| 10110 | | error_name_table_global.setInitializer(undef_init.toLlvm(&o.builder)); |
| 10111 | | error_name_table_global.setLinkage(.Private); |
| 10112 | | error_name_table_global.setGlobalConstant(.True); |
| 10113 | | error_name_table_global.setUnnamedAddr(.True); |
| 10114 | | error_name_table_global.setAlignment(slice_alignment); |
| 10115 | | |
| 10116 | | var global = Builder.Global{ |
| 10117 | | .linkage = .private, |
| 10118 | | .unnamed_addr = .unnamed_addr, |
| 10119 | | .type = .ptr, |
| 10120 | | .kind = .{ .variable = @enumFromInt(o.builder.variables.items.len) }, |
| 10121 | | }; |
| 10122 | | var variable = Builder.Variable{ |
| 10123 | | .global = @enumFromInt(o.builder.globals.count()), |
| 10124 | | .mutability = .constant, |
| 10125 | | .init = undef_init, |
| 10126 | | .alignment = Builder.Alignment.fromByteUnits(slice_alignment), |
| 10127 | | }; |
| 10128 | | try o.builder.llvm.globals.append(o.gpa, error_name_table_global); |
| 10129 | | _ = try o.builder.addGlobal(name, global); |
| 10130 | | try o.builder.variables.append(o.gpa, variable); |
| 9926 | // TODO: Address space |
| 9927 | const variable_index = |
| 9928 | try o.builder.addVariable(try o.builder.string("__zig_err_name_table"), .ptr, .default); |
| 9929 | variable_index.setLinkage(.private, &o.builder); |
| 9930 | variable_index.setMutability(.constant, &o.builder); |
| 9931 | variable_index.setUnnamedAddr(.unnamed_addr, &o.builder); |
| 9932 | variable_index.setAlignment( |
| 9933 | Builder.Alignment.fromByteUnits(Type.slice_const_u8_sentinel_0.abiAlignment(mod)), |
| 9934 | &o.builder, |
| 9935 | ); |
| 10131 | 9936 | |
| 10132 | | o.error_name_table = global.kind.variable; |
| 10133 | | return global.kind.variable; |
| 9937 | o.error_name_table = variable_index; |
| 9938 | return variable_index; |
| 10134 | 9939 | } |
| 10135 | 9940 | |
| 10136 | 9941 | /// Assumes the optional is not pointer-like and payload has bits. |
| ... | ... | @@ -11547,15 +11352,19 @@ fn buildAllocaInner( |
| 11547 | 11352 | |
| 11548 | 11353 | const alloca = blk: { |
| 11549 | 11354 | const prev_cursor = wip.cursor; |
| 11550 | | const prev_debug_location = wip.llvm.builder.getCurrentDebugLocation2(); |
| 11355 | const prev_debug_location = if (wip.builder.useLibLlvm()) |
| 11356 | wip.llvm.builder.getCurrentDebugLocation2() |
| 11357 | else |
| 11358 | undefined; |
| 11551 | 11359 | defer { |
| 11552 | 11360 | wip.cursor = prev_cursor; |
| 11553 | 11361 | if (wip.cursor.block == .entry) wip.cursor.instruction += 1; |
| 11554 | | if (di_scope_non_null) wip.llvm.builder.setCurrentDebugLocation2(prev_debug_location); |
| 11362 | if (wip.builder.useLibLlvm() and di_scope_non_null) |
| 11363 | wip.llvm.builder.setCurrentDebugLocation2(prev_debug_location); |
| 11555 | 11364 | } |
| 11556 | 11365 | |
| 11557 | 11366 | wip.cursor = .{ .block = .entry }; |
| 11558 | | wip.llvm.builder.clearCurrentDebugLocation(); |
| 11367 | if (wip.builder.useLibLlvm()) wip.llvm.builder.clearCurrentDebugLocation(); |
| 11559 | 11368 | break :blk try wip.alloca(.normal, llvm_ty, .none, alignment, address_space, ""); |
| 11560 | 11369 | }; |
| 11561 | 11370 | |
Jacob Young <jacobly@ziglang.org> 2023-08-08 10:15:11-04:00
src