authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-07-28 17:06:57-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-07-31 12:59:19-07:00
log075f93fa108030fb0dd12faa6e389ace302cfb4c
tree73cc539ff6dc66dc67cb35ba30af0c9c2a263098
parent1ab15b6c9c529c9acc85f4b0bf3fcaea97a5a48e

stage2 LLVM: Pass inline assembly outputs directly when not targeting memory

This change provides a basic implementation of #2349 for stage2. There's still quite a lot of work before this logic is as complete as what's in Clang (https://github.com/llvm/llvm-project/blob/b3645353041818f61e2580635409ddb81ff5a272/clang/lib/CodeGen/CGStmt.cpp#L2304-L2795), particularly considering the diversity of constraints across targets. It's probably not worth doing the complete work until there's a clearer picture for constraints in Zig's future dedicated ASM syntax, but at least this gives us a small improvement for now. As a bonus, this also fixes a bug with how we were handling `_` identifiers.

1 files changed, 109 insertions(+), 32 deletions(-)

src/codegen/llvm.zig+109-32
...@@ -5491,22 +5491,26 @@ pub const FuncGen = struct {...@@ -5491,22 +5491,26 @@ pub const FuncGen = struct {
5491 defer arena_allocator.deinit();5491 defer arena_allocator.deinit();
5492 const arena = arena_allocator.allocator();5492 const arena = arena_allocator.allocator();
54935493
5494 const return_count: u8 = for (outputs) |output| {5494 // The exact number of return / parameter values depends on which output values
5495 if (output == .none) break 1;5495 // are passed by reference as indirect outputs (determined below).
5496 } else 0;5496 const max_return_count = outputs.len;
5497 const llvm_params_len = inputs.len + outputs.len - return_count;5497 const llvm_ret_types = try arena.alloc(*const llvm.Type, max_return_count);
5498 const llvm_param_types = try arena.alloc(*const llvm.Type, llvm_params_len);5498 const llvm_ret_indirect = try arena.alloc(bool, max_return_count);
5499 const llvm_param_values = try arena.alloc(*const llvm.Value, llvm_params_len);5499
5500 const llvm_param_attrs = try arena.alloc(bool, llvm_params_len);5500 const max_param_count = inputs.len + outputs.len;
5501 const llvm_param_types = try arena.alloc(*const llvm.Type, max_param_count);
5502 const llvm_param_values = try arena.alloc(*const llvm.Value, max_param_count);
5503 const llvm_param_attrs = try arena.alloc(bool, max_param_count);
5501 const target = self.dg.module.getTarget();5504 const target = self.dg.module.getTarget();
55025505
5506 var llvm_ret_i: usize = 0;
5503 var llvm_param_i: usize = 0;5507 var llvm_param_i: usize = 0;
5504 var total_i: usize = 0;5508 var total_i: u16 = 0;
55055509
5506 var name_map: std.StringArrayHashMapUnmanaged(void) = .{};5510 var name_map: std.StringArrayHashMapUnmanaged(u16) = .{};
5507 try name_map.ensureUnusedCapacity(arena, outputs.len + inputs.len);5511 try name_map.ensureUnusedCapacity(arena, max_param_count);
55085512
5509 for (outputs) |output| {5513 for (outputs) |output, i| {
5510 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);5514 const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]);
5511 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);5515 const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0);
5512 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);5516 const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0);
...@@ -5519,15 +5523,30 @@ pub const FuncGen = struct {...@@ -5519,15 +5523,30 @@ pub const FuncGen = struct {
5519 llvm_constraints.appendAssumeCapacity(',');5523 llvm_constraints.appendAssumeCapacity(',');
5520 }5524 }
5521 llvm_constraints.appendAssumeCapacity('=');5525 llvm_constraints.appendAssumeCapacity('=');
5526
5527 // Pass any non-return outputs indirectly, if the constraint accepts a memory location
5528 llvm_ret_indirect[i] = (output != .none) and constraintAllowsMemory(constraint);
5522 if (output != .none) {5529 if (output != .none) {
5523 try llvm_constraints.ensureUnusedCapacity(self.gpa, llvm_constraints.capacity + 1);5530 try llvm_constraints.ensureUnusedCapacity(self.gpa, llvm_constraints.capacity + 1);
5524 llvm_constraints.appendAssumeCapacity('*');
5525
5526 const output_inst = try self.resolveInst(output);5531 const output_inst = try self.resolveInst(output);
5527 llvm_param_values[llvm_param_i] = output_inst;5532
5528 llvm_param_types[llvm_param_i] = output_inst.typeOf();5533 if (llvm_ret_indirect[i]) {
5529 llvm_param_attrs[llvm_param_i] = true;5534 // Pass the result by reference as an indirect output (e.g. "=*m")
5530 llvm_param_i += 1;5535 llvm_constraints.appendAssumeCapacity('*');
5536
5537 llvm_param_values[llvm_param_i] = output_inst;
5538 llvm_param_types[llvm_param_i] = output_inst.typeOf();
5539 llvm_param_attrs[llvm_param_i] = true;
5540 llvm_param_i += 1;
5541 } else {
5542 // Pass the result directly (e.g. "=r")
5543 llvm_ret_types[llvm_ret_i] = output_inst.typeOf().getElementType();
5544 llvm_ret_i += 1;
5545 }
5546 } else {
5547 const ret_ty = self.air.typeOfIndex(inst);
5548 llvm_ret_types[llvm_ret_i] = try self.dg.lowerType(ret_ty);
5549 llvm_ret_i += 1;
5531 }5550 }
55325551
5533 // LLVM uses commas internally to separate different constraints,5552 // LLVM uses commas internally to separate different constraints,
...@@ -5536,13 +5555,16 @@ pub const FuncGen = struct {...@@ -5536,13 +5555,16 @@ pub const FuncGen = struct {
5536 // to GCC's inline assembly.5555 // to GCC's inline assembly.
5537 // http://llvm.org/docs/LangRef.html#constraint-codes5556 // http://llvm.org/docs/LangRef.html#constraint-codes
5538 for (constraint[1..]) |byte| {5557 for (constraint[1..]) |byte| {
5539 llvm_constraints.appendAssumeCapacity(switch (byte) {5558 switch (byte) {
5540 ',' => '|',5559 ',' => llvm_constraints.appendAssumeCapacity('|'),
5541 else => byte,5560 '*' => {}, // Indirect outputs are handled above
5542 });5561 else => llvm_constraints.appendAssumeCapacity(byte),
5562 }
5543 }5563 }
55445564
5545 name_map.putAssumeCapacityNoClobber(name, {});5565 if (!std.mem.eql(u8, name, "_")) {
5566 name_map.putAssumeCapacityNoClobber(name, total_i);
5567 }
5546 total_i += 1;5568 total_i += 1;
5547 }5569 }
55485570
...@@ -5594,7 +5616,7 @@ pub const FuncGen = struct {...@@ -5594,7 +5616,7 @@ pub const FuncGen = struct {
5594 }5616 }
55955617
5596 if (!std.mem.eql(u8, name, "_")) {5618 if (!std.mem.eql(u8, name, "_")) {
5597 name_map.putAssumeCapacityNoClobber(name, {});5619 name_map.putAssumeCapacityNoClobber(name, total_i);
5598 }5620 }
55995621
5600 // In the case of indirect inputs, LLVM requires the callsite to have5622 // In the case of indirect inputs, LLVM requires the callsite to have
...@@ -5625,6 +5647,11 @@ pub const FuncGen = struct {...@@ -5625,6 +5647,11 @@ pub const FuncGen = struct {
5625 }5647 }
5626 }5648 }
56275649
5650 // We have finished scanning through all inputs/outputs, so the number of
5651 // parameters and return values is known.
5652 const param_count = llvm_param_i;
5653 const return_count = llvm_ret_i;
5654
5628 // For some targets, Clang unconditionally adds some clobbers to all inline assembly.5655 // For some targets, Clang unconditionally adds some clobbers to all inline assembly.
5629 // While this is probably not strictly necessary, if we don't follow Clang's lead5656 // While this is probably not strictly necessary, if we don't follow Clang's lead
5630 // here then we may risk tripping LLVM bugs since anything not used by Clang tends5657 // here then we may risk tripping LLVM bugs since anything not used by Clang tends
...@@ -5682,7 +5709,7 @@ pub const FuncGen = struct {...@@ -5682,7 +5709,7 @@ pub const FuncGen = struct {
5682 const name = asm_source[name_start..i];5709 const name = asm_source[name_start..i];
5683 state = .start;5710 state = .start;
56845711
5685 const index = name_map.getIndex(name) orelse {5712 const index = name_map.get(name) orelse {
5686 // we should validate the assembly in Sema; by now it is too late5713 // we should validate the assembly in Sema; by now it is too late
5687 return self.todo("unknown input or output name: '{s}'", .{name});5714 return self.todo("unknown input or output name: '{s}'", .{name});
5688 };5715 };
...@@ -5693,12 +5720,20 @@ pub const FuncGen = struct {...@@ -5693,12 +5720,20 @@ pub const FuncGen = struct {
5693 }5720 }
5694 }5721 }
56955722
5696 const ret_ty = self.air.typeOfIndex(inst);5723 const ret_llvm_ty = switch (return_count) {
5697 const ret_llvm_ty = try self.dg.lowerType(ret_ty);5724 0 => self.context.voidType(),
5725 1 => llvm_ret_types[0],
5726 else => self.context.structType(
5727 llvm_ret_types.ptr,
5728 @intCast(c_uint, return_count),
5729 .False,
5730 ),
5731 };
5732
5698 const llvm_fn_ty = llvm.functionType(5733 const llvm_fn_ty = llvm.functionType(
5699 ret_llvm_ty,5734 ret_llvm_ty,
5700 llvm_param_types.ptr,5735 llvm_param_types.ptr,
5701 @intCast(c_uint, llvm_param_types.len),5736 @intCast(c_uint, param_count),
5702 .False,5737 .False,
5703 );5738 );
5704 const asm_fn = llvm.getInlineAsm(5739 const asm_fn = llvm.getInlineAsm(
...@@ -5715,18 +5750,40 @@ pub const FuncGen = struct {...@@ -5715,18 +5750,40 @@ pub const FuncGen = struct {
5715 const call = self.builder.buildCall(5750 const call = self.builder.buildCall(
5716 asm_fn,5751 asm_fn,
5717 llvm_param_values.ptr,5752 llvm_param_values.ptr,
5718 @intCast(c_uint, llvm_param_values.len),5753 @intCast(c_uint, param_count),
5719 .C,5754 .C,
5720 .Auto,5755 .Auto,
5721 "",5756 "",
5722 );5757 );
5723 for (llvm_param_attrs) |need_elem_ty, i| {5758 for (llvm_param_attrs[0..param_count]) |need_elem_ty, i| {
5724 if (need_elem_ty) {5759 if (need_elem_ty) {
5725 const elem_ty = llvm_param_types[i].getElementType();5760 const elem_ty = llvm_param_types[i].getElementType();
5726 llvm.setCallElemTypeAttr(call, i, elem_ty);5761 llvm.setCallElemTypeAttr(call, i, elem_ty);
5727 }5762 }
5728 }5763 }
5729 return call;5764
5765 var ret_val = call;
5766 llvm_ret_i = 0;
5767 for (outputs) |output, i| {
5768 if (llvm_ret_indirect[i]) continue;
5769
5770 const output_value = if (return_count > 1) b: {
5771 break :b self.builder.buildExtractValue(call, @intCast(c_uint, llvm_ret_i), "");
5772 } else call;
5773
5774 if (output != .none) {
5775 const output_ptr = try self.resolveInst(output);
5776 const output_ptr_ty = self.air.typeOf(output);
5777
5778 const store_inst = self.builder.buildStore(output_value, output_ptr);
5779 store_inst.setAlignment(output_ptr_ty.ptrAlignment(target));
5780 } else {
5781 ret_val = output_value;
5782 }
5783 llvm_ret_i += 1;
5784 }
5785
5786 return ret_val;
5730 }5787 }
57315788
5732 fn airIsNonNull(5789 fn airIsNonNull(
...@@ -9709,10 +9766,30 @@ fn errUnionErrorOffset(payload_ty: Type, target: std.Target) u1 {...@@ -9709,10 +9766,30 @@ fn errUnionErrorOffset(payload_ty: Type, target: std.Target) u1 {
9709 return @boolToInt(Type.anyerror.abiAlignment(target) <= payload_ty.abiAlignment(target));9766 return @boolToInt(Type.anyerror.abiAlignment(target) <= payload_ty.abiAlignment(target));
9710}9767}
97119768
9769/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a memory location
9770///
9771/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
9712fn constraintAllowsMemory(constraint: []const u8) bool {9772fn constraintAllowsMemory(constraint: []const u8) bool {
9713 return constraint[0] == 'm';9773 // TODO: This implementation is woefully incomplete.
9774 for (constraint) |byte| {
9775 switch (byte) {
9776 '=', '*', ',', '&' => {},
9777 'm', 'o', 'X', 'g' => return true,
9778 else => {},
9779 }
9780 } else return false;
9714}9781}
97159782
9783/// Returns true for asm constraint (e.g. "=*m", "=r") if it accepts a register
9784///
9785/// See also TargetInfo::validateOutputConstraint, AArch64TargetInfo::validateAsmConstraint, etc. in Clang
9716fn constraintAllowsRegister(constraint: []const u8) bool {9786fn constraintAllowsRegister(constraint: []const u8) bool {
9717 return constraint[0] != 'm';9787 // TODO: This implementation is woefully incomplete.
9788 for (constraint) |byte| {
9789 switch (byte) {
9790 '=', '*', ',', '&' => {},
9791 'm', 'o' => {},
9792 else => return true,
9793 }
9794 } else return false;
9718}9795}