authorgravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2021-12-07 19:35:46+01:00
committergravatar for mail@isaacfreund.comIsaac Freund <mail@isaacfreund.com> 2021-12-11 00:29:31+01:00
log7bb6393b593dcf4c8b929fc6b04b576e55f34607
treeb9deb3d1d49eb0818489b71e566e3a483fa4189d
parent175463d75dbde1e8e4c5a55159ab4e9446fd211c
signaturelock-open Commit is signed but in an unrecognized format.

stage1: implement @prefetch() builtin

7 files changed, 251 insertions(+), 0 deletions(-)

src/stage1/all_types.hpp+34
......@@ -898,6 +898,18 @@ struct AstNodeFnCallExpr {
898898 bool seen; // used by @compileLog
899899};
900900
901// Must be kept in sync with std.builtin.PrefetchOptions.Rw
902enum PrefetchRw {
903 PrefetchRwRead,
904 PrefetchRwWrite,
905};
906
907// Must be kept in sync with std.builtin.PrefetchOptions.Cache
908enum PrefetchCache {
909 PrefetchCacheInstruction,
910 PrefetchCacheData,
911};
912
901913struct AstNodeArrayAccessExpr {
902914 AstNode *array_ref_expr;
903915 AstNode *subscript;
......@@ -1818,6 +1830,7 @@ enum BuiltinFnId {
18181830 BuiltinFnIdReduce,
18191831 BuiltinFnIdMaximum,
18201832 BuiltinFnIdMinimum,
1833 BuiltinFnIdPrefetch,
18211834};
18221835
18231836struct BuiltinFnEntry {
......@@ -2021,6 +2034,7 @@ struct CodeGen {
20212034 LLVMValueRef return_err_fn;
20222035 LLVMValueRef wasm_memory_size;
20232036 LLVMValueRef wasm_memory_grow;
2037 LLVMValueRef prefetch;
20242038 LLVMTypeRef anyframe_fn_type;
20252039
20262040 // reminder: hash tables must be initialized before use
......@@ -2647,6 +2661,7 @@ enum Stage1ZirInstId : uint8_t {
26472661 Stage1ZirInstIdWasmMemorySize,
26482662 Stage1ZirInstIdWasmMemoryGrow,
26492663 Stage1ZirInstIdSrc,
2664 Stage1ZirInstIdPrefetch,
26502665};
26512666
26522667// ir_render_* functions in codegen.cpp consume Gen instructions and produce LLVM IR.
......@@ -2743,6 +2758,7 @@ enum Stage1AirInstId : uint8_t {
27432758 Stage1AirInstIdWasmMemorySize,
27442759 Stage1AirInstIdWasmMemoryGrow,
27452760 Stage1AirInstIdExtern,
2761 Stage1AirInstIdPrefetch,
27462762};
27472763
27482764struct Stage1ZirInst {
......@@ -3683,6 +3699,24 @@ struct Stage1ZirInstSrc {
36833699 Stage1ZirInst base;
36843700};
36853701
3702struct Stage1ZirInstPrefetch {
3703 Stage1ZirInst base;
3704
3705 Stage1ZirInst *ptr;
3706 Stage1ZirInst *options;
3707};
3708
3709struct Stage1AirInstPrefetch {
3710 Stage1AirInst base;
3711
3712 Stage1AirInst *ptr;
3713 PrefetchRw rw;
3714 // Must be in the range 0-3 inclusive
3715 uint8_t locality;
3716 PrefetchCache cache;
3717};
3718
3719
36863720struct Stage1ZirInstSlice {
36873721 Stage1ZirInst base;
36883722
src/stage1/astgen.cpp+44
......@@ -349,6 +349,8 @@ void destroy_instruction_src(Stage1ZirInst *inst) {
349349 return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstWasmMemoryGrow *>(inst));
350350 case Stage1ZirInstIdSrc:
351351 return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstSrc *>(inst));
352 case Stage1ZirInstIdPrefetch:
353 return heap::c_allocator.destroy(reinterpret_cast<Stage1ZirInstPrefetch *>(inst));
352354 }
353355 zig_unreachable();
354356}
......@@ -941,6 +943,10 @@ static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstSrc *) {
941943 return Stage1ZirInstIdSrc;
942944}
943945
946static constexpr Stage1ZirInstId ir_inst_id(Stage1ZirInstPrefetch *) {
947 return Stage1ZirInstIdPrefetch;
948}
949
944950template<typename T>
945951static T *ir_create_instruction(Stage1AstGen *ag, Scope *scope, AstNode *source_node) {
946952 T *special_instruction = heap::c_allocator.create<T>();
......@@ -2870,6 +2876,21 @@ static Stage1ZirInst *ir_build_src(Stage1AstGen *ag, Scope *scope, AstNode *sour
28702876 return &instruction->base;
28712877}
28722878
2879static Stage1ZirInst *ir_build_prefetch(Stage1AstGen *ag, Scope *scope, AstNode *source_node,
2880 Stage1ZirInst *ptr, Stage1ZirInst *options)
2881{
2882 Stage1ZirInstPrefetch *prefetch_instruction = ir_build_instruction<Stage1ZirInstPrefetch>(
2883 ag, scope, source_node);
2884 prefetch_instruction->ptr = ptr;
2885 prefetch_instruction->options = options;
2886
2887 ir_ref_instruction(ptr, ag->current_basic_block);
2888 ir_ref_instruction(options, ag->current_basic_block);
2889
2890 return &prefetch_instruction->base;
2891}
2892
2893
28732894static void ir_count_defers(Stage1AstGen *ag, Scope *inner_scope, Scope *outer_scope, size_t *results) {
28742895 results[ReturnKindUnconditional] = 0;
28752896 results[ReturnKindError] = 0;
......@@ -5416,6 +5437,29 @@ static Stage1ZirInst *astgen_builtin_fn_call(Stage1AstGen *ag, Scope *scope, Ast
54165437 Stage1ZirInst *src_inst = ir_build_src(ag, scope, node);
54175438 return ir_lval_wrap(ag, scope, src_inst, lval, result_loc);
54185439 }
5440 case BuiltinFnIdPrefetch:
5441 {
5442 ZigType *options_type = get_builtin_type(ag->codegen, "PrefetchOptions");
5443 Stage1ZirInst *options_type_inst = ir_build_const_type(ag, scope, node, options_type);
5444 ResultLocCast *result_loc_cast = ir_build_cast_result_loc(ag, options_type_inst, no_result_loc());
5445
5446 AstNode *ptr_node = node->data.fn_call_expr.params.at(0);
5447 Stage1ZirInst *ptr_value = astgen_node(ag, ptr_node, scope);
5448 if (ptr_value == ag->codegen->invalid_inst_src)
5449 return ptr_value;
5450
5451 AstNode *options_node = node->data.fn_call_expr.params.at(1);
5452 Stage1ZirInst *options_value = astgen_node_extra(ag, options_node,
5453 scope, LValNone, &result_loc_cast->base);
5454 if (options_value == ag->codegen->invalid_inst_src)
5455 return options_value;
5456
5457 Stage1ZirInst *casted_options_value = ir_build_implicit_cast(
5458 ag, scope, options_node, options_value, result_loc_cast);
5459
5460 Stage1ZirInst *ir_extern = ir_build_prefetch(ag, scope, node, ptr_value, casted_options_value);
5461 return ir_lval_wrap(ag, scope, ir_extern, lval, result_loc);
5462 }
54195463 }
54205464 zig_unreachable();
54215465}
src/stage1/codegen.cpp+67
......@@ -1139,6 +1139,24 @@ static LLVMValueRef gen_wasm_memory_grow(CodeGen *g) {
11391139 return g->wasm_memory_grow;
11401140}
11411141
1142static LLVMValueRef gen_prefetch(CodeGen *g) {
1143 if (g->prefetch)
1144 return g->prefetch;
1145
1146 // declare void @llvm.prefetch(i8*, i32, i32, i32)
1147 LLVMTypeRef param_types[] = {
1148 LLVMPointerType(LLVMInt8Type(), 0),
1149 LLVMInt32Type(),
1150 LLVMInt32Type(),
1151 LLVMInt32Type(),
1152 };
1153 LLVMTypeRef fn_type = LLVMFunctionType(LLVMVoidType(), param_types, 4, false);
1154 g->prefetch = LLVMAddFunction(g->module, "llvm.prefetch.p0i8", fn_type);
1155 assert(LLVMGetIntrinsicID(g->prefetch));
1156
1157 return g->prefetch;
1158}
1159
11421160static LLVMValueRef get_stacksave_fn_val(CodeGen *g) {
11431161 if (g->stacksave_fn_val)
11441162 return g->stacksave_fn_val;
......@@ -5899,6 +5917,52 @@ static LLVMValueRef ir_render_wasm_memory_grow(CodeGen *g, Stage1Air *executable
58995917 return val;
59005918}
59015919
5920static LLVMValueRef ir_render_prefetch(CodeGen *g, Stage1Air *executable, Stage1AirInstPrefetch *instruction) {
5921 static_assert(PrefetchRwRead == 0, "");
5922 static_assert(PrefetchRwWrite == 1, "");
5923 assert(instruction->rw == PrefetchRwRead || instruction->rw == PrefetchRwWrite);
5924
5925 assert(instruction->locality >= 0 && instruction->locality <= 3);
5926
5927 static_assert(PrefetchCacheInstruction == 0, "");
5928 static_assert(PrefetchCacheData == 1, "");
5929 assert(instruction->cache == PrefetchCacheData || instruction->cache == PrefetchCacheInstruction);
5930
5931 // LLVM fails during codegen of instruction cache prefetchs for these architectures.
5932 // This is an LLVM bug as the prefetch intrinsic should be a noop if not supported by the target.
5933 // To work around this, simply don't emit llvm.prefetch in this case.
5934 // See https://bugs.llvm.org/show_bug.cgi?id=21037
5935 if (instruction->cache == PrefetchCacheInstruction) {
5936 switch (g->zig_target->arch) {
5937 case ZigLLVM_x86:
5938 case ZigLLVM_x86_64:
5939 return nullptr;
5940 default:
5941 break;
5942 }
5943 }
5944
5945 // Another case of the same LLVM bug described above
5946 if (instruction->rw == PrefetchRwWrite && instruction->cache == PrefetchCacheInstruction) {
5947 switch (g->zig_target->arch) {
5948 case ZigLLVM_arm:
5949 return nullptr;
5950 default:
5951 break;
5952 }
5953
5954 }
5955
5956 LLVMValueRef params[] = {
5957 LLVMBuildBitCast(g->builder, ir_llvm_value(g, instruction->ptr), LLVMPointerType(LLVMInt8Type(), 0), ""),
5958 LLVMConstInt(LLVMInt32Type(), instruction->rw, false),
5959 LLVMConstInt(LLVMInt32Type(), instruction->locality, false),
5960 LLVMConstInt(LLVMInt32Type(), instruction->cache, false),
5961 };
5962 LLVMValueRef val = LLVMBuildCall(g->builder, gen_prefetch(g), params, 4, "");
5963 return val;
5964}
5965
59025966static LLVMValueRef ir_render_slice(CodeGen *g, Stage1Air *executable, Stage1AirInstSlice *instruction) {
59035967 Error err;
59045968
......@@ -7150,6 +7214,8 @@ static LLVMValueRef ir_render_instruction(CodeGen *g, Stage1Air *executable, Sta
71507214 return ir_render_wasm_memory_grow(g, executable, (Stage1AirInstWasmMemoryGrow *) instruction);
71517215 case Stage1AirInstIdExtern:
71527216 return ir_render_extern(g, executable, (Stage1AirInstExtern *) instruction);
7217 case Stage1AirInstIdPrefetch:
7218 return ir_render_prefetch(g, executable, (Stage1AirInstPrefetch *) instruction);
71537219 }
71547220 zig_unreachable();
71557221}
......@@ -9120,6 +9186,7 @@ static void define_builtin_fns(CodeGen *g) {
91209186 create_builtin_fn(g, BuiltinFnIdReduce, "reduce", 2);
91219187 create_builtin_fn(g, BuiltinFnIdMaximum, "maximum", 2);
91229188 create_builtin_fn(g, BuiltinFnIdMinimum, "minimum", 2);
9189 create_builtin_fn(g, BuiltinFnIdPrefetch, "prefetch", 2);
91239190}
91249191
91259192static const char *bool_to_str(bool b) {
src/stage1/ir.cpp+56
......@@ -467,6 +467,8 @@ void destroy_instruction_gen(Stage1AirInst *inst) {
467467 return heap::c_allocator.destroy(reinterpret_cast<Stage1AirInstWasmMemoryGrow *>(inst));
468468 case Stage1AirInstIdExtern:
469469 return heap::c_allocator.destroy(reinterpret_cast<Stage1AirInstExtern *>(inst));
470 case Stage1AirInstIdPrefetch:
471 return heap::c_allocator.destroy(reinterpret_cast<Stage1AirInstPrefetch *>(inst));
470472 }
471473 zig_unreachable();
472474}
......@@ -1115,6 +1117,10 @@ static constexpr Stage1AirInstId ir_inst_id(Stage1AirInstExtern *) {
11151117 return Stage1AirInstIdExtern;
11161118}
11171119
1120static constexpr Stage1AirInstId ir_inst_id(Stage1AirInstPrefetch *) {
1121 return Stage1AirInstIdPrefetch;
1122}
1123
11181124template<typename T>
11191125static T *ir_create_inst_gen(IrBuilderGen *irb, Scope *scope, AstNode *source_node) {
11201126 T *special_instruction = heap::c_allocator.create<T>();
......@@ -24853,6 +24859,52 @@ static Stage1AirInst *ir_analyze_instruction_src(IrAnalyze *ira, Stage1ZirInstSr
2485324859 return ir_const_move(ira, instruction->base.scope, instruction->base.source_node, result);
2485424860}
2485524861
24862static Stage1AirInst *ir_analyze_instruction_prefetch(IrAnalyze *ira, Stage1ZirInstPrefetch *instruction) {
24863 Stage1AirInst *ptr = instruction->ptr->child;
24864 if (type_is_invalid(ptr->value->type))
24865 return ira->codegen->invalid_inst_gen;
24866
24867 Stage1AirInst *raw_options_inst = instruction->options->child;
24868 if (type_is_invalid(raw_options_inst->value->type))
24869 return ira->codegen->invalid_inst_gen;
24870
24871 ZigType *options_type = get_builtin_type(ira->codegen, "PrefetchOptions");
24872 Stage1AirInst *options_inst = ir_implicit_cast(ira, raw_options_inst, options_type);
24873 if (type_is_invalid(options_inst->value->type))
24874 return ira->codegen->invalid_inst_gen;
24875
24876 ZigValue *options_val = ir_resolve_const(ira, options_inst, UndefBad);
24877 if (options_val == nullptr)
24878 return ira->codegen->invalid_inst_gen;
24879
24880 ZigValue *rw_val = get_const_field(ira, options_inst->source_node, options_val, "rw", 0);
24881 if (rw_val == nullptr)
24882 return ira->codegen->invalid_inst_gen;
24883 PrefetchRw rw = (PrefetchRw)bigint_as_u8(&rw_val->data.x_enum_tag);
24884
24885 ZigValue *locality_val = get_const_field(ira, options_inst->source_node, options_val, "locality", 1);
24886 if (locality_val == nullptr)
24887 return ira->codegen->invalid_inst_gen;
24888 uint8_t locality = bigint_as_u8(&locality_val->data.x_bigint);
24889 assert(locality <= 3);
24890
24891 ZigValue *cache_val = get_const_field(ira, options_inst->source_node, options_val, "cache", 2);
24892 if (cache_val == nullptr)
24893 return ira->codegen->invalid_inst_gen;
24894 PrefetchCache cache = (PrefetchCache)bigint_as_u8(&cache_val->data.x_enum_tag);
24895
24896 Stage1AirInstPrefetch *air_instruction = ir_build_inst_void<Stage1AirInstPrefetch>(&ira->new_irb,
24897 instruction->base.scope, instruction->base.source_node);
24898 air_instruction->ptr = ptr;
24899 air_instruction->rw = rw;
24900 air_instruction->locality = locality;
24901 air_instruction->cache = cache;
24902
24903 ir_ref_inst_gen(ptr);
24904
24905 return &air_instruction->base;
24906}
24907
2485624908static Stage1AirInst *ir_analyze_instruction_base(IrAnalyze *ira, Stage1ZirInst *instruction) {
2485724909 switch (instruction->id) {
2485824910 case Stage1ZirInstIdInvalid:
......@@ -25138,6 +25190,8 @@ static Stage1AirInst *ir_analyze_instruction_base(IrAnalyze *ira, Stage1ZirInst
2513825190 return ir_analyze_instruction_wasm_memory_grow(ira, (Stage1ZirInstWasmMemoryGrow *)instruction);
2513925191 case Stage1ZirInstIdSrc:
2514025192 return ir_analyze_instruction_src(ira, (Stage1ZirInstSrc *)instruction);
25193 case Stage1ZirInstIdPrefetch:
25194 return ir_analyze_instruction_prefetch(ira, (Stage1ZirInstPrefetch *)instruction);
2514125195 }
2514225196 zig_unreachable();
2514325197}
......@@ -25305,6 +25359,7 @@ bool ir_inst_gen_has_side_effects(Stage1AirInst *instruction) {
2530525359 case Stage1AirInstIdSpillBegin:
2530625360 case Stage1AirInstIdWasmMemoryGrow:
2530725361 case Stage1AirInstIdExtern:
25362 case Stage1AirInstIdPrefetch:
2530825363 return true;
2530925364
2531025365 case Stage1AirInstIdPhi:
......@@ -25444,6 +25499,7 @@ bool ir_inst_src_has_side_effects(Stage1ZirInst *instruction) {
2544425499 case Stage1ZirInstIdAwait:
2544525500 case Stage1ZirInstIdSpillBegin:
2544625501 case Stage1ZirInstIdWasmMemoryGrow:
25502 case Stage1ZirInstIdPrefetch:
2544725503 return true;
2544825504
2544925505 case Stage1ZirInstIdPhi:
src/stage1/ir_print.cpp+22
......@@ -371,6 +371,8 @@ const char* ir_inst_src_type_str(Stage1ZirInstId id) {
371371 return "SrcWasmMemoryGrow";
372372 case Stage1ZirInstIdSrc:
373373 return "SrcSrc";
374 case Stage1ZirInstIdPrefetch:
375 return "SrcPrefetch";
374376 }
375377 zig_unreachable();
376378}
......@@ -559,6 +561,8 @@ const char* ir_inst_gen_type_str(Stage1AirInstId id) {
559561 return "GenWasmMemoryGrow";
560562 case Stage1AirInstIdExtern:
561563 return "GenExtern";
564 case Stage1AirInstIdPrefetch:
565 return "GenPrefetch";
562566 }
563567 zig_unreachable();
564568}
......@@ -2436,6 +2440,18 @@ static void ir_print_extern(IrPrintSrc *irp, Stage1ZirInstExtern *instruction) {
24362440 fprintf(irp->f, ")");
24372441}
24382442
2443static void ir_print_prefetch(IrPrintSrc *irp, Stage1ZirInstPrefetch *instruction) {
2444 fprintf(irp->f, "@prefetch(");
2445 ir_print_other_inst_src(irp, instruction->ptr);
2446 fprintf(irp->f, ",");
2447 ir_print_other_inst_src(irp, instruction->options);
2448 fprintf(irp->f, ")");
2449}
2450
2451static void ir_print_prefetch(IrPrintGen *irp, Stage1AirInstPrefetch *instruction) {
2452 fprintf(irp->f, "@prefetch(...)");
2453}
2454
24392455static void ir_print_error_return_trace(IrPrintSrc *irp, Stage1ZirInstErrorReturnTrace *instruction) {
24402456 fprintf(irp->f, "@errorReturnTrace(");
24412457 switch (instruction->optional) {
......@@ -3108,6 +3124,9 @@ static void ir_print_inst_src(IrPrintSrc *irp, Stage1ZirInst *instruction, bool
31083124 case Stage1ZirInstIdSrc:
31093125 ir_print_builtin_src(irp, (Stage1ZirInstSrc *)instruction);
31103126 break;
3127 case Stage1ZirInstIdPrefetch:
3128 ir_print_prefetch(irp, (Stage1ZirInstPrefetch *)instruction);
3129 break;
31113130 }
31123131 fprintf(irp->f, "\n");
31133132}
......@@ -3387,6 +3406,9 @@ static void ir_print_inst_gen(IrPrintGen *irp, Stage1AirInst *instruction, bool
33873406 case Stage1AirInstIdExtern:
33883407 ir_print_extern(irp, (Stage1AirInstExtern *)instruction);
33893408 break;
3409 case Stage1AirInstIdPrefetch:
3410 ir_print_prefetch(irp, (Stage1AirInstPrefetch *)instruction);
3411 break;
33903412
33913413 }
33923414 fprintf(irp->f, "\n");
test/behavior.zig+1
......@@ -169,6 +169,7 @@ test {
169169 _ = @import("behavior/optional_stage1.zig");
170170 _ = @import("behavior/pointers_stage1.zig");
171171 _ = @import("behavior/popcount_stage1.zig");
172 _ = @import("behavior/prefetch.zig");
172173 _ = @import("behavior/ptrcast_stage1.zig");
173174 _ = @import("behavior/reflection.zig");
174175 _ = @import("behavior/saturating_arithmetic_stage1.zig");
test/behavior/prefetch.zig created+27
......@@ -0,0 +1,27 @@
1const std = @import("std");
2
3test "@prefetch()" {
4 var a: u32 = 42;
5
6 @prefetch(&a, .{});
7
8 @prefetch(&a, .{ .rw = .read, .locality = 3, .cache = .data });
9 @prefetch(&a, .{ .rw = .read, .locality = 2, .cache = .data });
10 @prefetch(&a, .{ .rw = .read, .locality = 1, .cache = .data });
11 @prefetch(&a, .{ .rw = .read, .locality = 0, .cache = .data });
12
13 @prefetch(&a, .{ .rw = .write, .locality = 3, .cache = .data });
14 @prefetch(&a, .{ .rw = .write, .locality = 2, .cache = .data });
15 @prefetch(&a, .{ .rw = .write, .locality = 1, .cache = .data });
16 @prefetch(&a, .{ .rw = .write, .locality = 0, .cache = .data });
17
18 @prefetch(&a, .{ .rw = .read, .locality = 3, .cache = .instruction });
19 @prefetch(&a, .{ .rw = .read, .locality = 2, .cache = .instruction });
20 @prefetch(&a, .{ .rw = .read, .locality = 1, .cache = .instruction });
21 @prefetch(&a, .{ .rw = .read, .locality = 0, .cache = .instruction });
22
23 @prefetch(&a, .{ .rw = .write, .locality = 3, .cache = .instruction });
24 @prefetch(&a, .{ .rw = .write, .locality = 2, .cache = .instruction });
25 @prefetch(&a, .{ .rw = .write, .locality = 1, .cache = .instruction });
26 @prefetch(&a, .{ .rw = .write, .locality = 0, .cache = .instruction });
27}